diff --git a/src/app/api/generate/route.ts b/src/app/api/generate/route.ts index 47e100c4..6d79fa68 100644 --- a/src/app/api/generate/route.ts +++ b/src/app/api/generate/route.ts @@ -19,6 +19,8 @@ const MODEL_MAP: Record = { interface MultiProviderGenerateRequest extends GenerateRequest { selectedModel?: SelectedModel; parameters?: Record; + /** Dynamic inputs from schema-based connections (e.g., image_url, tail_image_url, prompt) */ + dynamicInputs?: Record; } /** @@ -272,25 +274,35 @@ async function generateWithReplicate( }; } + const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; + // Build input for the prediction const predictionInput: Record = { - prompt: input.prompt, ...input.parameters, }; + // Add dynamic inputs if provided (these come from schema-mapped connections) + if (hasDynamicInputs) { + Object.assign(predictionInput, input.dynamicInputs); + console.log(`[API:${requestId}] Using dynamic inputs:`, Object.keys(input.dynamicInputs!).join(", ")); + } else { + // Fallback to legacy behavior + predictionInput.prompt = input.prompt; + + // Add image input if provided (for img2img workflows) + // Note: Different Replicate models use different parameter names + // Using 'image' as it's most common for img2img models + if (input.images && input.images.length > 0) { + predictionInput.image = input.images[0]; + console.log(`[API:${requestId}] Added image input to prediction (${input.images[0].substring(0, 50)}...)`); + } + } + // Log parameters being passed if (input.parameters && Object.keys(input.parameters).length > 0) { console.log(`[API:${requestId}] Custom parameters:`, JSON.stringify(input.parameters)); } - // Add image input if provided (for img2img workflows) - // Note: Different Replicate models use different parameter names - // Using 'image' as it's most common for img2img models - if (input.images && input.images.length > 0) { - predictionInput.image = input.images[0]; - console.log(`[API:${requestId}] Added image input to prediction (${input.images[0].substring(0, 50)}...)`); - } - // Create a prediction console.log(`[API:${requestId}] Creating Replicate prediction...`); const createResponse = await fetch(`${REPLICATE_API_BASE}/predictions`, { @@ -474,28 +486,39 @@ async function generateWithFal( console.log(`[API:${requestId}] - Model: ${input.model.id}`); console.log(`[API:${requestId}] - Prompt length: ${input.prompt.length} chars`); console.log(`[API:${requestId}] - Images count: ${input.images?.length || 0}`); + console.log(`[API:${requestId}] - Dynamic inputs: ${input.dynamicInputs ? Object.keys(input.dynamicInputs).join(", ") : "none"}`); console.log(`[API:${requestId}] - API key: ${apiKey ? "provided" : "not provided (using rate-limited access)"}`); const modelId = input.model.id; + const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; // Build request body + // If we have dynamic inputs, they take precedence (they already contain prompt, image_url, etc.) const requestBody: Record = { - prompt: input.prompt, ...input.parameters, }; + // Add dynamic inputs if provided (these come from schema-mapped connections) + if (hasDynamicInputs) { + Object.assign(requestBody, input.dynamicInputs); + console.log(`[API:${requestId}] Using dynamic inputs:`, Object.keys(input.dynamicInputs!).join(", ")); + } else { + // Fallback to legacy behavior: use prompt and images array + requestBody.prompt = input.prompt; + + // Add image_url if provided (for img2img workflows) + // fal.ai accepts both URLs and data URIs in this field + if (input.images && input.images.length > 0) { + requestBody.image_url = input.images[0]; + console.log(`[API:${requestId}] Added image_url to request (${input.images[0].substring(0, 50)}...)`); + } + } + // Log parameters being passed if (input.parameters && Object.keys(input.parameters).length > 0) { console.log(`[API:${requestId}] Custom parameters:`, JSON.stringify(input.parameters)); } - // Add image_url if provided (for img2img workflows) - // fal.ai accepts both URLs and data URIs in this field - if (input.images && input.images.length > 0) { - requestBody.image_url = input.images[0]; - console.log(`[API:${requestId}] Added image_url to request (${input.images[0].substring(0, 50)}...)`); - } - // Build headers const headers: Record = { "Content-Type": "application/json", @@ -639,9 +662,12 @@ export async function POST(request: NextRequest) { useGoogleSearch, selectedModel, parameters, + dynamicInputs, } = body; - if (!prompt) { + // Prompt is required unless provided via dynamicInputs + const hasPrompt = prompt || (dynamicInputs && dynamicInputs.prompt); + if (!hasPrompt) { console.error(`[API:${requestId}] Validation failed: missing prompt`); return NextResponse.json( { @@ -689,6 +715,31 @@ export async function POST(request: NextRequest) { } } + // Process dynamicInputs: convert large images in image-type inputs to URLs + const processedDynamicInputs: Record | undefined = dynamicInputs + ? { ...dynamicInputs } + : undefined; + + if (processedDynamicInputs) { + // Image input names that might contain base64 data + const imageInputNames = ["image", "image_url", "tail_image_url", "first_frame", "last_frame", + "start_image", "end_image", "reference_image", "init_image", "mask_image", "control_image"]; + + for (const key of Object.keys(processedDynamicInputs)) { + const value = processedDynamicInputs[key]; + // Check if this is an image input with base64 data + if (imageInputNames.some(name => key.toLowerCase().includes(name.toLowerCase())) && + value && value.startsWith("data:image")) { + if (shouldUseImageUrl(value)) { + const { url, id } = uploadImageForUrl(value, baseUrl); + uploadedImageIds.push(id); + processedDynamicInputs[key] = url; + console.log(`[API:${requestId}] Converted large dynamic input '${key}' to URL: ${url}`); + } + } + } + } + try { // Build generation input const genInput: GenerationInput = { @@ -699,9 +750,10 @@ export async function POST(request: NextRequest) { capabilities: ["text-to-image"], description: null, }, - prompt, + prompt: prompt || "", images: processedImages, parameters, + dynamicInputs: processedDynamicInputs, }; const result = await generateWithReplicate(requestId, replicateApiKey, genInput); @@ -777,6 +829,31 @@ export async function POST(request: NextRequest) { } } + // Process dynamicInputs: convert large images in image-type inputs to URLs + const processedDynamicInputs: Record | undefined = dynamicInputs + ? { ...dynamicInputs } + : undefined; + + if (processedDynamicInputs) { + // Image input names that might contain base64 data + const imageInputNames = ["image_url", "tail_image_url", "first_frame", "last_frame", + "start_image", "end_image", "reference_image", "init_image", "mask_image", "control_image"]; + + for (const key of Object.keys(processedDynamicInputs)) { + const value = processedDynamicInputs[key]; + // Check if this is an image input with base64 data + if (imageInputNames.some(name => key.toLowerCase().includes(name.toLowerCase())) && + value && value.startsWith("data:image")) { + if (shouldUseImageUrl(value)) { + const { url, id } = uploadImageForUrl(value, baseUrl); + uploadedImageIds.push(id); + processedDynamicInputs[key] = url; + console.log(`[API:${requestId}] Converted large dynamic input '${key}' to URL: ${url}`); + } + } + } + } + try { // Build generation input const genInput: GenerationInput = { @@ -787,9 +864,10 @@ export async function POST(request: NextRequest) { capabilities: ["text-to-image"], description: null, }, - prompt, + prompt: prompt || "", images: processedImages, parameters, + dynamicInputs: processedDynamicInputs, }; const result = await generateWithFal(requestId, falApiKey, genInput); diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 36f5c079..3b48e875 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -57,14 +57,32 @@ const edgeTypes: EdgeTypes = { // Connection validation rules // - Image handles (green) can only connect to image handles // - Text handles (blue) can only connect to text handles +// Helper to determine handle type from handle ID +// For dynamic handles, we use naming convention: image inputs contain "image", text inputs are "prompt" or "negative_prompt" +const getHandleType = (handleId: string | null | undefined): "image" | "text" | null => { + if (!handleId) return null; + // Standard handles + if (handleId === "image" || handleId === "text") return handleId; + // Dynamic handles - check naming patterns + if (handleId.includes("image") || handleId.includes("frame")) return "image"; + if (handleId === "prompt" || handleId === "negative_prompt" || handleId.includes("prompt")) return "text"; + return null; +}; + +// Connection validation: ensures type matching between source and target handles +// - Image handles can connect to image handles +// - Text handles can connect to text handles // - NanoBanana image input accepts multiple connections // - All other inputs accept only one connection const isValidConnection = (connection: Edge | Connection): boolean => { const sourceHandle = connection.sourceHandle; const targetHandle = connection.targetHandle; + const sourceType = getHandleType(sourceHandle); + const targetType = getHandleType(targetHandle); + // Strict type matching: image <-> image, text <-> text - if (sourceHandle === "image" && targetHandle !== "image") { + if (sourceType === "image" && targetType !== "image") { logger.warn('connection.validation', 'Connection validation failed: type mismatch', { source: connection.source, target: connection.target, @@ -74,7 +92,7 @@ const isValidConnection = (connection: Edge | Connection): boolean => { }); return false; } - if (sourceHandle === "text" && targetHandle !== "text") { + if (sourceType === "text" && targetType !== "text") { logger.warn('connection.validation', 'Connection validation failed: type mismatch', { source: connection.source, target: connection.target, diff --git a/src/components/nodes/GenerateImageNode.tsx b/src/components/nodes/GenerateImageNode.tsx index 92ac1a68..24ade0f3 100644 --- a/src/components/nodes/GenerateImageNode.tsx +++ b/src/components/nodes/GenerateImageNode.tsx @@ -335,23 +335,54 @@ export function GenerateImageNode({ id, data, selected }: NodeProps - {/* Image input - accepts multiple connections */} - - {/* Text input - single connection */} - + {/* Dynamic input handles based on model schema (external providers only) */} + {!isGeminiProvider && nodeData.inputSchema && nodeData.inputSchema.length > 0 ? ( + // Render handles from schema + nodeData.inputSchema.map((input, index) => { + const total = nodeData.inputSchema!.length; + const topPercent = ((index + 1) / (total + 1)) * 100; + return ( +
+ + {/* Handle label */} +
+ {input.label} + {input.required && *} +
+
+ ); + }) + ) : ( + // Default handles for Gemini or when no schema + <> + + + + )} {/* Image output */} - {/* Image input - accepts multiple connections */} - - {/* Text input - single connection */} - - {/* Video output - we use "image" handle type for now since the connection system expects image/text */} + {/* Dynamic input handles based on model schema */} + {nodeData.inputSchema && nodeData.inputSchema.length > 0 ? ( + // Render handles from schema + nodeData.inputSchema.map((input, index) => { + const total = nodeData.inputSchema!.length; + const topPercent = ((index + 1) / (total + 1)) * 100; + return ( +
+ + {/* Handle label */} +
+ {input.label} + {input.required && *} +
+
+ ); + }) + ) : ( + // Default handles when no schema + <> + + + + )} + {/* Video output */} ; + /** Dynamic inputs mapped from schema (e.g., { "image_url": "data:...", "tail_image_url": "data:..." }) */ + dynamicInputs?: Record; } /** diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index 06eefc43..2d54309f 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -115,7 +115,7 @@ interface WorkflowStore { // Helpers getNodeById: (id: string) => WorkflowNode | undefined; - getConnectedInputs: (nodeId: string) => { images: string[]; text: string | null }; + getConnectedInputs: (nodeId: string) => { images: string[]; text: string | null; dynamicInputs: Record }; validateWorkflow: () => { valid: boolean; errors: string[] }; // Global Image History @@ -816,6 +816,37 @@ export const useWorkflowStore = create((set, get) => ({ const { edges, nodes } = get(); const images: string[] = []; let text: string | null = null; + const dynamicInputs: Record = {}; + + // Helper to determine if a handle ID is an image type + const isImageHandle = (handleId: string | null | undefined): boolean => { + if (!handleId) return false; + return handleId === "image" || handleId.includes("image") || handleId.includes("frame"); + }; + + // Helper to determine if a handle ID is a text type + const isTextHandle = (handleId: string | null | undefined): boolean => { + if (!handleId) return false; + return handleId === "text" || handleId.includes("prompt"); + }; + + // Helper to extract output from source node + const getSourceOutput = (sourceNode: WorkflowNode): { type: "image" | "text"; value: string | null } => { + if (sourceNode.type === "imageInput") { + return { type: "image", value: (sourceNode.data as ImageInputNodeData).image }; + } else if (sourceNode.type === "annotation") { + return { type: "image", value: (sourceNode.data as AnnotationNodeData).outputImage }; + } else if (sourceNode.type === "nanoBanana") { + return { type: "image", value: (sourceNode.data as NanoBananaNodeData).outputImage }; + } else if (sourceNode.type === "generateVideo") { + return { type: "image", value: (sourceNode.data as GenerateVideoNodeData).outputVideo }; + } else if (sourceNode.type === "prompt") { + return { type: "text", value: (sourceNode.data as PromptNodeData).prompt }; + } else if (sourceNode.type === "llmGenerate") { + return { type: "text", value: (sourceNode.data as LLMGenerateNodeData).outputText }; + } + return { type: "image", value: null }; + }; edges .filter((edge) => edge.target === nodeId) @@ -824,35 +855,24 @@ export const useWorkflowStore = create((set, get) => ({ if (!sourceNode) return; const handleId = edge.targetHandle; + const { value } = getSourceOutput(sourceNode); - if (handleId === "image" || !handleId) { - // Get image from source node - collect all connected images - if (sourceNode.type === "imageInput") { - const sourceImage = (sourceNode.data as ImageInputNodeData).image; - if (sourceImage) images.push(sourceImage); - } else if (sourceNode.type === "annotation") { - const sourceImage = (sourceNode.data as AnnotationNodeData).outputImage; - if (sourceImage) images.push(sourceImage); - } else if (sourceNode.type === "nanoBanana") { - const sourceImage = (sourceNode.data as NanoBananaNodeData).outputImage; - if (sourceImage) images.push(sourceImage); - } else if (sourceNode.type === "generateVideo") { - // Video output can be used as input to downstream nodes - const sourceVideo = (sourceNode.data as GenerateVideoNodeData).outputVideo; - if (sourceVideo) images.push(sourceVideo); - } + if (!value) return; + + // Store in dynamic inputs if handle ID is specific (not just "image" or "text") + if (handleId && handleId !== "image" && handleId !== "text") { + dynamicInputs[handleId] = value; } - if (handleId === "text") { - if (sourceNode.type === "prompt") { - text = (sourceNode.data as PromptNodeData).prompt; - } else if (sourceNode.type === "llmGenerate") { - text = (sourceNode.data as LLMGenerateNodeData).outputText; - } + // Also populate legacy arrays for backward compatibility + if (isImageHandle(handleId) || !handleId) { + images.push(value); + } else if (isTextHandle(handleId)) { + text = value; } }); - return { images, text }; + return { images, text, dynamicInputs }; }, validateWorkflow: () => { @@ -1041,9 +1061,11 @@ export const useWorkflowStore = create((set, get) => ({ break; case "nanoBanana": { - const { images, text } = getConnectedInputs(node.id); + const { images, text, dynamicInputs } = getConnectedInputs(node.id); - if (!text) { + // For dynamic inputs, check if we have at least a prompt + const promptText = text || dynamicInputs.prompt || null; + if (!promptText) { logger.error('node.error', 'nanoBanana node missing text input', { nodeId: node.id, }); @@ -1058,7 +1080,7 @@ export const useWorkflowStore = create((set, get) => ({ updateNodeData(node.id, { inputImages: images, - inputPrompt: text, + inputPrompt: promptText, status: "loading", error: null, }); @@ -1069,13 +1091,14 @@ export const useWorkflowStore = create((set, get) => ({ const requestPayload = { images, - prompt: text, + prompt: promptText, aspectRatio: nodeData.aspectRatio, resolution: nodeData.resolution, model: nodeData.model, useGoogleSearch: nodeData.useGoogleSearch, selectedModel: nodeData.selectedModel, parameters: nodeData.parameters, + dynamicInputs, // Pass dynamic inputs for schema-mapped connections }; // Build headers with API keys for external providers @@ -1102,7 +1125,7 @@ export const useWorkflowStore = create((set, get) => ({ aspectRatio: nodeData.aspectRatio, resolution: nodeData.resolution, imageCount: images.length, - prompt: text, + prompt: promptText, }); const response = await fetch("/api/generate", { @@ -1148,7 +1171,7 @@ export const useWorkflowStore = create((set, get) => ({ get().addToGlobalHistory({ image: result.image, timestamp, - prompt: text, + prompt: promptText, aspectRatio: nodeData.aspectRatio, model: nodeData.model, }); @@ -1157,7 +1180,7 @@ export const useWorkflowStore = create((set, get) => ({ const newHistoryItem = { id: imageId, timestamp, - prompt: text, + prompt: promptText, aspectRatio: nodeData.aspectRatio, model: nodeData.model, }; @@ -1237,15 +1260,17 @@ export const useWorkflowStore = create((set, get) => ({ } case "generateVideo": { - const { images, text } = getConnectedInputs(node.id); + const { images, text, dynamicInputs } = getConnectedInputs(node.id); - if (!text) { - logger.error('node.error', 'generateVideo node missing text input', { + // For dynamic inputs, check if we have at least a prompt + const hasPrompt = text || dynamicInputs.prompt || dynamicInputs.negative_prompt; + if (!hasPrompt && images.length === 0) { + logger.error('node.error', 'generateVideo node missing inputs', { nodeId: node.id, }); updateNodeData(node.id, { status: "error", - error: "Missing text input", + error: "Missing required inputs", }); set({ isRunning: false, currentNodeId: null }); await logger.endSession(); @@ -1282,6 +1307,7 @@ export const useWorkflowStore = create((set, get) => ({ prompt: text, selectedModel: nodeData.selectedModel, parameters: nodeData.parameters, + dynamicInputs, // Pass dynamic inputs for schema-mapped connections }; // Build headers with API keys for external providers