diff --git a/src/app/api/generate/route.ts b/src/app/api/generate/route.ts index c2fc44b0..ebc31bad 100644 --- a/src/app/api/generate/route.ts +++ b/src/app/api/generate/route.ts @@ -365,7 +365,7 @@ async function generateWithReplicate( }; } - // Extract output image(s) + // Extract output const output = currentPrediction.output; if (!output) { return { @@ -380,36 +380,60 @@ async function generateWithReplicate( if (outputUrls.length === 0) { return { success: false, - error: "No output images from prediction", + error: "No output from prediction", }; } - // Fetch the first output image and convert to base64 - const imageUrl = outputUrls[0]; - console.log(`[API:${requestId}] Fetching output image from: ${imageUrl}`); - const imageResponse = await fetch(imageUrl); + // Fetch the first output and convert to base64 + const mediaUrl = outputUrls[0]; + console.log(`[API:${requestId}] Fetching output from: ${mediaUrl}`); + const mediaResponse = await fetch(mediaUrl); - if (!imageResponse.ok) { + if (!mediaResponse.ok) { return { success: false, - error: `Failed to fetch output image: ${imageResponse.status}`, + error: `Failed to fetch output: ${mediaResponse.status}`, }; } - const imageArrayBuffer = await imageResponse.arrayBuffer(); - const imageBase64 = Buffer.from(imageArrayBuffer).toString("base64"); - // Determine MIME type from response - const contentType = imageResponse.headers.get("content-type") || "image/png"; + const contentType = mediaResponse.headers.get("content-type") || "image/png"; + const isVideo = contentType.startsWith("video/"); + + const mediaArrayBuffer = await mediaResponse.arrayBuffer(); + const mediaSizeBytes = mediaArrayBuffer.byteLength; + const mediaSizeMB = mediaSizeBytes / (1024 * 1024); + + // Log warning for large files + if (mediaSizeMB > 10) { + console.warn(`[API:${requestId}] Large output file: ${mediaSizeMB.toFixed(2)}MB`); + } + + // For very large videos (>20MB), return URL directly instead of base64 + if (isVideo && mediaSizeMB > 20) { + console.log(`[API:${requestId}] Replicate video generation successful (URL only, too large for base64)`); + return { + success: true, + outputs: [ + { + type: "video", + data: mediaUrl, // Return URL directly for very large videos + url: mediaUrl, + }, + ], + }; + } - console.log(`[API:${requestId}] Replicate generation successful`); + const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); + + console.log(`[API:${requestId}] Replicate ${isVideo ? "video" : "image"} generation successful`); return { success: true, outputs: [ { - type: "image", - data: `data:${contentType};base64,${imageBase64}`, - url: imageUrl, + type: isVideo ? "video" : "image", + data: `data:${contentType};base64,${mediaBase64}`, + url: mediaUrl, }, ], }; @@ -470,51 +494,85 @@ async function generateWithFal( const result = await response.json(); - // fal.ai response typically has "images" array with url field - // or "image" object with url field depending on the model - let imageUrl: string | null = null; - - if (result.images && Array.isArray(result.images) && result.images.length > 0) { - imageUrl = result.images[0].url; + // fal.ai response can have different structures: + // - images: array with url field (image models) + // - image: object with url field (image models) + // - video: object with url field (video models) + // - output: string URL (some models) + let mediaUrl: string | null = null; + let isVideoModel = false; + + // Check for video output first (video models) + if (result.video && result.video.url) { + mediaUrl = result.video.url; + isVideoModel = true; + console.log(`[API:${requestId}] Found video URL in response`); + } else if (result.images && Array.isArray(result.images) && result.images.length > 0) { + mediaUrl = result.images[0].url; } else if (result.image && result.image.url) { - imageUrl = result.image.url; + mediaUrl = result.image.url; } else if (result.output && typeof result.output === "string") { // Some models return URL directly in output - imageUrl = result.output; + mediaUrl = result.output; } - if (!imageUrl) { + if (!mediaUrl) { + console.error(`[API:${requestId}] No media URL found in fal.ai response:`, JSON.stringify(result, null, 2)); return { success: false, - error: "No image URL in response", + error: "No media URL in response", }; } - // Fetch the image and convert to base64 - console.log(`[API:${requestId}] Fetching output image from: ${imageUrl}`); - const imageResponse = await fetch(imageUrl); + // Fetch the media and convert to base64 + console.log(`[API:${requestId}] Fetching output from: ${mediaUrl}`); + const mediaResponse = await fetch(mediaUrl); - if (!imageResponse.ok) { + if (!mediaResponse.ok) { return { success: false, - error: `Failed to fetch output image: ${imageResponse.status}`, + error: `Failed to fetch output: ${mediaResponse.status}`, }; } - const imageArrayBuffer = await imageResponse.arrayBuffer(); - const imageBase64 = Buffer.from(imageArrayBuffer).toString("base64"); - // Determine MIME type from response - const contentType = imageResponse.headers.get("content-type") || "image/png"; + const contentType = mediaResponse.headers.get("content-type") || (isVideoModel ? "video/mp4" : "image/png"); + const isVideo = contentType.startsWith("video/") || isVideoModel; - console.log(`[API:${requestId}] fal.ai generation successful`); + const mediaArrayBuffer = await mediaResponse.arrayBuffer(); + const mediaSizeBytes = mediaArrayBuffer.byteLength; + const mediaSizeMB = mediaSizeBytes / (1024 * 1024); + + // Log warning for large files + if (mediaSizeMB > 10) { + console.warn(`[API:${requestId}] Large output file: ${mediaSizeMB.toFixed(2)}MB`); + } + + // For very large videos (>20MB), return URL directly instead of base64 + if (isVideo && mediaSizeMB > 20) { + console.log(`[API:${requestId}] fal.ai video generation successful (URL only, too large for base64)`); + return { + success: true, + outputs: [ + { + type: "video", + data: mediaUrl, // Return URL directly for very large videos + url: mediaUrl, + }, + ], + }; + } + + const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); + + console.log(`[API:${requestId}] fal.ai ${isVideo ? "video" : "image"} generation successful`); return { success: true, outputs: [ { - type: "image", - data: `data:${contentType};base64,${imageBase64}`, - url: imageUrl, + type: isVideo ? "video" : "image", + data: `data:${contentType};base64,${mediaBase64}`, + url: mediaUrl, }, ], }; @@ -614,21 +672,34 @@ export async function POST(request: NextRequest) { ); } - // Return first output image - const outputImage = result.outputs?.[0]?.data; - if (!outputImage) { + // Return first output (image or video) + const output = result.outputs?.[0]; + if (!output?.data) { return NextResponse.json( { success: false, - error: "No image in generation output", + error: "No output in generation result", }, { status: 500 } ); } + // Return appropriate fields based on output type + if (output.type === "video") { + // Check if data is a URL (for large videos) or base64 + const isUrl = output.data.startsWith("http"); + return NextResponse.json({ + success: true, + video: isUrl ? undefined : output.data, + videoUrl: isUrl ? output.data : undefined, + contentType: "video", + }); + } + return NextResponse.json({ success: true, - image: outputImage, + image: output.data, + contentType: "image", }); } finally { // Clean up uploaded images @@ -689,21 +760,34 @@ export async function POST(request: NextRequest) { ); } - // Return first output image - const outputImage = result.outputs?.[0]?.data; - if (!outputImage) { + // Return first output (image or video) + const output = result.outputs?.[0]; + if (!output?.data) { return NextResponse.json( { success: false, - error: "No image in generation output", + error: "No output in generation result", }, { status: 500 } ); } + // Return appropriate fields based on output type + if (output.type === "video") { + // Check if data is a URL (for large videos) or base64 + const isUrl = output.data.startsWith("http"); + return NextResponse.json({ + success: true, + video: isUrl ? undefined : output.data, + videoUrl: isUrl ? output.data : undefined, + contentType: "video", + }); + } + return NextResponse.json({ success: true, - image: outputImage, + image: output.data, + contentType: "image", }); } finally { // Clean up uploaded images diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index cc16661a..162a3d54 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -1228,6 +1228,171 @@ export const useWorkflowStore = create((set, get) => ({ break; } + case "generateVideo": { + const { images, text } = getConnectedInputs(node.id); + + if (!text) { + logger.error('node.error', 'generateVideo node missing text input', { + nodeId: node.id, + }); + updateNodeData(node.id, { + status: "error", + error: "Missing text input", + }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + + const nodeData = node.data as GenerateVideoNodeData; + + if (!nodeData.selectedModel?.modelId) { + logger.error('node.error', 'generateVideo node missing model selection', { + nodeId: node.id, + }); + updateNodeData(node.id, { + status: "error", + error: "No model selected", + }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + + updateNodeData(node.id, { + inputImages: images, + inputPrompt: text, + status: "loading", + error: null, + }); + + try { + const providerSettingsState = get().providerSettings; + + const requestPayload = { + images, + prompt: text, + selectedModel: nodeData.selectedModel, + }; + + // Build headers with API keys for external providers + const headers: Record = { + "Content-Type": "application/json", + }; + if (nodeData.selectedModel.provider === "replicate") { + const replicateConfig = providerSettingsState.providers.replicate; + if (replicateConfig?.apiKey) { + headers["X-Replicate-API-Key"] = replicateConfig.apiKey; + } + } else if (nodeData.selectedModel.provider === "fal") { + const falConfig = providerSettingsState.providers.fal; + if (falConfig?.apiKey) { + headers["X-Fal-API-Key"] = falConfig.apiKey; + } + } + + const provider = nodeData.selectedModel.provider; + logger.info('node.execution', `Calling ${provider} API for video generation`, { + nodeId: node.id, + provider, + model: nodeData.selectedModel.modelId, + imageCount: images.length, + prompt: text, + }); + + const response = await fetch("/api/generate", { + method: "POST", + headers, + body: JSON.stringify(requestPayload), + }); + + if (!response.ok) { + const errorText = await response.text(); + let errorMessage = `HTTP ${response.status}: ${response.statusText}`; + try { + const errorJson = JSON.parse(errorText); + errorMessage = errorJson.error || errorMessage; + } catch { + if (errorText) errorMessage += ` - ${errorText.substring(0, 200)}`; + } + + logger.error('api.error', `${provider} API request failed`, { + nodeId: node.id, + provider, + status: response.status, + statusText: response.statusText, + errorMessage, + }); + + updateNodeData(node.id, { + status: "error", + error: errorMessage, + }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + + const result = await response.json(); + + // Handle video response (video or videoUrl field) + const videoData = result.video || result.videoUrl; + if (result.success && videoData) { + updateNodeData(node.id, { + outputVideo: videoData, + status: "complete", + error: null, + }); + } else if (result.success && result.image) { + // Some models might return an image preview; treat as video for now + updateNodeData(node.id, { + outputVideo: result.image, + status: "complete", + error: null, + }); + } else { + logger.error('api.error', `${provider} API video generation failed`, { + nodeId: node.id, + provider, + error: result.error, + }); + updateNodeData(node.id, { + status: "error", + error: result.error || "Video generation failed", + }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + } catch (error) { + let errorMessage = "Video generation failed"; + if (error instanceof DOMException && error.name === 'AbortError') { + errorMessage = "Request timed out. Video generation may take longer."; + } else if (error instanceof TypeError && error.message.includes('NetworkError')) { + errorMessage = "Network error. Check your connection and try again."; + } else if (error instanceof TypeError) { + errorMessage = `Network error: ${error.message}`; + } else if (error instanceof Error) { + errorMessage = error.message; + } + + logger.error('node.error', 'GenerateVideo node execution failed', { + nodeId: node.id, + provider: nodeData.selectedModel?.provider, + errorMessage, + }, error instanceof Error ? error : undefined); + + updateNodeData(node.id, { + status: "error", + error: errorMessage, + }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + break; + } + case "llmGenerate": { const { images, text } = getConnectedInputs(node.id); @@ -1723,6 +1888,128 @@ export const useWorkflowStore = create((set, get) => ({ error: result.error || "LLM generation failed", }); } + } else if (node.type === "generateVideo") { + const nodeData = node.data as GenerateVideoNodeData; + const providerSettingsState = get().providerSettings; + + // Get fresh connected inputs + const inputs = getConnectedInputs(nodeId); + const images = inputs.images.length > 0 ? inputs.images : nodeData.inputImages; + const text = inputs.text ?? nodeData.inputPrompt; + + if (!text) { + logger.error('node.error', 'generateVideo regeneration failed: missing text input', { + nodeId, + }); + updateNodeData(nodeId, { + status: "error", + error: "Missing text input", + }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + + if (!nodeData.selectedModel?.modelId) { + logger.error('node.error', 'generateVideo regeneration failed: no model selected', { + nodeId, + }); + updateNodeData(nodeId, { + status: "error", + error: "No model selected", + }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + + updateNodeData(nodeId, { + inputImages: images, + status: "loading", + error: null, + }); + + // Build headers with API keys for external providers + const headers: Record = { + "Content-Type": "application/json", + }; + if (nodeData.selectedModel.provider === "replicate") { + const replicateConfig = providerSettingsState.providers.replicate; + if (replicateConfig?.apiKey) { + headers["X-Replicate-API-Key"] = replicateConfig.apiKey; + } + } else if (nodeData.selectedModel.provider === "fal") { + const falConfig = providerSettingsState.providers.fal; + if (falConfig?.apiKey) { + headers["X-Fal-API-Key"] = falConfig.apiKey; + } + } + + const provider = nodeData.selectedModel.provider; + logger.info('node.execution', `Calling ${provider} API for video regeneration`, { + nodeId, + provider, + model: nodeData.selectedModel.modelId, + imageCount: images.length, + prompt: text, + }); + + const response = await fetch("/api/generate", { + method: "POST", + headers, + body: JSON.stringify({ + images, + prompt: text, + selectedModel: nodeData.selectedModel, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + let errorMessage = `HTTP ${response.status}`; + try { + const errorJson = JSON.parse(errorText); + errorMessage = errorJson.error || errorMessage; + } catch { + if (errorText) errorMessage += ` - ${errorText.substring(0, 200)}`; + } + logger.error('api.error', `${provider} API video regeneration failed`, { + nodeId, + provider, + status: response.status, + errorMessage, + }); + updateNodeData(nodeId, { status: "error", error: errorMessage }); + set({ isRunning: false, currentNodeId: null }); + await logger.endSession(); + return; + } + + const result = await response.json(); + const videoData = result.video || result.videoUrl; + if (result.success && videoData) { + updateNodeData(nodeId, { + outputVideo: videoData, + status: "complete", + error: null, + }); + } else if (result.success && result.image) { + updateNodeData(nodeId, { + outputVideo: result.image, + status: "complete", + error: null, + }); + } else { + logger.error('api.error', `${provider} API video regeneration failed`, { + nodeId, + provider, + error: result.error, + }); + updateNodeData(nodeId, { + status: "error", + error: result.error || "Video generation failed", + }); + } } else if (node.type === "splitGrid") { const nodeData = node.data as SplitGridNodeData;