/** * GenerateVideo Executor * * Unified executor for generateVideo nodes. * Used by both executeWorkflow and regenerateNode. */ import type { GenerateVideoNodeData, SelectedModel } from "@/types"; import { buildGenerateHeaders } from "@/store/utils/buildApiHeaders"; import { createNewApiWGDefaultVideoModel, shouldPreferNewApiWGVideoModel, } from "@/store/utils/defaultImageModel"; import { pollGenerateTask } from "./pollTaskCompletion"; import { runWithFallback } from "./runWithFallback"; import type { NodeExecutionContext } from "./types"; import { isHttpMediaUrl } from "@/utils/mediaResolver"; import { DEFAULT_VIDEO_RESOLUTION, getDefaultVideoSoundEnabled, getVideoSoundParameterName, isValidVideoDurationSeconds, normalizeVideoDurationSeconds, VIDEO_DURATION_PARAMETER_NAMES, VIDEO_RESOLUTION_PARAMETER_NAMES, VIDEO_SOUND_PARAMETER_NAMES, } from "@/utils/videoGenerationSettings"; import { cleanupTemporaryImages, uploadTemporaryDynamicInputsForGeneration, uploadTemporaryImagesForGeneration, } from "@/utils/temporaryImageUpload"; export interface GenerateVideoOptions { /** When true, falls back to stored inputImages/inputPrompt if no connections provide them. */ useStoredFallback?: boolean; } function buildVideoParameters( nodeData: GenerateVideoNodeData, modelToUse: SelectedModel, parametersOverride?: Record ): Record | undefined { const parameters = { ...(parametersOverride ?? nodeData.parameters ?? {}) }; const durationSeconds = normalizeVideoDurationSeconds(nodeData.durationSeconds); const resolution = nodeData.resolution ?? DEFAULT_VIDEO_RESOLUTION; const soundParameterName = getVideoSoundParameterName(modelToUse); if (nodeData.aspectRatio) parameters.aspectRatio = nodeData.aspectRatio; for (const name of VIDEO_RESOLUTION_PARAMETER_NAMES) delete parameters[name]; parameters.resolution = resolution; for (const name of VIDEO_DURATION_PARAMETER_NAMES) delete parameters[name]; if (modelToUse.provider === "gemini") { parameters.durationSeconds = durationSeconds; } else { parameters.duration = durationSeconds; } if (nodeData.fps !== undefined) parameters.fps = nodeData.fps; for (const name of VIDEO_SOUND_PARAMETER_NAMES) delete parameters[name]; if (soundParameterName) { parameters[soundParameterName] = nodeData.soundEnabled ?? getDefaultVideoSoundEnabled(modelToUse); } return Object.keys(parameters).length > 0 ? parameters : undefined; } const hasDynamicVideoInput = (dynamicInputs: Record): boolean => Object.entries(dynamicInputs).some(([key, value]) => { if (!key.toLowerCase().includes("video")) return false; return Array.isArray(value) ? value.length > 0 : Boolean(value); }); function mergeConnectedVideosIntoDynamicInputs( dynamicInputs: Record, videos: string[] ): Record { if (videos.length === 0 || hasDynamicVideoInput(dynamicInputs)) { return dynamicInputs; } return { ...dynamicInputs, video: videos[0], videos, }; } export async function executeGenerateVideo( ctx: NodeExecutionContext, options: GenerateVideoOptions = {} ): Promise { const { node, getConnectedInputs, updateNodeData, getFreshNode, signal, providerSettings, providerEnvStatus, addIncurredCost, generationsPath, getNodes, trackSaveGeneration, } = ctx; const { useStoredFallback = false } = options; const { images: connectedImages, videos: connectedVideos, text: connectedText, audio: connectedAudio, dynamicInputs } = getConnectedInputs(node.id); const requestDynamicInputs = mergeConnectedVideosIntoDynamicInputs(dynamicInputs, connectedVideos); // Get fresh node data from store const freshNode = getFreshNode(node.id); const nodeData = (freshNode?.data || node.data) as GenerateVideoNodeData; if (nodeData.durationSeconds !== undefined && !isValidVideoDurationSeconds(nodeData.durationSeconds)) { const error = "Video duration must be at least 4 seconds"; updateNodeData(node.id, { status: "error", error }); throw new Error(error); } // Determine images and text let images: string[]; let text: string | null; if (useStoredFallback) { images = connectedImages.length > 0 ? connectedImages : nodeData.inputImages; text = connectedText ?? nodeData.inputPrompt; const hasPrompt = text || requestDynamicInputs.prompt || requestDynamicInputs.negative_prompt; const hasAudio = connectedAudio.length > 0; const hasVideo = connectedVideos.length > 0 || hasDynamicVideoInput(requestDynamicInputs); if (!hasPrompt && images.length === 0 && !hasVideo && !hasAudio) { updateNodeData(node.id, { status: "error", error: "Missing required inputs", }); throw new Error("Missing required inputs"); } } else { images = connectedImages; text = connectedText; const hasPrompt = text || requestDynamicInputs.prompt || requestDynamicInputs.negative_prompt; const hasAudio = connectedAudio.length > 0; const hasVideo = connectedVideos.length > 0 || hasDynamicVideoInput(requestDynamicInputs); if (!hasPrompt && images.length === 0 && !hasVideo && !hasAudio) { updateNodeData(node.id, { status: "error", error: "Missing required inputs", }); throw new Error("Missing required inputs"); } } let primaryModel = nodeData.selectedModel; let primaryParameters: Record | undefined; let fallbackModel = nodeData.fallbackModel; if (!primaryModel?.modelId && shouldPreferNewApiWGVideoModel(providerSettings, providerEnvStatus)) { primaryModel = createNewApiWGDefaultVideoModel(); updateNodeData(node.id, { selectedModel: primaryModel }); } if ( primaryModel?.provider === "gemini" && shouldPreferNewApiWGVideoModel(providerSettings, providerEnvStatus) ) { const promotedModel = fallbackModel && fallbackModel.provider !== "gemini" ? fallbackModel : createNewApiWGDefaultVideoModel(); primaryModel = promotedModel; primaryParameters = fallbackModel === promotedModel ? nodeData.fallbackParameters : nodeData.parameters; fallbackModel = undefined; updateNodeData(node.id, { selectedModel: promotedModel, parameters: primaryParameters ?? {}, fallbackModel: undefined, fallbackParameters: undefined, status: nodeData.status === "error" ? "idle" : nodeData.status, error: nodeData.status === "error" ? null : nodeData.error, __usedFallback: undefined, __fallbackModelUsed: undefined, __primaryError: undefined, inputSchema: undefined, }); } if (!primaryModel?.modelId) { updateNodeData(node.id, { status: "error", error: "No model selected", }); throw new Error("No model selected"); } updateNodeData(node.id, { inputImages: images, inputPrompt: text, status: "loading", error: null, }); const runOnce = async (modelToUse: SelectedModel, parametersOverride?: Record): Promise => { const provider = modelToUse.provider; const headers = buildGenerateHeaders(provider, providerSettings); let temporaryMediaIds: string[] = []; try { const preparedImages = provider === "newapiwg" ? await uploadTemporaryImagesForGeneration(images) : { images, cleanupIds: [] }; const preparedDynamicInputs = provider === "newapiwg" ? await uploadTemporaryDynamicInputsForGeneration(requestDynamicInputs) : { dynamicInputs: requestDynamicInputs, cleanupIds: [] }; temporaryMediaIds = [ ...preparedImages.cleanupIds, ...preparedDynamicInputs.cleanupIds, ]; const requestPayload = { images: preparedImages.images, prompt: text, selectedModel: modelToUse, parameters: buildVideoParameters(nodeData, modelToUse, parametersOverride), dynamicInputs: preparedDynamicInputs.dynamicInputs, mediaType: "video" as const, }; const response = await fetch("/api/generate", { method: "POST", headers, body: JSON.stringify(requestPayload), ...(signal ? { signal } : {}), }); if (!response.ok) { const errorText = await response.text(); let errorMessage = `HTTP ${response.status}`; try { const errorJson = JSON.parse(errorText); errorMessage = errorJson.error || errorMessage; if (errorJson.requestId) { errorMessage = `${errorMessage} (request ${errorJson.requestId})`; } } catch { if (errorText) errorMessage += ` - ${errorText.substring(0, 200)}`; } updateNodeData(node.id, { status: "error", error: errorMessage, }); throw new Error(errorMessage); } let result = await response.json(); // Handle polling response (long-running Kie tasks) if (result.polling) { result = await pollGenerateTask({ taskId: result.taskId, provider: result.pollProvider, modelId: result.pollModelId, modelName: result.pollModelName, mediaType: result.pollMediaType, headers, signal, }); if (!result.success) { updateNodeData(node.id, { status: "error", error: result.error || "Video generation failed", }); throw new Error(result.error || "Video generation failed"); } } // Handle video response (video or videoUrl field) const videoData = result.video || result.videoUrl; if (result.success && (videoData || result.image)) { const outputContent = videoData || result.image; const timestamp = Date.now(); const videoId = `${timestamp}`; const usedModelId = modelToUse.modelId || ""; const usedModelDisplayName = modelToUse.displayName || usedModelId; // Add to node's video history const newHistoryItem = { id: videoId, video: outputContent, timestamp, prompt: text || "", model: usedModelId, modelDisplayName: usedModelDisplayName, modelProvider: modelToUse.provider, }; const updatedHistory = [newHistoryItem, ...(nodeData.videoHistory || [])].slice(0, 50); const isRemoteVideo = Boolean(videoData && isHttpMediaUrl(outputContent)); const storageStatus = isRemoteVideo ? generationsPath ? "localizing" : "remote-only" : undefined; updateNodeData(node.id, { outputVideo: outputContent, outputVideoRef: undefined, outputVideoRemoteUrl: isRemoteVideo ? outputContent : undefined, outputVideoStorageStatus: storageStatus, status: "complete", error: null, videoHistory: updatedHistory, selectedVideoHistoryIndex: 0, lastUsedModel: modelToUse, }); // Track cost if (modelToUse.provider === "fal" && modelToUse.pricing) { addIncurredCost(modelToUse.pricing.amount); } // Auto-save to generations folder if configured if (generationsPath) { const saveContent = videoData ? { video: videoData } : { image: result.image }; const savePromise = fetch("/api/save-generation", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ directoryPath: generationsPath, ...saveContent, prompt: text, imageId: videoId, }), }) .then((res) => res.json()) .then((saveResult) => { if (saveResult.success && saveResult.imageId) { const currentNode = getNodes().find((n) => n.id === node.id); if (currentNode) { const currentData = currentNode.data as GenerateVideoNodeData; const histCopy = [...(currentData.videoHistory || [])]; const entryIndex = histCopy.findIndex((h) => h.id === videoId); const updates: Partial = {}; if (entryIndex !== -1 && saveResult.imageId !== videoId) { histCopy[entryIndex] = { ...histCopy[entryIndex], id: saveResult.imageId }; updates.videoHistory = histCopy; } if (videoData) { updates.outputVideoRef = saveResult.imageId; updates.outputVideoStorageStatus = "localized"; const currentOutputVideo = currentData.outputVideo; if (isHttpMediaUrl(currentOutputVideo) && currentOutputVideo) { updates.outputVideoRemoteUrl = currentData.outputVideoRemoteUrl || currentOutputVideo; } } updateNodeData(node.id, updates); } } }) .catch((err) => { console.error("Failed to save video generation:", err); if (videoData && isRemoteVideo) { updateNodeData(node.id, { outputVideoStorageStatus: "remote-only" }); } }); trackSaveGeneration(videoId, savePromise); } } else { updateNodeData(node.id, { status: "error", error: result.error || "Video generation failed", }); throw new Error(result.error || "Video generation failed"); } } catch (error) { if (error instanceof DOMException && error.name === "AbortError") { throw error; } let errorMessage = "Video generation failed"; if (error instanceof TypeError && error.message.includes("NetworkError")) { errorMessage = "Network error. Check your connection and try again."; } else if (error instanceof TypeError) { errorMessage = `Network error: ${error.message}`; } else if (error instanceof Error) { errorMessage = error.message; } updateNodeData(node.id, { status: "error", error: errorMessage, }); throw new Error(errorMessage); } finally { await cleanupTemporaryImages(temporaryMediaIds); } }; await runWithFallback({ nodeId: node.id, primary: primaryModel, primaryParameters, fallback: fallbackModel, fallbackParameters: nodeData.fallbackParameters, updateNodeData, runOnce, clearOutput: { outputVideo: null }, }); }