From 987d28a0c1b19969cb5e8bbf56b5d0cf4b090bf7 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Fri, 6 Feb 2026 00:14:43 +1300 Subject: [PATCH] feat: add Veo 3 video generation models via Kie.ai Add Veo 3 and Veo 3 Fast models (text-to-video + image-to-video) with custom API integration using Kie's /api/v1/veo/ endpoints and numeric successFlag polling. Also document Kie model integration SOP in CLAUDE.md. Co-Authored-By: Claude Opus 4.5 --- CLAUDE.md | 39 ++++++ src/app/api/generate/route.ts | 190 ++++++++++++++++++++++++++ src/app/api/models/[modelId]/route.ts | 34 +++++ src/app/api/models/route.ts | 36 +++++ 4 files changed, 299 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index e61e42ed..b8805ad5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,6 +151,45 @@ Use descriptive handle IDs matching the data type: - Connection validation: `isValidConnection()` in `WorkflowCanvas.tsx` - Workflow validation: `validateWorkflow()` in `workflowStore.ts` +## Adding New Kie.ai Models (SOP) + +Reference docs: https://docs.kie.ai/llms.txt lists all available model API pages. + +### Step 1: Gather API Details +Visit the model's doc page on https://docs.kie.ai/ and collect: +- Model ID(s) (the `model` param sent to the API) +- Capabilities: text-to-image, image-to-image, text-to-video, image-to-video +- API endpoint (standard: `/api/v1/jobs/createTask`, or model-specific like Veo's `/api/v1/veo/generate`) +- All input parameters: name, type, enum values, defaults, required status +- Image/video input parameter name (e.g., `image_urls`, `imageUrls`, `input_urls`) +- Polling endpoint (standard: `/api/v1/jobs/recordInfo`, or model-specific) +- Response format and status field names +- Pricing (per-run cost if available) + +### Step 2: Add Model Registry Entry +**File:** `src/app/api/models/route.ts` — Add to `KIE_MODELS` array. +Each model entry needs: `id`, `name`, `description`, `provider: "kie"`, `capabilities`, `pricing`, `pageUrl`. +Use separate entries for each capability variant (e.g., `model/text-to-video` and `model/image-to-video`). + +### Step 3: Add Parameter Schema +**File:** `src/app/api/models/[modelId]/route.ts` — Add to `getKieSchema()`. +Define `parameters` (user-configurable settings) and `inputs` (connectable handles like prompt, images). + +### Step 4: Add Default Parameters +**File:** `src/app/api/generate/route.ts` — Add case to `getKieModelDefaults()`. +Provide required defaults that must be present even if the user doesn't set them. + +### Step 5: Add Image Input Key Mapping +**File:** `src/app/api/generate/route.ts` — Add to `getKieImageInputKey()`. +Map the model to its correct image parameter name if it differs from the default `image_urls`. + +### Step 6: Handle Non-Standard API (if applicable) +If the model uses different endpoints than `/api/v1/jobs/createTask` and `/api/v1/jobs/recordInfo`: +- Add a detection function (e.g., `isVeoModel()`) +- Add a model-ID-to-API-model mapping function +- Add a custom polling function for the model's status endpoint +- Add a branch in `generateWithKie()` for the custom request format + ## API Routes All routes in `src/app/api/`: diff --git a/src/app/api/generate/route.ts b/src/app/api/generate/route.ts index 4967a13a..72a2c42a 100644 --- a/src/app/api/generate/route.ts +++ b/src/app/api/generate/route.ts @@ -1366,6 +1366,15 @@ function getKieModelDefaults(modelId: string): Record { upscale_factor: "2", }; + // Veo 3 models + case "veo3/text-to-video": + case "veo3/image-to-video": + case "veo3-fast/text-to-video": + case "veo3-fast/image-to-video": + return { + aspect_ratio: "16:9", + }; + default: return {}; } @@ -1387,6 +1396,8 @@ function getKieImageInputKey(modelId: string): string { if (modelId === "kling-2.6/motion-control") return "input_urls"; // Topaz video upscale uses video_url (singular) if (modelId === "topaz/video-upscale") return "video_url"; + // Veo 3 models use imageUrls + if (modelId.startsWith("veo3")) return "imageUrls"; // Default for most models return "image_urls"; } @@ -1544,6 +1555,61 @@ async function pollKieTaskCompletion( } } +// ============ Veo 3 Helpers ============ + +function isVeoModel(modelId: string): boolean { + return modelId.startsWith("veo3/") || modelId.startsWith("veo3-fast/"); +} + +function getVeoApiModelId(modelId: string): string { + if (modelId.startsWith("veo3-fast/")) return "veo3_fast"; + return "veo3"; +} + +async function pollVeoTaskCompletion( + requestId: string, + apiKey: string, + taskId: string, +): Promise<{ success: boolean; data?: Record; error?: string }> { + const maxWaitTime = 10 * 60 * 1000; + const pollInterval = 2000; + const startTime = Date.now(); + let lastStatus = -1; + + const pollUrl = `https://api.kie.ai/api/v1/veo/record-info?taskId=${encodeURIComponent(taskId)}`; + + while (true) { + if (Date.now() - startTime > maxWaitTime) { + return { success: false, error: "Generation timed out after 10 minutes" }; + } + await new Promise(resolve => setTimeout(resolve, pollInterval)); + + const response = await fetch(pollUrl, { + headers: { "Authorization": `Bearer ${apiKey}` }, + }); + if (!response.ok) { + return { success: false, error: `Failed to poll status: ${response.status}` }; + } + + const result = await response.json(); + const successFlag = result.data?.successFlag ?? -1; + + if (successFlag !== lastStatus) { + console.log(`[API:${requestId}] Veo task successFlag: ${successFlag}`); + lastStatus = successFlag; + } + + if (successFlag === 1) { + return { success: true, data: result.data }; + } + if (successFlag === 2 || successFlag === 3) { + const errorMessage = result.data?.errorMessage || "Generation failed"; + return { success: false, error: errorMessage }; + } + // successFlag === 0 means still generating, continue polling + } +} + /** * Generate image/video using Kie.ai API */ @@ -1642,6 +1708,130 @@ async function generateWithKie( } } + // Veo 3 models use a different API endpoint and request format + if (isVeoModel(modelId)) { + const veoBody: Record = { + prompt: inputParams.prompt, + model: getVeoApiModelId(modelId), + aspect_ratio: inputParams.aspect_ratio || "16:9", + }; + + // Add image URLs if present (for image-to-video) + if (inputParams.imageUrls) { + veoBody.imageUrls = Array.isArray(inputParams.imageUrls) + ? inputParams.imageUrls + : [inputParams.imageUrls]; + } + + // Add optional seed + if (inputParams.seeds !== undefined) { + veoBody.seeds = inputParams.seeds; + } + + const veoUrl = "https://api.kie.ai/api/v1/veo/generate"; + console.log(`[API:${requestId}] Calling Veo API: ${veoUrl}`); + console.log(`[API:${requestId}] Veo request body:`, JSON.stringify(veoBody, null, 2)); + + const createResponse = await fetch(veoUrl, { + method: "POST", + headers: { + "Authorization": `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(veoBody), + }); + + if (!createResponse.ok) { + const errorText = await createResponse.text(); + let errorDetail = errorText; + try { + const errorJson = JSON.parse(errorText); + errorDetail = errorJson.message || errorJson.error || errorJson.detail || errorText; + } catch { + // Keep original text + } + if (createResponse.status === 429) { + return { success: false, error: `${input.model.name}: Rate limit exceeded. Try again in a moment.` }; + } + return { success: false, error: `${input.model.name}: ${errorDetail}` }; + } + + const createResult = await createResponse.json(); + if (createResult.code && createResult.code !== 200) { + return { success: false, error: `${input.model.name}: ${createResult.msg || "API error"}` }; + } + + const taskId = createResult.data?.taskId || createResult.taskId; + if (!taskId) { + console.error(`[API:${requestId}] No taskId in Veo response:`, createResult); + return { success: false, error: "No task ID in Veo response" }; + } + + console.log(`[API:${requestId}] Veo task created: ${taskId}`); + + // Poll with Veo-specific polling + const pollResult = await pollVeoTaskCompletion(requestId, apiKey, taskId); + if (!pollResult.success) { + return { success: false, error: `${input.model.name}: ${pollResult.error}` }; + } + + // Extract video URL from Veo response format + const data = pollResult.data; + let mediaUrl: string | null = null; + + const responseObj = data?.response as Record | undefined; + const resultUrls = (responseObj?.resultUrls || data?.resultUrls) as string[] | undefined; + if (resultUrls && resultUrls.length > 0) { + mediaUrl = resultUrls[0]; + } + + if (!mediaUrl) { + console.error(`[API:${requestId}] No media URL found in Veo response:`, data); + return { success: false, error: "No output URL in Veo response" }; + } + + // Validate URL before fetching + const mediaUrlCheck = validateMediaUrl(mediaUrl); + if (!mediaUrlCheck.valid) { + return { success: false, error: `Invalid media URL: ${mediaUrlCheck.error}` }; + } + + // Fetch the video and convert to base64 + console.log(`[API:${requestId}] Fetching Veo output from: ${mediaUrl.substring(0, 80)}...`); + const mediaResponse = await fetch(mediaUrl); + if (!mediaResponse.ok) { + return { success: false, error: `Failed to fetch output: ${mediaResponse.status}` }; + } + + const MAX_MEDIA_SIZE = 500 * 1024 * 1024; + const mediaContentLength = parseInt(mediaResponse.headers.get("content-length") || "0", 10); + if (mediaContentLength > MAX_MEDIA_SIZE) { + return { success: false, error: `Media too large: ${(mediaContentLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; + } + + const contentType = mediaResponse.headers.get("content-type") || "video/mp4"; + const mediaArrayBuffer = await mediaResponse.arrayBuffer(); + const mediaSizeMB = mediaArrayBuffer.byteLength / (1024 * 1024); + + console.log(`[API:${requestId}] Veo output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); + + // For very large videos (>20MB), return URL directly + if (mediaSizeMB > 20) { + console.log(`[API:${requestId}] SUCCESS - Returning URL for large Veo video`); + return { + success: true, + outputs: [{ type: "video", data: mediaUrl, url: mediaUrl }], + }; + } + + const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); + console.log(`[API:${requestId}] SUCCESS - Returning Veo video`); + return { + success: true, + outputs: [{ type: "video", data: `data:${contentType};base64,${mediaBase64}`, url: mediaUrl }], + }; + } + // All remaining Kie models use the standard createTask endpoint const requestBody: Record = { model: modelId, diff --git a/src/app/api/models/[modelId]/route.ts b/src/app/api/models/[modelId]/route.ts index 4512ec5e..d29679e9 100644 --- a/src/app/api/models/[modelId]/route.ts +++ b/src/app/api/models/[modelId]/route.ts @@ -727,6 +727,40 @@ function getKieSchema(modelId: string): ExtractedSchema { { name: "video_url", type: "image", required: true, label: "Video" }, ], }, + "veo3/text-to-video": { + parameters: [ + { name: "aspect_ratio", type: "string", description: "Output aspect ratio", enum: ["16:9", "9:16"], default: "16:9" }, + { name: "seeds", type: "integer", description: "Random seed (10000-99999)", minimum: 10000, maximum: 99999 }, + ], + inputs: [{ name: "prompt", type: "text", required: true, label: "Prompt" }], + }, + "veo3/image-to-video": { + parameters: [ + { name: "aspect_ratio", type: "string", description: "Output aspect ratio", enum: ["16:9", "9:16"], default: "16:9" }, + { name: "seeds", type: "integer", description: "Random seed (10000-99999)", minimum: 10000, maximum: 99999 }, + ], + inputs: [ + { name: "prompt", type: "text", required: true, label: "Prompt" }, + { name: "imageUrls", type: "image", required: true, label: "Image", isArray: true }, + ], + }, + "veo3-fast/text-to-video": { + parameters: [ + { name: "aspect_ratio", type: "string", description: "Output aspect ratio", enum: ["16:9", "9:16"], default: "16:9" }, + { name: "seeds", type: "integer", description: "Random seed (10000-99999)", minimum: 10000, maximum: 99999 }, + ], + inputs: [{ name: "prompt", type: "text", required: true, label: "Prompt" }], + }, + "veo3-fast/image-to-video": { + parameters: [ + { name: "aspect_ratio", type: "string", description: "Output aspect ratio", enum: ["16:9", "9:16"], default: "16:9" }, + { name: "seeds", type: "integer", description: "Random seed (10000-99999)", minimum: 10000, maximum: 99999 }, + ], + inputs: [ + { name: "prompt", type: "text", required: true, label: "Prompt" }, + { name: "imageUrls", type: "image", required: true, label: "Image", isArray: true }, + ], + }, }; return schemas[modelId] || { parameters: [], inputs: [] }; diff --git a/src/app/api/models/route.ts b/src/app/api/models/route.ts index c7d0e2be..b95a87e5 100644 --- a/src/app/api/models/route.ts +++ b/src/app/api/models/route.ts @@ -271,6 +271,42 @@ const KIE_MODELS: ProviderModel[] = [ coverImage: undefined, pageUrl: "https://kie.ai/topaz", }, + { + id: "veo3/text-to-video", + name: "Veo 3", + description: "Google Veo 3.1 high-quality text-to-video generation with audio via Kie.ai.", + provider: "kie", + capabilities: ["text-to-video"], + coverImage: undefined, + pageUrl: "https://docs.kie.ai/veo3-api/quickstart", + }, + { + id: "veo3/image-to-video", + name: "Veo 3 I2V", + description: "Google Veo 3.1 image-to-video generation via Kie.ai. Supports 1-2 reference images.", + provider: "kie", + capabilities: ["image-to-video"], + coverImage: undefined, + pageUrl: "https://docs.kie.ai/veo3-api/quickstart", + }, + { + id: "veo3-fast/text-to-video", + name: "Veo 3 Fast", + description: "Google Veo 3.1 fast text-to-video generation with audio via Kie.ai.", + provider: "kie", + capabilities: ["text-to-video"], + coverImage: undefined, + pageUrl: "https://docs.kie.ai/veo3-api/quickstart", + }, + { + id: "veo3-fast/image-to-video", + name: "Veo 3 Fast I2V", + description: "Google Veo 3.1 fast image-to-video generation via Kie.ai. Supports 1-2 reference images.", + provider: "kie", + capabilities: ["image-to-video"], + coverImage: undefined, + pageUrl: "https://docs.kie.ai/veo3-api/quickstart", + }, ]; // Gemini image models (hardcoded - these don't come from an external API)