feat: implement generateWithGeminiVideo() provider function

Add Veo video generation using @google/genai SDK with: - Model ID mapping (veo-3.1/veo-3.1-fast to API model IDs) - Async generation with polling (10s intervals, 5min timeout) - Image-to-video support with base64 image input - Video download and base64 encoding (URL fallback for >20MB) - Configurable aspect ratio, duration, resolution, negative prompt, seed Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
5 months ago · 30a70cd877
1 changed files with 152 additions and 1 deletions
--- a/src/app/api/generate/providers/gemini.ts
+++ b/src/app/api/generate/providers/gemini.ts
@ -1,12 +1,13 @@
 /**
 * Gemini Provider for Generate API Route
 *
- * Handles image generation using Google's Gemini API models.
+ * Handles image generation and video generation using Google's Gemini API models.
 */

 import { NextResponse } from "next/server";
 import { GoogleGenAI } from "@google/genai";
 import { GenerateResponse, ModelType } from "@/types";
+import { GenerationOutput } from "@/lib/providers/types";

 /**
 * Map model types to Gemini model IDs
@ -189,3 +190,153 @@ export async function generateWithGemini(
    { status: 500 }
  );
 }
+
+/**
+ * Map internal Veo model IDs to Gemini API model IDs
+ */
+const VEO_MODEL_MAP: Record<string, string> = {
+  "veo-3.1/text-to-video": "veo-3.1-generate-preview",
+  "veo-3.1/image-to-video": "veo-3.1-generate-preview",
+  "veo-3.1-fast/text-to-video": "veo-3.1-fast-generate-preview",
+  "veo-3.1-fast/image-to-video": "veo-3.1-fast-generate-preview",
+};
+
+/**
+ * Generate video using Gemini API (Veo models)
+ */
+export async function generateWithGeminiVideo(
+  requestId: string,
+  apiKey: string,
+  modelId: string,
+  prompt: string,
+  images: string[],
+  parameters: Record<string, unknown> = {},
+): Promise<GenerationOutput> {
+  const apiModelId = VEO_MODEL_MAP[modelId];
+  if (!apiModelId) {
+    return { success: false, error: `Unknown Veo model: ${modelId}` };
+  }
+
+  console.log(`[API:${requestId}] Gemini video generation - Model: ${apiModelId}, Prompt: ${prompt?.length || 0} chars, Images: ${images?.length || 0}`);
+
+  const ai = new GoogleGenAI({ apiKey });
+
+  // Build config from parameters
+  const config: Record<string, unknown> = {
+    numberOfVideos: 1,
+  };
+
+  if (parameters.aspectRatio) {
+    config.aspectRatio = parameters.aspectRatio;
+  }
+  if (parameters.durationSeconds) {
+    config.durationSeconds = Number(parameters.durationSeconds);
+  }
+  if (parameters.resolution) {
+    config.resolution = parameters.resolution;
+  }
+  if (parameters.negativePrompt) {
+    config.negativePrompt = parameters.negativePrompt;
+  }
+  if (parameters.seed !== undefined && parameters.seed !== null && parameters.seed !== "") {
+    config.seed = Number(parameters.seed);
+  }
+
+  // Build request args
+  const requestArgs: Record<string, unknown> = {
+    model: apiModelId,
+    prompt,
+    config,
+  };
+
+  // Add image for image-to-video models
+  if (images && images.length > 0 && modelId.includes("image-to-video")) {
+    const imageInput = images[0];
+    if (imageInput.includes("base64,")) {
+      const [header, data] = imageInput.split("base64,");
+      const mimeMatch = header.match(/data:([^;]+)/);
+      const mimeType = mimeMatch ? mimeMatch[1] : "image/png";
+      requestArgs.image = {
+        imageBytes: data,
+        mimeType,
+      };
+    } else {
+      requestArgs.image = {
+        imageBytes: imageInput,
+        mimeType: "image/png",
+      };
+    }
+  }
+
+  console.log(`[API:${requestId}] Veo config: ${JSON.stringify(config)}`);
+
+  // Start video generation (async operation)
+  const startTime = Date.now();
+  let operation = await ai.models.generateVideos(requestArgs as Parameters<typeof ai.models.generateVideos>[0]);
+
+  // Poll for completion (10s intervals, 5min timeout)
+  const POLL_INTERVAL = 10_000;
+  const TIMEOUT = 5 * 60 * 1000;
+
+  while (!operation.done) {
+    const elapsed = Date.now() - startTime;
+    if (elapsed > TIMEOUT) {
+      console.error(`[API:${requestId}] Veo generation timed out after ${(elapsed / 1000).toFixed(0)}s`);
+      return { success: false, error: "Video generation timed out after 5 minutes" };
+    }
+
+    console.log(`[API:${requestId}] Veo polling... (${(elapsed / 1000).toFixed(0)}s elapsed)`);
+    await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL));
+    operation = await ai.operations.getVideosOperation({ operation });
+  }
+
+  const duration = Date.now() - startTime;
+  console.log(`[API:${requestId}] Veo generation completed in ${(duration / 1000).toFixed(1)}s`);
+
+  // Extract generated video
+  const generatedVideos = operation.response?.generatedVideos;
+  if (!generatedVideos || generatedVideos.length === 0) {
+    console.error(`[API:${requestId}] No generated videos in Veo response`);
+    return { success: false, error: "No video generated. The content may have been filtered by safety policies." };
+  }
+
+  const videoUri = generatedVideos[0]?.video?.uri;
+  if (!videoUri) {
+    console.error(`[API:${requestId}] No video URI in Veo response`);
+    return { success: false, error: "No video URI in response" };
+  }
+
+  // Fetch the video (append API key for authentication)
+  const videoUrl = `${videoUri}&key=${apiKey}`;
+  console.log(`[API:${requestId}] Fetching video from URI...`);
+
+  const videoResponse = await fetch(videoUrl);
+  if (!videoResponse.ok) {
+    console.error(`[API:${requestId}] Failed to fetch video: ${videoResponse.status}`);
+    return { success: false, error: `Failed to download generated video: ${videoResponse.status}` };
+  }
+
+  const videoBuffer = await videoResponse.arrayBuffer();
+  const videoSizeMB = (videoBuffer.byteLength / (1024 * 1024)).toFixed(2);
+  console.log(`[API:${requestId}] Video downloaded: ${videoSizeMB}MB`);
+
+  // Convert to base64 data URL if small enough, otherwise return URL
+  const MAX_BASE64_SIZE = 20 * 1024 * 1024; // 20MB
+  if (videoBuffer.byteLength > MAX_BASE64_SIZE) {
+    console.log(`[API:${requestId}] Video too large for base64 (${videoSizeMB}MB), returning URL`);
+    return {
+      success: true,
+      outputs: [{ type: "video", data: "", url: videoUrl }],
+    };
+  }
+
+  const base64Video = Buffer.from(videoBuffer).toString("base64");
+  const dataUrl = `data:video/mp4;base64,${base64Video}`;
+
+  console.log(`[API:${requestId}] SUCCESS - Returning ${videoSizeMB}MB video`);
+
+  return {
+    success: true,
+    outputs: [{ type: "video", data: dataUrl }],
+  };
+}