diff --git a/src/app/api/generate/providers/__tests__/newapiwg.test.ts b/src/app/api/generate/providers/__tests__/newapiwg.test.ts index 9c934127..4537a622 100644 --- a/src/app/api/generate/providers/__tests__/newapiwg.test.ts +++ b/src/app/api/generate/providers/__tests__/newapiwg.test.ts @@ -746,7 +746,7 @@ describe("NewApiWG generation payloads", () => { )).toBe(false); }); - it("resolves temporary local reference image URLs before OpenAI chat image requests", async () => { + it("passes temporary local reference image URLs through OpenAI chat image requests", async () => { const imageId = storeImage("data:image/png;base64,bG9jYWw="); try { const result = await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ @@ -766,7 +766,7 @@ describe("NewApiWG generation payloads", () => { role: "user", content: [ { type: "text", text: "喝咖啡" }, - { type: "image_url", image_url: { url: "data:image/png;base64,bG9jYWw=" } }, + { type: "image_url", image_url: { url: `http://localhost:3000/api/images/${imageId}` } }, ], }]); } finally { @@ -774,9 +774,8 @@ describe("NewApiWG generation payloads", () => { } }); - it("returns a clear error when an OpenAI chat temporary reference image is missing", async () => { - const fetchMock = vi.mocked(fetch); - + it("passes missing temporary local reference image URLs through OpenAI chat image requests", async () => { + const missingUrl = "http://localhost:3000/api/images/00000000-0000-0000-0000-000000000000"; const result = await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ model: { id: "gpt-image-2-all", @@ -786,17 +785,20 @@ describe("NewApiWG generation payloads", () => { capabilities: ["text-to-image", "image-to-image"], }, prompt: "喝咖啡", - images: ["http://localhost:3000/api/images/00000000-0000-0000-0000-000000000000"], + images: [missingUrl], })); - expect(result.success).toBe(false); - expect(result.error).toContain("Local temporary reference image was not found"); - expect(fetchMock.mock.calls.some(([url]) => - String(url).includes("/chat/completions") - )).toBe(false); + expect(result.success).toBe(true); + expect(capturedBody!.messages).toEqual([{ + role: "user", + content: [ + { type: "text", text: "喝咖啡" }, + { type: "image_url", image_url: { url: missingUrl } }, + ], + }]); }); - it("lets composer aspect ratio override stale OpenAI-compatible size parameters", async () => { + it("maps gpt-image-2-all resolution controls to supported chat completion size", async () => { await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ model: { id: "gpt-image-2-all", @@ -817,7 +819,53 @@ describe("NewApiWG generation payloads", () => { })); expect(capturedBody!.quality).toBe("auto"); - expect(capturedBody!.size).toBe("1024x1536"); + expect(capturedBody!.size).toBe("2048x1152"); + expect(capturedBody!.aspectRatio).toBeUndefined(); + expect(capturedBody!.aspect_ratio).toBeUndefined(); + expect(capturedBody!.resolution).toBeUndefined(); + expect(capturedBody!.imageSize).toBeUndefined(); + }); + + it("downgrades unsupported gpt-image-2-all 4K requests to 2K size", async () => { + await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ + model: { + id: "gpt-image-2-all", + name: "gpt-image-2-all", + description: null, + provider: "newapiwg", + capabilities: ["text-to-image", "image-to-image"], + }, + prompt: "draw a wide landscape", + parameters: { + resolution: "4K", + imageSize: "4K", + }, + })); + + expect(capturedBody!.size).toBe("2048x1152"); + }); + + it("maps gpt-image-2-vip resolution controls to chat completion size", async () => { + const result = await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ + model: { + id: "gpt-image-2-vip", + name: "gpt-image-2-vip", + description: null, + provider: "newapiwg", + capabilities: ["text-to-image", "image-to-image"], + }, + prompt: "draw a wide landscape", + parameters: { + resolution: "2K", + imageSize: "2K", + aspectRatio: "16:9", + aspect_ratio: "16:9", + }, + })); + + expect(result.success).toBe(true); + expect(capturedBody!.model).toBe("gpt-image-2-vip"); + expect(capturedBody!.size).toBe("2048x1152"); expect(capturedBody!.aspectRatio).toBeUndefined(); expect(capturedBody!.aspect_ratio).toBeUndefined(); expect(capturedBody!.resolution).toBeUndefined(); diff --git a/src/app/api/generate/providers/newapiwg.ts b/src/app/api/generate/providers/newapiwg.ts index 21e0b2f3..0a382ce9 100644 --- a/src/app/api/generate/providers/newapiwg.ts +++ b/src/app/api/generate/providers/newapiwg.ts @@ -958,10 +958,38 @@ function isImagenGenerationModel(modelId: string): boolean { } function isOpenAIChatImageModel(modelId: string): boolean { - return /(^|[_\-/])sora[_\-/]?image$/i.test(modelId) || modelId === "gpt-image-2-all"; + return /(^|[_\-/])sora[_\-/]?image$/i.test(modelId) || + modelId === "gpt-image-2-all" || + modelId === "gpt-image-2-vip"; } -function resolveOpenAIChatImageSize(parameters: Record | undefined): string | null { +function resolveGptImage2Size( + modelId: string, + parameters: Record | undefined +): string | null { + const resolution = + getStringParameter(parameters, "resolution") ?? + getStringParameter(parameters, "imageSize") ?? + getStringParameter(parameters, "image_size") ?? + getStringParameter(parameters, "output_size") ?? + getStringParameter(parameters, "size"); + const normalizedResolution = resolution?.trim().toUpperCase(); + if (normalizedResolution === "1K") return "1280x720"; + if (normalizedResolution === "2K") return "2048x1152"; + if (normalizedResolution === "4K" && modelId === "gpt-image-2-all") return "2048x1152"; + if (normalizedResolution === "4K") return "3840x2160"; + + return getStringParameter(parameters, "size"); +} + +function resolveOpenAIChatImageSize( + modelId: string, + parameters: Record | undefined +): string | null { + if (modelId === "gpt-image-2-all" || modelId === "gpt-image-2-vip") { + return resolveGptImage2Size(modelId, parameters) ?? "2048x1152"; + } + const aspectRatio = getStringParameter(parameters, "aspectRatio") ?? getStringParameter(parameters, "aspect_ratio"); @@ -976,10 +1004,11 @@ function resolveOpenAIChatImageSize(parameters: Record | undefi } function normalizeOpenAIChatImageParameters( + modelId: string, parameters: Record | undefined ): Record { const normalized = { ...(parameters || {}) }; - const size = resolveOpenAIChatImageSize(parameters); + const size = resolveOpenAIChatImageSize(modelId, parameters); if (size) normalized.size = size; delete normalized.aspectRatio; delete normalized.aspect_ratio; @@ -1109,12 +1138,7 @@ function extractNewApiWGImage(value: unknown): string | null { } function prepareOpenAIChatImageUrl(image: string): string { - const temporaryDataUrl = resolveTemporaryImageUrlToDataUrl(image); - if (temporaryDataUrl) return temporaryDataUrl; - - if (!isLoopbackTemporaryImageUrl(image)) return image; - - throw new Error("Local temporary reference image was not found. Please retry the generation."); + return image; } async function generateImageViaNewApiWGChat( @@ -1150,7 +1174,7 @@ async function generateImageViaNewApiWGChat( messages: [{ role: "user", content }], stream: false, ...dynamicInputsWithoutPrompt(input), - ...normalizeOpenAIChatImageParameters(input.parameters), + ...normalizeOpenAIChatImageParameters(input.model.id, input.parameters), }), }, input.model.name, "NewApiWG chat image API error"); diff --git a/src/app/api/generate/route.ts b/src/app/api/generate/route.ts index 6d012e73..52c7e06c 100644 --- a/src/app/api/generate/route.ts +++ b/src/app/api/generate/route.ts @@ -207,7 +207,10 @@ export async function POST(request: NextRequest) { dynamicInputs, mediaType, } = body; - const requestImages = resolveTemporaryImageUrlsInArray(images); + const provider: ProviderType = selectedModel?.provider || "gemini"; + const requestImages = provider === "newapiwg" + ? images || [] + : resolveTemporaryImageUrlsInArray(images); // Prompt is required unless: // - Provided via dynamicInputs @@ -228,7 +231,6 @@ export async function POST(request: NextRequest) { } // Determine which provider to use - const provider: ProviderType = selectedModel?.provider || "gemini"; const loginToken = getRequestToken(request); if (provider !== "newapiwg") { requireLogin(request); diff --git a/src/app/api/images/upload/route.ts b/src/app/api/images/upload/route.ts index 6e8eb078..22e4a77d 100644 --- a/src/app/api/images/upload/route.ts +++ b/src/app/api/images/upload/route.ts @@ -5,6 +5,10 @@ function extensionFromMime(mimeType: string): string { if (mimeType === "image/jpeg") return "jpg"; if (mimeType === "image/webp") return "webp"; if (mimeType === "image/gif") return "gif"; + if (mimeType === "video/mp4") return "mp4"; + if (mimeType === "video/webm") return "webm"; + if (mimeType === "video/quicktime") return "mov"; + if (mimeType.startsWith("video/")) return "mp4"; return "png"; } @@ -19,9 +23,9 @@ export async function POST(request: NextRequest): Promise { ); } - if (!file.type.startsWith("image/")) { + if (!file.type.startsWith("image/") && !file.type.startsWith("video/")) { return NextResponse.json( - { success: false, error: "Only image uploads are supported" }, + { success: false, error: "Only image and video uploads are supported" }, { status: 400 } ); } diff --git a/src/app/api/points/estimate/__tests__/route.test.ts b/src/app/api/points/estimate/__tests__/route.test.ts index 0e3770ad..45f3a2bc 100644 --- a/src/app/api/points/estimate/__tests__/route.test.ts +++ b/src/app/api/points/estimate/__tests__/route.test.ts @@ -3,7 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { POST } from "../route"; const originalFetch = global.fetch; -const originalBaseUrl = process.env.NEWAPIWG_BASE_URL; +const originalBaseUrl = process.env.POPIART_API_BASE_URL; function createPostRequest(body: unknown, headers?: Record): NextRequest { return new NextRequest("http://localhost/api/points/estimate", { @@ -19,12 +19,12 @@ function createPostRequest(body: unknown, headers?: Record): Nex describe("/api/points/estimate route", () => { afterEach(() => { global.fetch = originalFetch; - process.env.NEWAPIWG_BASE_URL = originalBaseUrl; + process.env.POPIART_API_BASE_URL = originalBaseUrl; vi.restoreAllMocks(); }); it("returns 401 when user token is missing", async () => { - const response = await POST(createPostRequest({ model_name: "doubao-seedance-2-0-260128" })); + const response = await POST(createPostRequest({ modelName: "doubao-seedance-2-0-260128" })); await expect(response.json()).resolves.toEqual({ success: false, @@ -34,11 +34,12 @@ describe("/api/points/estimate route", () => { expect(response.status).toBe(401); }); - it("forwards Seedance estimate requests to the NewApiWG points API with user auth", async () => { + it("forwards Seedance estimate requests to the PopiArt points API with user auth", async () => { + process.env.POPIART_API_BASE_URL = "https://popi.example"; const payload = { - model_name: "doubao-seedance-2-0-260128", - estimation_type: "seedance_video", - seedance_video: { + modelName: "doubao-seedance-2-0-260128", + estimationType: "seedance_video", + seedanceVideo: { resolution: "720p", aspect_ratio: "16:9", input_video_duration_seconds: 0, @@ -48,8 +49,6 @@ describe("/api/points/estimate route", () => { const upstreamBody = { success: true, data: { - model_name: "doubao-seedance-2-0-260128", - estimation_type: "seedance_video", estimated_points: 123, }, }; @@ -63,16 +62,16 @@ describe("/api/points/estimate route", () => { const response = await POST(createPostRequest(payload, { Authorization: "Bearer user-token", - "X-NewApiWG-Base-URL": "https://newapi.example/v1", })); await expect(response.json()).resolves.toEqual(upstreamBody); expect(response.status).toBe(200); expect(fetchMock).toHaveBeenCalledWith( - "https://newapi.example/api/points/estimate", + "https://popi.example/api_client/anime/ai/model/estimatePoints", { method: "POST", headers: { + Accept: "application/json", "Content-Type": "application/json", Authorization: "Bearer user-token", token: "user-token", diff --git a/src/app/api/points/estimate/route.ts b/src/app/api/points/estimate/route.ts index 297f9532..11ef6d43 100644 --- a/src/app/api/points/estimate/route.ts +++ b/src/app/api/points/estimate/route.ts @@ -1,31 +1,37 @@ import { NextRequest, NextResponse } from "next/server"; -import { normalizeNewApiWGBaseUrl } from "@/app/api/generate/providers/newapiwg"; import { requireLogin } from "@/app/api/_auth"; import { ApiError, apiErrorResponse } from "@/app/api/_errors"; import { logGatewayRequest } from "@/app/api/_gatewayLogging"; type PointsEstimateBody = { - model_name?: unknown; - estimation_type?: unknown; + modelName?: unknown; + estimationType?: unknown; + seedanceVideo?: unknown; }; -function getNewApiWGServerRootUrl(baseUrl?: string | null): string { - return normalizeNewApiWGBaseUrl(baseUrl).replace(/\/v1(?:beta)?$/i, ""); +const UPSTREAM_ESTIMATE_POINTS_PATH = "/api_client/anime/ai/model/estimatePoints"; + +function getUpstreamUrl(request: NextRequest): string { + const baseUrl = + process.env.POPIART_API_BASE_URL || + process.env.NEXT_PUBLIC_APP_URL || + request.nextUrl.origin; + return baseUrl.replace(/\/+$/, "") + UPSTREAM_ESTIMATE_POINTS_PATH; } export async function POST(request: NextRequest) { try { const { token } = requireLogin(request); const body = (await request.json()) as PointsEstimateBody; - const serverRootUrl = getNewApiWGServerRootUrl(request.headers.get("X-NewApiWG-Base-URL")); - if (!serverRootUrl) { - throw ApiError.config("NEWAPIWG_BASE_URL is not configured"); + if (!body.modelName || !body.estimationType) { + throw ApiError.badRequest("modelName and estimationType are required"); } - const url = `${serverRootUrl}/api/points/estimate`; + const url = getUpstreamUrl(request); const init: RequestInit = { method: "POST", headers: { + Accept: "application/json", "Content-Type": "application/json", Authorization: `Bearer ${token}`, token, @@ -34,8 +40,8 @@ export async function POST(request: NextRequest) { cache: "no-store", }; logGatewayRequest("Calling points estimate API", url, init, { - modelName: typeof body.model_name === "string" ? body.model_name : undefined, - estimationType: typeof body.estimation_type === "string" ? body.estimation_type : undefined, + modelName: typeof body.modelName === "string" ? body.modelName : undefined, + estimationType: typeof body.estimationType === "string" ? body.estimationType : undefined, }); const response = await fetch(url, init); diff --git a/src/components/__tests__/GenerationComposer.test.tsx b/src/components/__tests__/GenerationComposer.test.tsx index 43e8d065..474ebcf8 100644 --- a/src/components/__tests__/GenerationComposer.test.tsx +++ b/src/components/__tests__/GenerationComposer.test.tsx @@ -470,6 +470,15 @@ describe("GenerationComposer", () => { expect(screen.getByRole("button", { name: "2 points" })).toBeInTheDocument(); }); + it("multiplies NewApiWG image point cost by image count", () => { + useWorkflowStore.setState({ + nodes: [pricedImageNode("img-1", true, { imageCount: 4 })], + }); + render(); + + expect(screen.getByRole("button", { name: "8 points" })).toBeInTheDocument(); + }); + it("uses NewApiWG pricing variant matching the selected resolution", () => { useWorkflowStore.setState({ nodes: [ @@ -552,8 +561,6 @@ describe("GenerationComposer", () => { return new Response(JSON.stringify({ success: true, data: { - model_name: "doubao-seedance-2-0-260128", - estimation_type: "seedance_video", estimated_points: 123, }, }), { status: 200 }); @@ -592,9 +599,9 @@ describe("GenerationComposer", () => { headers: { "Content-Type": "application/json" }, }); expect(JSON.parse(String((estimateCall?.[1] as RequestInit).body))).toEqual({ - model_name: "doubao-seedance-2-0-260128", - estimation_type: "seedance_video", - seedance_video: { + modelName: "doubao-seedance-2-0-260128", + estimationType: "seedance_video", + seedanceVideo: { resolution: "720p", aspect_ratio: "16:9", input_video_duration_seconds: 0, diff --git a/src/components/auth/LoginModal.tsx b/src/components/auth/LoginModal.tsx index 8de937f5..ab2eae2b 100644 --- a/src/components/auth/LoginModal.tsx +++ b/src/components/auth/LoginModal.tsx @@ -421,9 +421,9 @@ export function LoginModal() { >
{reason === "unauthorized" ? ( - + ) : null} - {error ? : null} + {error ? : null} ) : (
- {wxQrError ? : null} + {wxQrError ? : null} {wxQrUrl ? (
- {captchaError ? : null} + {captchaError ? : null}
{captchaImage ? ( { const ASPECT_RATIOS: AspectRatio[] = ["1:1", "3:4", "4:3", "9:16", "16:9", "21:9"]; const RESOLUTIONS: Resolution[] = ["1K", "2K", "4K"]; +const GPT_IMAGE_2_ALL_RESOLUTIONS: Resolution[] = ["1K", "2K"]; const VIDEO_DURATION_QUICK_OPTIONS = [4, 5, 8, 10, 15]; const IMAGE_COUNTS: ImageGenerationCount[] = [1, 2, 4]; const MAX_REFERENCE_IMAGES = 4; @@ -171,6 +172,17 @@ function getLegacyImageModel(model: SelectedModel): ModelType { return "nano-banana-pro"; } +function getImageResolutionOptions(model: SelectedModel): Resolution[] { + return model.provider === "newapiwg" && model.modelId === "gpt-image-2-all" + ? GPT_IMAGE_2_ALL_RESOLUTIONS + : RESOLUTIONS; +} + +function normalizeImageResolutionForModel(model: SelectedModel, resolution: string): string { + const options = getImageResolutionOptions(model); + return options.includes(resolution as Resolution) ? resolution : options.at(-1) ?? "2K"; +} + function selectedModelFromProvider(model: ProviderModel): SelectedModel { return { provider: model.provider, @@ -306,9 +318,9 @@ function buildSeedanceVideoEstimatePayload( inputVideoDurationSeconds: number ): SeedanceVideoEstimatePayload { return { - model_name: draft.selectedModel.modelId, - estimation_type: "seedance_video", - seedance_video: { + modelName: draft.selectedModel.modelId, + estimationType: "seedance_video", + seedanceVideo: { resolution: draft.resolution || DEFAULT_VIDEO_RESOLUTION, aspect_ratio: draft.aspectRatio || "16:9", input_video_duration_seconds: inputVideoDurationSeconds, @@ -838,11 +850,15 @@ export function GenerationComposer() { const seedanceEstimateKey = seedanceEstimatePayload ? JSON.stringify({ - baseUrl: newApiWGBaseUrl || "default", ...seedanceEstimatePayload, }) : null; + const imageResolutionOptions = useMemo( + () => getImageResolutionOptions(draft.selectedModel), + [draft.selectedModel.modelId, draft.selectedModel.provider] + ); + useEffect(() => { const selectedModel = draft.selectedModel; if ( @@ -965,7 +981,6 @@ export function GenerationComposer() { const headers: Record = { "Content-Type": "application/json", }; - if (newApiWGBaseUrl) headers["X-NewApiWG-Base-URL"] = newApiWGBaseUrl; const response = await fetch("/api/points/estimate", { method: "POST", @@ -989,7 +1004,7 @@ export function GenerationComposer() { return () => { cancelled = true; }; - }, [newApiWGBaseUrl, seedanceEstimateKey, seedanceEstimatePayload]); + }, [seedanceEstimateKey, seedanceEstimatePayload]); const isVideoStitchNode = context.mode === "process-node" && context.node?.type === "videoStitch"; const isEaseCurveNode = context.mode === "process-node" && context.node?.type === "easeCurve"; @@ -1151,7 +1166,10 @@ export function GenerationComposer() { : errorMessage ? t("composer.retryCurrentNode") : nodeEditGenerateTitle; - const selectedPointAmount = seedanceEstimatedPointAmount ?? getSelectedPointAmount(draft, referenceImages.length); + const selectedUnitPointAmount = seedanceEstimatedPointAmount ?? getSelectedPointAmount(draft, referenceImages.length); + const selectedPointAmount = selectedUnitPointAmount !== null && seedanceEstimatedPointAmount === null && activeCapability === "image" + ? selectedUnitPointAmount * draft.imageCount + : selectedUnitPointAmount; const modelPointCostLabel = selectedPointAmount !== null ? t("composer.pointCost", { count: formatPointAmount(selectedPointAmount) }) : null; @@ -1211,6 +1229,20 @@ export function GenerationComposer() { }); }, []); + useEffect(() => { + if (context.capability !== "image" && context.mode !== "empty-create") return; + const normalizedResolution = normalizeImageResolutionForModel(draft.selectedModel, draft.resolution); + if (normalizedResolution === draft.resolution) return; + markDraft({ resolution: normalizedResolution }, ["resolution"]); + }, [ + context.capability, + context.mode, + draft.resolution, + draft.selectedModel.modelId, + draft.selectedModel.provider, + markDraft, + ]); + const updateVideoQuickDraft = useCallback( (patch: Partial, fields: DraftField[]) => { const activeContext = contextRef.current; @@ -1279,6 +1311,9 @@ export function GenerationComposer() { (model: ProviderModel) => { const nextModel = selectedModelFromProvider(model); const activeContext = contextRef.current; + const nextResolution = modelSupportsCapability(nextModel, "video") + ? DEFAULT_VIDEO_RESOLUTION + : normalizeImageResolutionForModel(nextModel, draftRef.current.resolution); if (activeContext.mode === "node-edit") { const activeAdapter = getAdapter(activeContext.nodeType); @@ -1289,7 +1324,7 @@ export function GenerationComposer() { { ...draftRef.current, selectedModel: nextModel, - resolution: modelSupportsCapability(nextModel, "video") ? DEFAULT_VIDEO_RESOLUTION : draftRef.current.resolution, + resolution: nextResolution, durationSeconds: modelSupportsCapability(nextModel, "video") ? DEFAULT_VIDEO_DURATION_SECONDS : draftRef.current.durationSeconds, @@ -1301,7 +1336,7 @@ export function GenerationComposer() { setDraft((current) => ({ ...current, selectedModel: nextModel, - resolution: modelSupportsCapability(nextModel, "video") ? DEFAULT_VIDEO_RESOLUTION : current.resolution, + resolution: nextResolution, durationSeconds: modelSupportsCapability(nextModel, "video") ? DEFAULT_VIDEO_DURATION_SECONDS : current.durationSeconds, @@ -1316,7 +1351,9 @@ export function GenerationComposer() { setDraft((current) => ({ ...current, selectedModel: nextModel, - resolution: modelSupportsCapability(nextModel, "video") ? DEFAULT_VIDEO_RESOLUTION : current.resolution, + resolution: modelSupportsCapability(nextModel, "video") + ? DEFAULT_VIDEO_RESOLUTION + : normalizeImageResolutionForModel(nextModel, current.resolution), durationSeconds: modelSupportsCapability(nextModel, "video") ? DEFAULT_VIDEO_DURATION_SECONDS : current.durationSeconds, @@ -2035,11 +2072,11 @@ export function GenerationComposer() { diff --git a/src/store/execution/__tests__/generateVideoExecutor.test.ts b/src/store/execution/__tests__/generateVideoExecutor.test.ts index cc6f7eaf..d586fd16 100644 --- a/src/store/execution/__tests__/generateVideoExecutor.test.ts +++ b/src/store/execution/__tests__/generateVideoExecutor.test.ts @@ -201,6 +201,63 @@ describe("executeGenerateVideo", () => { expect(body.dynamicInputs).toEqual({ video_urls: "data:video/mp4;base64,input" }); }); + it("uploads large NewApiWG video inputs before sending the generate request", async () => { + const largeVideo = `data:video/mp4;base64,${"a".repeat(400_000)}`; + const node = makeNode({ + selectedModel: { + provider: "newapiwg", + modelId: "doubao-seedance-2-0-260128", + displayName: "Seedance", + capabilities: ["text-to-video", "image-to-video"], + }, + }); + const ctx = makeCtx(node, { + getConnectedInputs: vi.fn().mockReturnValue({ + images: [], + videos: [largeVideo], + audio: [], + text: null, + dynamicInputs: {}, + easeCurve: null, + }), + }); + + mockFetch.mockImplementation(async (url: string) => { + if (url === "/api/images/upload") { + return { + ok: true, + json: () => Promise.resolve({ + success: true, + id: "tmp-video-1", + url: "http://localhost:3000/api/images/tmp-video-1", + }), + }; + } + if (url === "/api/generate") { + return { + ok: true, + json: () => Promise.resolve({ success: true, videoUrl: "https://cdn.example.com/out.mp4" }), + }; + } + throw new Error(`Unexpected fetch URL: ${url}`); + }); + + await executeGenerateVideo(ctx); + + const generateCall = mockFetch.mock.calls.find(([url]) => url === "/api/generate"); + const cleanupCall = mockFetch.mock.calls.find( + ([url, init]) => url === "/api/images/upload" && init?.method === "DELETE" + ); + const body = JSON.parse(generateCall?.[1].body); + + expect(body.dynamicInputs).toEqual({ + video: "http://localhost:3000/api/images/tmp-video-1", + videos: ["http://localhost:3000/api/images/tmp-video-1"], + }); + expect(JSON.stringify(body)).not.toContain("base64"); + expect(cleanupCall?.[1].body).toBe(JSON.stringify({ ids: ["tmp-video-1"] })); + }); + it("maps first-class video fields into provider parameters", async () => { const node = makeNode({ selectedModel: { provider: "kie", modelId: "bytedance/seedance-2/text-to-video", displayName: "Seedance" }, diff --git a/src/store/execution/generateVideoExecutor.ts b/src/store/execution/generateVideoExecutor.ts index 0378e67a..2c482e14 100644 --- a/src/store/execution/generateVideoExecutor.ts +++ b/src/store/execution/generateVideoExecutor.ts @@ -25,6 +25,11 @@ import { VIDEO_RESOLUTION_PARAMETER_NAMES, VIDEO_SOUND_PARAMETER_NAMES, } from "@/utils/videoGenerationSettings"; +import { + cleanupTemporaryImages, + uploadTemporaryDynamicInputsForGeneration, + uploadTemporaryImagesForGeneration, +} from "@/utils/temporaryImageUpload"; export interface GenerateVideoOptions { /** When true, falls back to stored inputImages/inputPrompt if no connections provide them. */ @@ -199,17 +204,29 @@ export async function executeGenerateVideo( const runOnce = async (modelToUse: SelectedModel, parametersOverride?: Record): Promise => { const provider = modelToUse.provider; const headers = buildGenerateHeaders(provider, providerSettings); - - const requestPayload = { - images, - prompt: text, - selectedModel: modelToUse, - parameters: buildVideoParameters(nodeData, modelToUse, parametersOverride), - dynamicInputs: requestDynamicInputs, - mediaType: "video" as const, - }; + let temporaryMediaIds: string[] = []; try { + const preparedImages = provider === "newapiwg" + ? await uploadTemporaryImagesForGeneration(images) + : { images, cleanupIds: [] }; + const preparedDynamicInputs = provider === "newapiwg" + ? await uploadTemporaryDynamicInputsForGeneration(requestDynamicInputs) + : { dynamicInputs: requestDynamicInputs, cleanupIds: [] }; + temporaryMediaIds = [ + ...preparedImages.cleanupIds, + ...preparedDynamicInputs.cleanupIds, + ]; + + const requestPayload = { + images: preparedImages.images, + prompt: text, + selectedModel: modelToUse, + parameters: buildVideoParameters(nodeData, modelToUse, parametersOverride), + dynamicInputs: preparedDynamicInputs.dynamicInputs, + mediaType: "video" as const, + }; + const response = await fetch("/api/generate", { method: "POST", headers, @@ -381,6 +398,8 @@ export async function executeGenerateVideo( error: errorMessage, }); throw new Error(errorMessage); + } finally { + await cleanupTemporaryImages(temporaryMediaIds); } }; diff --git a/src/store/execution/nanoBananaExecutor.ts b/src/store/execution/nanoBananaExecutor.ts index af5f4c73..2df32027 100644 --- a/src/store/execution/nanoBananaExecutor.ts +++ b/src/store/execution/nanoBananaExecutor.ts @@ -29,6 +29,7 @@ import { import { isHttpMediaUrl } from "@/utils/mediaResolver"; import { cleanupTemporaryImages, + uploadTemporaryDynamicInputsForGeneration, uploadTemporaryImagesForGeneration, } from "@/utils/temporaryImageUpload"; @@ -146,7 +147,13 @@ export async function executeNanoBanana( const preparedImages = provider === "newapiwg" ? await uploadTemporaryImagesForGeneration(images) : { images, cleanupIds: [] }; - temporaryImageIds = preparedImages.cleanupIds; + const preparedDynamicInputs = provider === "newapiwg" + ? await uploadTemporaryDynamicInputsForGeneration(sanitizedDynamicInputs) + : { dynamicInputs: sanitizedDynamicInputs, cleanupIds: [] }; + temporaryImageIds = [ + ...preparedImages.cleanupIds, + ...preparedDynamicInputs.cleanupIds, + ]; const requestPayload = { images: preparedImages.images, @@ -158,7 +165,7 @@ export async function executeNanoBanana( useImageSearch: (parametersOverride?.useImageSearch as boolean) ?? nodeData.useImageSearch, selectedModel: modelToUse, parameters: parametersOverride ?? nodeData.parameters, - dynamicInputs: sanitizedDynamicInputs, + dynamicInputs: preparedDynamicInputs.dynamicInputs, }; // Final guard: assert that prompt is a string before sending to API diff --git a/src/utils/temporaryImageUpload.ts b/src/utils/temporaryImageUpload.ts index 5b133660..2563ac2c 100644 --- a/src/utils/temporaryImageUpload.ts +++ b/src/utils/temporaryImageUpload.ts @@ -1,10 +1,20 @@ -const TEMP_IMAGE_UPLOAD_THRESHOLD_BYTES = 256 * 1024; +const TEMP_MEDIA_UPLOAD_THRESHOLD_BYTES = 256 * 1024; type UploadedTemporaryImages = { images: string[]; cleanupIds: string[]; }; +type UploadedTemporaryMedia = { + media: string[]; + cleanupIds: string[]; +}; + +type UploadedTemporaryDynamicInputs = { + dynamicInputs: Record; + cleanupIds: string[]; +}; + function dataUrlMimeType(dataUrl: string): string | null { return dataUrl.match(/^data:([^;]+);base64,/)?.[1] ?? null; } @@ -29,21 +39,27 @@ function dataUrlToBlob(dataUrl: string): Blob { return new Blob([bytes], { type: mimeType }); } -function shouldUploadTemporaryImage(image: string): boolean { - return image.startsWith("data:image/") && - approximateDataUrlBytes(image) > TEMP_IMAGE_UPLOAD_THRESHOLD_BYTES; +function shouldUploadTemporaryMedia(media: string): boolean { + return ( + (media.startsWith("data:image/") || media.startsWith("data:video/")) && + approximateDataUrlBytes(media) > TEMP_MEDIA_UPLOAD_THRESHOLD_BYTES + ); } function extensionFromMime(mimeType: string | null): string { if (mimeType === "image/jpeg") return "jpg"; if (mimeType === "image/webp") return "webp"; if (mimeType === "image/gif") return "gif"; + if (mimeType === "video/mp4") return "mp4"; + if (mimeType === "video/webm") return "webm"; + if (mimeType === "video/quicktime") return "mov"; + if (mimeType?.startsWith("video/")) return "mp4"; return "png"; } -async function uploadTemporaryImage(image: string): Promise<{ url: string; id: string }> { - const mimeType = dataUrlMimeType(image); - const blob = dataUrlToBlob(image); +async function uploadTemporaryMedia(media: string): Promise<{ url: string; id: string }> { + const mimeType = dataUrlMimeType(media); + const blob = dataUrlToBlob(media); const formData = new FormData(); formData.append("file", blob, `reference.${extensionFromMime(mimeType)}`); @@ -68,20 +84,61 @@ async function uploadTemporaryImage(image: string): Promise<{ url: string; id: s return { url: sameOriginUrl, id: result.id }; } +export async function uploadTemporaryMediaForGeneration( + mediaItems: string[] +): Promise { + const cleanupIds: string[] = []; + const preparedMedia = await Promise.all( + mediaItems.map(async (media) => { + if (!shouldUploadTemporaryMedia(media)) return media; + const uploaded = await uploadTemporaryMedia(media); + cleanupIds.push(uploaded.id); + return uploaded.url; + }) + ); + + return { media: preparedMedia, cleanupIds }; +} + export async function uploadTemporaryImagesForGeneration( images: string[] ): Promise { + const prepared = await uploadTemporaryMediaForGeneration(images); + + return { images: prepared.media, cleanupIds: prepared.cleanupIds }; +} + +export async function uploadTemporaryDynamicInputsForGeneration( + dynamicInputs: Record +): Promise { const cleanupIds: string[] = []; - const preparedImages = await Promise.all( - images.map(async (image) => { - if (!shouldUploadTemporaryImage(image)) return image; - const uploaded = await uploadTemporaryImage(image); - cleanupIds.push(uploaded.id); - return uploaded.url; + const uploadCache = new Map>(); + const prepareMedia = async (media: string): Promise => { + if (!shouldUploadTemporaryMedia(media)) return media; + let upload = uploadCache.get(media); + if (!upload) { + upload = uploadTemporaryMedia(media).then((uploaded) => ({ + media: uploaded.url, + cleanupIds: [uploaded.id], + })); + uploadCache.set(media, upload); + } + const prepared = await upload; + cleanupIds.push(...prepared.cleanupIds); + return prepared.media; + }; + + const entries = await Promise.all( + Object.entries(dynamicInputs).map(async ([key, value]) => { + if (typeof value === "string") { + return [key, await prepareMedia(value)] as const; + } + + return [key, await Promise.all(value.map(prepareMedia))] as const; }) ); - return { images: preparedImages, cleanupIds }; + return { dynamicInputs: Object.fromEntries(entries), cleanupIds: [...new Set(cleanupIds)] }; } export async function cleanupTemporaryImages(ids: string[]): Promise {