import { GenerationInput, GenerationOutput, ModelCapability, ProviderModel } from "@/lib/providers/types"; import { DEFAULT_NEWAPIWG_BASE_URL } from "@/store/utils/localStorage"; import { resolveTemporaryImageUrlToDataUrl } from "@/lib/images/temporaryUrls"; import { validateMediaUrl } from "@/utils/urlValidation"; import { buildGatewayHeaders } from "@/app/api/_auth"; import { logGatewayRequest, logGatewayResponse } from "@/app/api/_gatewayLogging"; import type { SelectedModel } from "@/types"; import { filterReferenceSubjectList, modelSupportsReferenceMedia } from "@/utils/generationInputSupport"; type NewApiWGModelObject = { id?: string; name?: string; description?: string | null; owned_by?: string; points?: unknown; point?: unknown; point_cost?: unknown; pointCost?: unknown; credits?: unknown; credit?: unknown; quota?: unknown; cost?: unknown; price?: unknown; model_price?: unknown; modelPrice?: unknown; fixed_price?: unknown; fixedPrice?: unknown; pricing?: unknown; supported_endpoint_types?: unknown; capabilities?: unknown; capability?: unknown; tags?: unknown; tag?: unknown; modalities?: unknown; metadata?: { capabilities?: unknown; tags?: unknown; modalities?: unknown; description?: string; points?: unknown; point?: unknown; point_cost?: unknown; pointCost?: unknown; credits?: unknown; credit?: unknown; quota?: unknown; cost?: unknown; price?: unknown; model_price?: unknown; modelPrice?: unknown; fixed_price?: unknown; fixedPrice?: unknown; pricing?: unknown; }; }; type NewApiWGModelsResponse = { data?: NewApiWGModelObject[]; models?: NewApiWGModelObject[]; }; type NewApiWGPricingObject = { model_name?: string; name?: string; model?: string; owner_by?: string; owned_by?: string; quota_type?: unknown; model_ratio?: unknown; model_price?: unknown; fixed_price?: unknown; price?: unknown; supported_endpoint_types?: unknown; capabilities?: unknown; tags?: unknown; modalities?: unknown; }; type NewApiWGPricingResponse = { data?: NewApiWGPricingObject[]; }; type NewApiWGBillingCatalogItem = { scene?: string; model_name?: string; channel_types?: unknown; base_points_per_call?: unknown; base_points?: unknown; variants?: unknown; supported_endpoint_types?: unknown; capabilities?: unknown; tags?: unknown; modalities?: unknown; }; type NewApiWGBillingCatalogResponse = { data?: { page?: number; page_size?: number; total?: number; items?: NewApiWGBillingCatalogItem[]; }; success?: boolean; }; const NEWAPIWG_USD_TO_POPI_POINTS = 70; const NEWAPIWG_QUOTA_PER_USD = 500_000; const NEWAPIWG_FALLBACK_MODELS: ProviderModel[] = [ { id: "gpt-image-1", name: "GPT Image 1", description: "Fallback OpenAI-compatible image generation model. Remote /models is currently unavailable.", provider: "newapiwg", capabilities: ["text-to-image", "image-to-image"], }, { id: "dall-e-3", name: "DALL-E 3", description: "Fallback OpenAI-compatible text-to-image model. Remote /models is currently unavailable.", provider: "newapiwg", capabilities: ["text-to-image"], }, { id: "sora-2", name: "Sora 2", description: "Fallback OpenAI-compatible video generation model. Remote /models is currently unavailable.", provider: "newapiwg", capabilities: ["text-to-video", "image-to-video"], }, { id: "tts-1", name: "TTS 1", description: "Fallback OpenAI-compatible speech model. Remote /models is currently unavailable.", provider: "newapiwg", capabilities: ["text-to-audio"], }, ]; const DEFAULT_NEWAPIWG_CAPABILITIES: ModelCapability[] = [ "text-to-image", "image-to-image", "text-to-video", "image-to-video", "text-to-3d", "image-to-3d", "text-to-audio", "audio-to-video", ]; export function normalizeNewApiWGBaseUrl(baseUrl?: string | null): string { return (baseUrl || process.env.NEWAPIWG_BASE_URL || DEFAULT_NEWAPIWG_BASE_URL).replace(/\/+$/, ""); } function isDoubaoChatModelName(text: string): boolean { return /(^|[\s_\-/])doubao[\s_\-/](?!seedance|seedream)(?:seed|lite|pro|vision|flash|thinking|1|[0-9])/i.test(text); } function flattenTags(value: unknown): string[] { if (!value) return []; if (Array.isArray(value)) return value.flatMap(flattenTags); if (typeof value === "string") return value.split(/[,\s]+/).filter(Boolean); if (typeof value === "object") return Object.values(value as Record).flatMap(flattenTags); return []; } function addCapability(capabilities: Set, capability: string): void { if ([ "text-to-image", "image-to-image", "text-to-video", "image-to-video", "text-to-3d", "image-to-3d", "text-to-audio", "audio-to-video", ].includes(capability)) { capabilities.add(capability as ModelCapability); } } function numberFromUnknown(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string") { const parsed = Number.parseFloat(value.trim()); return Number.isFinite(parsed) ? parsed : null; } return null; } function usdToPopiPoints(usd: number): number { if (usd <= 0) return 0; return Math.max(1, Math.round(usd * NEWAPIWG_USD_TO_POPI_POINTS)); } function quotaToPopiPoints(quota: number): number { return usdToPopiPoints(quota / NEWAPIWG_QUOTA_PER_USD); } function pointCostFromRecord(value: unknown): number | null { if (!value || typeof value !== "object") return null; const record = value as Record; const directCost = [ record.points, record.point, record.point_cost, record.pointCost, record.credits, record.credit, record.cost, record.amount, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; if (directCost !== null) return directCost; const quotaCost = [ record.quota, record.quota_cost, record.quotaCost, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; if (quotaCost !== null) return quotaToPopiPoints(quotaCost); const usdPrice = [ record.model_price, record.modelPrice, record.fixed_price, record.fixedPrice, record.price, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; return usdPrice !== null ? usdToPopiPoints(usdPrice) : null; } function getNewApiWGModelPointCost(model: NewApiWGModelObject): number | null { const directCost = [ model.points, model.point, model.point_cost, model.pointCost, model.credits, model.credit, model.cost, pointCostFromRecord(model.pricing), model.metadata?.points, model.metadata?.point, model.metadata?.point_cost, model.metadata?.pointCost, model.metadata?.credits, model.metadata?.credit, model.metadata?.cost, pointCostFromRecord(model.metadata?.pricing), ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; if (directCost !== null) return directCost; const quotaCost = [ model.quota, model.metadata?.quota, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; if (quotaCost !== null) return quotaToPopiPoints(quotaCost); const usdPrice = [ model.model_price, model.modelPrice, model.fixed_price, model.fixedPrice, model.price, model.metadata?.model_price, model.metadata?.modelPrice, model.metadata?.fixed_price, model.metadata?.fixedPrice, model.metadata?.price, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; return usdPrice !== null ? usdToPopiPoints(usdPrice) : null; } function normalizeNewApiWGServerRootUrl(baseUrl?: string | null): string { return normalizeNewApiWGBaseUrl(baseUrl).replace(/\/v1(?:beta)?$/i, ""); } function pricingModelName(row: NewApiWGPricingObject): string | null { const value = row.model_name ?? row.model ?? row.name; return typeof value === "string" && value.length > 0 ? value : null; } function getNewApiWGPricingPointCost(row: NewApiWGPricingObject): number | null { const quotaType = numberFromUnknown(row.quota_type); const fixedPrice = [ row.model_price, row.fixed_price, row.price, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; if (fixedPrice !== null && (quotaType === 1 || row.quota_type === undefined)) { return usdToPopiPoints(fixedPrice); } return null; } function billingCatalogModelName(row: NewApiWGBillingCatalogItem): string | null { return typeof row.model_name === "string" && row.model_name.length > 0 ? row.model_name : null; } function getBillingCatalogVariantPointCost(variant: Record): number | null { return [ variant.effective_points, variant.base_points, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; } function getBillingCatalogPointCost(row: NewApiWGBillingCatalogItem): number | null { const defaultVariant = Array.isArray(row.variants) ? row.variants.find((variant) => variant && typeof variant === "object" && (variant as Record).variant_key === "default" ) as Record | undefined : undefined; return [ defaultVariant ? getBillingCatalogVariantPointCost(defaultVariant) : null, row.base_points, ].map(numberFromUnknown).find((amount): amount is number => amount !== null) ?? null; } function billingCatalogDimensionValue(value: unknown): string | number | boolean | null { if (typeof value === "string" && value.length > 0) return value; if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "boolean") return value; return null; } function billingCatalogVariantDimensions(value: unknown): Record | null { if (!value || typeof value !== "object" || Array.isArray(value)) return null; const entries = Object.entries(value as Record) .map(([key, rawValue]) => [key, billingCatalogDimensionValue(rawValue)] as const) .filter((entry): entry is readonly [string, string | number | boolean] => entry[1] !== null); return entries.length > 0 ? Object.fromEntries(entries) : null; } function getBillingCatalogPricing(row: NewApiWGBillingCatalogItem): ProviderModel["pricing"] | undefined { const pointCost = getBillingCatalogPointCost(row); if (pointCost === null) return undefined; const variants = Array.isArray(row.variants) ? row.variants .map((variant) => { if (!variant || typeof variant !== "object") return null; const record = variant as Record; const dimensions = billingCatalogVariantDimensions(record.dimensions); const amount = getBillingCatalogVariantPointCost(record); return dimensions && amount !== null ? { dimensions, amount } : null; }) .filter((variant): variant is { dimensions: Record; amount: number } => Boolean(variant)) : []; return { type: "per-run", amount: pointCost, currency: "points", variants: variants.length > 0 ? variants : undefined, }; } function capabilitiesFromBillingScene(row: NewApiWGBillingCatalogItem, modelName: string): ModelCapability[] { if (row.scene === "image") return ["text-to-image", "image-to-image"]; if (row.scene === "video") return ["text-to-video", "image-to-video"]; if (row.scene === "audio") return ["text-to-audio"]; return inferNewApiWGCapabilities({ id: modelName, name: modelName, supported_endpoint_types: row.supported_endpoint_types, capabilities: row.capabilities, tags: row.tags, modalities: row.modalities, }); } async function fetchNewApiWGBillingCatalogModels( baseUrl?: string | null, searchQuery?: string ): Promise { const rootUrl = normalizeNewApiWGServerRootUrl(baseUrl); const pageSize = 1000; const allItems: NewApiWGBillingCatalogItem[] = []; for (let page = 1; page <= 20; page++) { const params = new URLSearchParams({ p: String(page), page_size: String(pageSize), }); if (searchQuery) params.set("keyword", searchQuery); const response = await Promise.resolve(fetch(`${rootUrl}/api/billing_catalog?${params.toString()}`)).catch(() => null); if (!response?.ok) break; const data = (await response.json().catch(() => ({}))) as NewApiWGBillingCatalogResponse; const items = Array.isArray(data.data?.items) ? data.data.items : []; allItems.push(...items); const total = numberFromUnknown(data.data?.total) ?? allItems.length; if (items.length === 0 || allItems.length >= total) break; } return allItems .map((row) => { const modelName = billingCatalogModelName(row); if (!modelName) return null; const capabilities = capabilitiesFromBillingScene(row, modelName); if (capabilities.length === 0) return null; const pricing = getBillingCatalogPricing(row); const model: ProviderModel = { id: modelName, name: modelName, description: null, provider: "newapiwg", capabilities, pricing, }; return model; }) .filter((model): model is ProviderModel => Boolean(model)); } async function fetchNewApiWGPricingModels(baseUrl?: string | null): Promise { const url = `${normalizeNewApiWGServerRootUrl(baseUrl)}/api/pricing`; const response = await Promise.resolve(fetch(url)).catch(() => null); if (!response?.ok) return []; const data = (await response.json().catch(() => ({}))) as NewApiWGPricingResponse; const rows = Array.isArray(data.data) ? data.data : []; return rows .map((row) => { const modelName = pricingModelName(row); if (!modelName) return null; const capabilities = inferNewApiWGCapabilities({ id: modelName, name: modelName, owned_by: row.owned_by ?? row.owner_by, supported_endpoint_types: row.supported_endpoint_types, capabilities: row.capabilities, tags: row.tags, modalities: row.modalities, }); if (capabilities.length === 0) return null; const pointCost = getNewApiWGPricingPointCost(row); const model: ProviderModel = { id: modelName, name: modelName, description: null, provider: "newapiwg" as const, capabilities, pricing: pointCost !== null ? { type: "per-run" as const, amount: pointCost, currency: "points" } : undefined, }; return model; }) .filter((model): model is ProviderModel => Boolean(model)); } function mergeNewApiWGPricing(models: ProviderModel[], pricingModels: ProviderModel[]): ProviderModel[] { if (pricingModels.length === 0) return models; const pricingById = new Map( pricingModels .filter((model) => model.pricing) .map((model) => [model.id, model.pricing!]) ); const existingIds = new Set(models.map((model) => model.id)); const mergedModels = models.map((model) => { const pricing = pricingById.get(model.id) ?? model.pricing; return pricing ? { ...model, pricing } : model; }); return [ ...mergedModels, ...pricingModels.filter((model) => !existingIds.has(model.id)), ]; } export function inferNewApiWGCapabilities(model: NewApiWGModelObject): ModelCapability[] { const rawTags = [ ...flattenTags(model.capabilities), ...flattenTags(model.capability), ...flattenTags(model.tags), ...flattenTags(model.tag), ...flattenTags(model.modalities), ...flattenTags(model.metadata?.capabilities), ...flattenTags(model.metadata?.tags), ...flattenTags(model.metadata?.modalities), ].map((tag) => tag.toLowerCase()); const capabilities = new Set(); rawTags.forEach((tag) => addCapability(capabilities, tag)); if (capabilities.size > 0) return [...capabilities]; const text = [ model.id, model.name, model.owned_by, model.description, model.metadata?.description, ...flattenTags(model.supported_endpoint_types), ...rawTags, ].filter(Boolean).join(" ").toLowerCase(); const modelNameText = [model.id, model.name].filter(Boolean).join(" ").toLowerCase(); const isDoubaoChatOnly = isDoubaoChatModelName(modelNameText); if (isDoubaoChatOnly) return []; const isImageNamedSora = /(^|[\s_\-/])sora[_\-/]?image(?=$|[\s_\-/])/.test(text); const hasImage = /image|img|vision|picture|photo|flux|sdxl|stable-diffusion|midjourney|dall-e|gpt-image|imagen|seedream|banana|ernie-image/.test(text); const hasVideo = !isImageNamedSora && /video|vidu|veo|sora|kling|runway|luma|wan|hailuo|seedance|dreamactor|gen[_-]?video|animate|motion/.test(text); const hasAudio = /audio|speech|voice|tts|music|suno|sound/.test(text); const has3d = /3d|mesh|tripo|hunyuan3d|point-e|shap-e/.test(text); const isChatOnly = /kimi|moonshot|deepseek|qwen|glm|claude|chat|embedding|rerank|(^|[\s_\-/])gpt-[45]/.test(text) || /minimax-m\d/i.test(text); const hasInputImage = /image-to-|image_to_|img2|i2|edit|inpaint|upscale|restore|controlnet|vision|gpt-image|banana|seedream|gemini.*image|image-preview/.test(text); if (has3d) capabilities.add(hasInputImage ? "image-to-3d" : "text-to-3d"); if (hasVideo) { capabilities.add("text-to-video"); capabilities.add("image-to-video"); if (/audio|sound|music|speech|voice/.test(text)) capabilities.add("audio-to-video"); } if (hasAudio && !hasVideo) capabilities.add("text-to-audio"); if (hasImage && !hasVideo && !hasAudio && !has3d) { capabilities.add("text-to-image"); if (hasInputImage) capabilities.add("image-to-image"); } if (capabilities.size === 0 && isChatOnly) return []; return capabilities.size > 0 ? [...capabilities] : DEFAULT_NEWAPIWG_CAPABILITIES; } export async function fetchNewApiWGModels( apiKey: string, baseUrl?: string | null, searchQuery?: string, loginToken?: string | null ): Promise { const url = `${normalizeNewApiWGBaseUrl(baseUrl)}/models`; const init: RequestInit = { headers: buildGatewayHeaders(apiKey, loginToken), }; const startedAt = Date.now(); const context = { searchQuery }; logGatewayRequest("Calling NewApiWG models API", url, init, context); const response = await fetch(url, init); await logGatewayResponse("NewApiWG models API response", url, init, response, context, startedAt); if (!response.ok) { const message = await response.text().catch(() => ""); const catalogModels = await fetchNewApiWGBillingCatalogModels(baseUrl, searchQuery); if (catalogModels.length > 0) { return searchQuery ? filterNewApiWGModels(catalogModels, searchQuery) : catalogModels; } const pricingModels = await fetchNewApiWGPricingModels(baseUrl); if (pricingModels.length > 0) { return searchQuery ? filterNewApiWGModels(pricingModels, searchQuery) : pricingModels; } if (response.status === 401 && message.includes("missing bearer key")) { return searchQuery ? filterNewApiWGFallbackModels(searchQuery) : NEWAPIWG_FALLBACK_MODELS; } throw new Error(`NewApiWG models API error: ${response.status}${message ? ` ${message.slice(0, 160)}` : ""}`); } const data = (await response.json()) as NewApiWGModelsResponse; const catalogModels = await fetchNewApiWGBillingCatalogModels(baseUrl, searchQuery); const pricingModels = catalogModels.length > 0 ? catalogModels : await fetchNewApiWGPricingModels(baseUrl); const rawModels = data.data ?? data.models ?? []; const normalized = rawModels .filter((model) => model.id) .map((model) => { const capabilities = inferNewApiWGCapabilities(model); const pointCost = getNewApiWGModelPointCost(model); return { id: model.id!, name: model.name || model.id!, description: model.description ?? model.metadata?.description ?? null, provider: "newapiwg" as const, capabilities, pricing: pointCost !== null ? { type: "per-run" as const, amount: pointCost, currency: "points" } : undefined, }; }) .filter((model) => model.capabilities.length > 0); const pricedModels = mergeNewApiWGPricing(normalized, pricingModels); if (!searchQuery) return pricedModels; return filterNewApiWGModels(pricedModels, searchQuery); } function filterNewApiWGModels(models: ProviderModel[], searchQuery: string): ProviderModel[] { const query = searchQuery.toLowerCase(); return models.filter((model) => model.id.toLowerCase().includes(query) || model.name.toLowerCase().includes(query) || !!model.description?.toLowerCase().includes(query) ); } function filterNewApiWGFallbackModels(searchQuery: string): ProviderModel[] { return filterNewApiWGModels(NEWAPIWG_FALLBACK_MODELS, searchQuery); } function firstString(value: unknown): string | undefined { if (typeof value === "string") return value; if (Array.isArray(value)) return value.map(firstString).find(Boolean); if (value && typeof value === "object") { return firstString(Object.values(value as Record)); } return undefined; } function getNewApiWGErrorMessage(data: unknown, fallbackError: string): string { if (data && typeof data === "object") { const record = data as Record; const error = record.error; if (error && typeof error === "object") { const errorRecord = error as Record; if (typeof errorRecord.message === "string" && errorRecord.message.length > 0) { return errorRecord.message; } if (typeof errorRecord.code === "string" && errorRecord.code.length > 0) { return errorRecord.code; } } } return firstString(data) || fallbackError; } const NEWAPIWG_TRANSIENT_RETRY_DELAYS_MS = [500, 1500]; function isNewApiWGTransientError(message: string): boolean { return /system\s+disk\s+overloaded/i.test(message); } function isNewApiWGInsufficientPointsError(message: string): boolean { return /insufficient[_\s-]*(points?|credits?|balance|quota)|not enough (points?|credits?|balance|quota)|余额不足|积分不足/i.test(message); } export function normalizeNewApiWGError(message: string, modelName = "NewApiWG"): string { const normalized = message.trim(); if (isNewApiWGTransientError(normalized)) { return `${modelName}: provider is temporarily overloaded. Please retry in a moment or switch models.`; } if (isNewApiWGInsufficientPointsError(normalized)) { return `${modelName}: NewApiWG account has insufficient points for this generation. Add balance, switch to a cheaper model, or configure another API key.`; } if ( /InputImageSensitiveContentDetected\.PrivacyInformation/i.test(normalized) || /input image may contain real person/i.test(normalized) ) { return `${modelName}: input image was rejected by the provider safety review because it may contain a real person or privacy information. Use a non-person image, anonymize the face, or try text-to-video.`; } return normalized; } async function wait(ms: number): Promise { await new Promise((resolve) => setTimeout(resolve, ms)); } async function fetchNewApiWGJsonWithRetry( url: string, init: RequestInit, modelName: string, fallbackError: string ): Promise<{ response: Response; data: unknown }> { for (let attempt = 0; attempt <= NEWAPIWG_TRANSIENT_RETRY_DELAYS_MS.length; attempt++) { const context = { modelName, attempt: attempt + 1, }; const startedAt = Date.now(); logGatewayRequest("Calling NewApiWG JSON API", url, init, context); const response = await fetch(url, init); await logGatewayResponse("NewApiWG JSON API response", url, init, response, context, startedAt); const data = await response.json().catch(() => ({})); if (response.ok) { return { response, data }; } const message = getNewApiWGErrorMessage(data, fallbackError); const canRetry = attempt < NEWAPIWG_TRANSIENT_RETRY_DELAYS_MS.length && isNewApiWGTransientError(message); if (!canRetry) { return { response, data }; } console.warn( `[NewApiWG] ${modelName} returned transient overload; retrying attempt ${attempt + 2}` ); await wait(NEWAPIWG_TRANSIENT_RETRY_DELAYS_MS[attempt]); } throw new Error("unreachable"); } function getNewApiWGTaskId(data: unknown): string | null { if (!data || typeof data !== "object") return null; const record = data as Record; const id = record.task_id ?? record.id; return typeof id === "string" ? id : null; } function getNewApiWGVideoUrl(data: unknown): string | null { if (!data || typeof data !== "object") return null; const record = data as Record; const nestedData = record.data && typeof record.data === "object" ? record.data as Record : null; const metadata = nestedData?.metadata && typeof nestedData.metadata === "object" ? nestedData.metadata as Record : null; const taskData = nestedData?.data && typeof nestedData.data === "object" ? nestedData.data as Record : null; const creations = taskData?.creations; const firstCreation = Array.isArray(creations) && creations[0] && typeof creations[0] === "object" ? creations[0] as Record : null; const candidates = [ record.url, record.video_url, nestedData?.result_url, metadata?.url, firstCreation?.url, ]; return candidates.find((value): value is string => typeof value === "string" && value.length > 0) ?? null; } function getNewApiWGTaskStatus(data: unknown): string { if (!data || typeof data !== "object") return ""; const record = data as Record; const nestedData = record.data && typeof record.data === "object" ? record.data as Record : null; const taskData = nestedData?.data && typeof nestedData.data === "object" ? nestedData.data as Record : null; return String(taskData?.state ?? nestedData?.status ?? record.status ?? "").toLowerCase(); } async function pollNewApiWGVideo( apiKey: string, baseUrl: string, taskId: string, loginToken?: string | null ): Promise { const deadline = Date.now() + 9 * 60 * 1000; while (Date.now() < deadline) { await new Promise((resolve) => setTimeout(resolve, 3000)); const url = `${baseUrl}/video/generations/${encodeURIComponent(taskId)}`; const init: RequestInit = { headers: buildGatewayHeaders(apiKey, loginToken), }; const context = { taskId }; const startedAt = Date.now(); logGatewayRequest("Polling NewApiWG video generation", url, init, context); const response = await fetch(url, init); await logGatewayResponse("NewApiWG video poll response", url, init, response, context, startedAt); if (!response.ok) { const text = await response.text().catch(() => ""); let message = text || `NewApiWG video polling error: ${response.status}`; try { message = getNewApiWGErrorMessage(JSON.parse(text), message); } catch { // Keep the plain text response. } return { success: false, error: normalizeNewApiWGError(message) }; } const data = await response.json(); const videoUrl = getNewApiWGVideoUrl(data); if (videoUrl) { return { success: true, outputs: [{ type: "video", data: "", url: videoUrl }] }; } const status = getNewApiWGTaskStatus(data); if (status === "failed" || status === "fail" || status === "error") { const message = getNewApiWGErrorMessage(data, "NewApiWG video generation failed"); return { success: false, error: normalizeNewApiWGError(message) }; } } return { success: false, error: "NewApiWG video generation timed out" }; } function promptFromInput(input: GenerationInput): string { const dynamicPrompt = input.dynamicInputs?.prompt; if (typeof dynamicPrompt === "string") return dynamicPrompt; if (Array.isArray(dynamicPrompt)) return dynamicPrompt.find((value) => typeof value === "string" && value.length > 0) ?? input.prompt; return input.prompt; } function dynamicInputsWithoutPrompt(input: GenerationInput): Record { const dynamicInputs = { ...(input.dynamicInputs || {}) }; delete dynamicInputs.prompt; return dynamicInputs; } const NEWAPIWG_VIDEO_IMAGE_INPUTS = new Set([ "image", "images", "image_url", "image_urls", "first_frame", "first_frame_url", "last_frame", "last_frame_url", "tail_image_url", "reference_image", "reference_image_url", "reference_image_urls", ]); const NEWAPIWG_VIDEO_INPUTS = new Set([ "video", "videos", "video_url", "video_urls", "reference_video", "reference_video_url", "reference_video_urls", ]); const NEWAPIWG_VIDEO_AUDIO_INPUTS = new Set([ "voices", ]); const NEWAPIWG_LEGACY_VIDEO_AUDIO_INPUTS = new Set([ "audio", "audios", "audio_url", "audio_urls", "reference_audio", "reference_audio_url", "reference_audio_urls", ]); const NEWAPIWG_IMAGE_RESOLUTION_PIXELS: Record = { "512": 512 * 512, "1K": 1024 * 1024, "2K": 2560 * 1440, "4K": 3840 * 2160, }; const NEWAPIWG_SEEDREAM_MIN_IMAGE_PIXELS = NEWAPIWG_IMAGE_RESOLUTION_PIXELS["2K"]; const REFERENCE_IMAGE_FETCH_TIMEOUT_MS = 30_000; const MAX_REFERENCE_IMAGE_BYTES = 25 * 1024 * 1024; function asStringArray(value: string | string[] | undefined): string[] { if (typeof value === "string") return value.length > 0 ? [value] : []; if (Array.isArray(value)) return value.filter((item) => typeof item === "string" && item.length > 0); return []; } function uniqueStrings(values: string[]): string[] { return [...new Set(values)]; } function isViduVideoModel(modelId: string): boolean { return /vidu/i.test(modelId); } function appendDynamicInputArray( inputs: Record, key: string, values: string[] ): void { const merged = uniqueStrings([ ...asStringArray(inputs[key]), ...values, ]); if (merged.length > 0) inputs[key] = merged; } function normalizeNewApiWGVideoInputs( input: GenerationInput, dynamicInputs: Record ): { dynamicInputs: Record; mediaInputs: Record; } { const normalizedDynamicInputs: Record = {}; const images = [...(input.images || [])]; const videos: string[] = []; const voices: string[] = []; const shouldUseViduReferenceVideoField = isViduVideoModel(input.model.id); const selectedModel = toSelectedModel(input.model); const supportsVideoReference = modelSupportsReferenceMedia(selectedModel, "video", undefined, input.parameters); const supportsAudioReference = modelSupportsReferenceMedia(selectedModel, "audio", undefined, input.parameters); for (const [key, value] of Object.entries(dynamicInputs)) { if (NEWAPIWG_VIDEO_IMAGE_INPUTS.has(key)) { images.push(...asStringArray(value)); } else if (NEWAPIWG_VIDEO_INPUTS.has(key)) { if (!supportsVideoReference) continue; const videoValues = asStringArray(value); if (shouldUseViduReferenceVideoField) { appendDynamicInputArray(normalizedDynamicInputs, "reference_video_urls", videoValues); } else { videos.push(...videoValues); } } else if (NEWAPIWG_VIDEO_AUDIO_INPUTS.has(key)) { if (!supportsAudioReference) continue; voices.push(...asStringArray(value)); } else if (NEWAPIWG_LEGACY_VIDEO_AUDIO_INPUTS.has(key)) { continue; } else { normalizedDynamicInputs[key] = value; } } const mediaInputs: Record = {}; const uniqueImages = uniqueStrings(images); const uniqueVideos = uniqueStrings(videos); const uniqueVoices = uniqueStrings(voices); if (uniqueImages.length > 0) mediaInputs.images = uniqueImages; if (uniqueVideos.length > 0) mediaInputs.videos = uniqueVideos; if (uniqueVoices.length > 0) mediaInputs.voices = uniqueVoices; return { dynamicInputs: normalizedDynamicInputs, mediaInputs }; } function toSelectedModel(model: GenerationInput["model"]): SelectedModel { return { provider: model.provider, modelId: model.id, displayName: model.name, capabilities: model.capabilities, metadata: model.metadata, }; } function sanitizeNewApiWGVideoParameters( model: GenerationInput["model"], parameters: Record | undefined ): Record | undefined { if (!parameters) return undefined; const normalized = { ...parameters }; for (const key of NEWAPIWG_LEGACY_VIDEO_AUDIO_INPUTS) { delete normalized[key]; } if (Object.prototype.hasOwnProperty.call(normalized, "referenceSubjectList")) { normalized.referenceSubjectList = filterReferenceSubjectList( normalized.referenceSubjectList, toSelectedModel(model), undefined, parameters ); if (Array.isArray(normalized.referenceSubjectList) && normalized.referenceSubjectList.length === 0) { delete normalized.referenceSubjectList; } } return Object.keys(normalized).length > 0 ? normalized : undefined; } function buildNewApiWGImageInputs(input: GenerationInput): Record { if (!input.images?.length) return {}; return { image: input.images.length === 1 ? input.images[0] : input.images }; } function getStringParameter(parameters: Record | undefined, key: string): string | null { const value = parameters?.[key]; return typeof value === "string" && value.length > 0 ? value : null; } function parseAspectRatio(value: string | null): { width: number; height: number } { const match = value?.match(/^(\d+):(\d+)$/); if (!match) return { width: 1, height: 1 }; const width = Number(match[1]); const height = Number(match[2]); if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { return { width: 1, height: 1 }; } return { width, height }; } function ceilToMultiple(value: number, multiple: number): number { return Math.ceil(value / multiple) * multiple; } function isSeedreamImageModel(modelId: string): boolean { return /seedream/i.test(modelId); } function parsePixelSize(value: string): { width: number; height: number } | null { const match = value.match(/^(\d+)x(\d+)$/i); if (!match) return null; const width = Number(match[1]); const height = Number(match[2]); if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) { return null; } return { width, height }; } function pixelSizeForAspectRatio(targetPixels: number, ratio: { width: number; height: number }): string { const width = ceilToMultiple(Math.sqrt(targetPixels * ratio.width / ratio.height), 8); const height = ceilToMultiple(Math.sqrt(targetPixels * ratio.height / ratio.width), 8); return `${width}x${height}`; } function clampPixelSize(size: string, minPixels: number): string { if (minPixels <= 0) return size; const parsed = parsePixelSize(size); if (!parsed || parsed.width * parsed.height >= minPixels) return size; return pixelSizeForAspectRatio(minPixels, parsed); } function resolveNewApiWGImageSize( parameters: Record | undefined, modelId: string ): string { const minPixels = isSeedreamImageModel(modelId) ? NEWAPIWG_SEEDREAM_MIN_IMAGE_PIXELS : 0; const requestedSize = getStringParameter(parameters, "imageSize") ?? getStringParameter(parameters, "resolution"); if (requestedSize) { if (/^\d+x\d+$/i.test(requestedSize)) return clampPixelSize(requestedSize, minPixels); const targetPixels = NEWAPIWG_IMAGE_RESOLUTION_PIXELS[requestedSize]; if (targetPixels) { const ratio = parseAspectRatio( getStringParameter(parameters, "aspectRatio") ?? getStringParameter(parameters, "aspect_ratio") ); return pixelSizeForAspectRatio(Math.max(targetPixels, minPixels), ratio); } } const explicitSize = getStringParameter(parameters, "size"); if (explicitSize) return clampPixelSize(explicitSize, minPixels); return clampPixelSize("1024x1024", minPixels); } function isImagenGenerationModel(modelId: string): boolean { return /(^|[\/_\-.])imagen(\d|[\/_\-.]|$)/i.test(modelId); } function isOpenAIChatImageModel(modelId: string): boolean { return /(^|[_\-/])sora[_\-/]?image$/i.test(modelId) || modelId === "gpt-image-2-all" || modelId === "gpt-image-2-vip"; } const GPT_IMAGE_2_1K_SIZE_BY_RATIO: Record = { "1:1": "1024x1024", "3:4": "768x1024", "4:3": "1024x768", "9:16": "720x1280", "16:9": "1280x720", }; function aspectRatioParameter( parameters: Record | undefined ): string | null { return getStringParameter(parameters, "aspectRatio") ?? getStringParameter(parameters, "aspect_ratio"); } function aspectRatioKey(ratio: { width: number; height: number }): string { return `${ratio.width}:${ratio.height}`; } function pixelSizeForLongEdge( ratio: { width: number; height: number }, longEdge: number ): string { if (ratio.width >= ratio.height) { return `${longEdge}x${ceilToMultiple((longEdge * ratio.height) / ratio.width, 8)}`; } return `${ceilToMultiple((longEdge * ratio.width) / ratio.height, 8)}x${longEdge}`; } function resolveGptImage2Size( modelId: string, parameters: Record | undefined ): string | null { const resolution = getStringParameter(parameters, "resolution") ?? getStringParameter(parameters, "imageSize") ?? getStringParameter(parameters, "image_size") ?? getStringParameter(parameters, "output_size") ?? getStringParameter(parameters, "size"); const normalizedResolution = resolution?.trim().toUpperCase(); const ratio = parseAspectRatio(aspectRatioParameter(parameters) ?? "16:9"); const ratioKey = aspectRatioKey(ratio); if (normalizedResolution === "1K") { return GPT_IMAGE_2_1K_SIZE_BY_RATIO[ratioKey] ?? pixelSizeForLongEdge(ratio, 1024); } if (normalizedResolution === "2K") return pixelSizeForLongEdge(ratio, 2048); if (normalizedResolution === "4K" && modelId === "gpt-image-2-all") { return pixelSizeForLongEdge(ratio, 2048); } if (normalizedResolution === "4K") return pixelSizeForLongEdge(ratio, 3840); return getStringParameter(parameters, "size"); } function resolveOpenAIChatImageSize( modelId: string, parameters: Record | undefined ): string | null { if (modelId === "gpt-image-2-all" || modelId === "gpt-image-2-vip") { return resolveGptImage2Size(modelId, parameters) ?? "2048x1152"; } const aspectRatio = aspectRatioParameter(parameters); if (aspectRatio) { const ratio = parseAspectRatio(aspectRatio); if (ratio.width < ratio.height) return "1024x1536"; if (ratio.width > ratio.height) return "1536x1024"; return "1024x1024"; } return getStringParameter(parameters, "size"); } function normalizeOpenAIChatImageParameters( modelId: string, parameters: Record | undefined ): Record { const normalized = { ...(parameters || {}) }; const size = resolveOpenAIChatImageSize(modelId, parameters); if (size) normalized.size = size; delete normalized.aspectRatio; delete normalized.aspect_ratio; delete normalized.resolution; delete normalized.imageSize; delete normalized.image_size; delete normalized.output_size; delete normalized.dimensions; return normalized; } function normalizeNewApiWGImageGenerationParameters( parameters: Record | undefined ): Record { const normalized = { ...(parameters || {}) }; delete normalized.size; delete normalized.image_size; delete normalized.output_size; delete normalized.dimensions; delete normalized.aspectRatio; delete normalized.aspect_ratio; delete normalized.resolution; delete normalized.imageSize; return normalized; } function isGeminiNativeImageModel(modelId: string): boolean { return /gemini-.*image-preview/i.test(modelId) || /nano[_\-/]?banana/i.test(modelId); } function toGeminiNativeModelName(modelId: string): string { if (/nano[_\-/]?banana[_\-/]?pro/i.test(modelId)) return "gemini-3-pro-image-preview"; if (/nano[_\-/]?banana[_\-/]?2/i.test(modelId)) return "gemini-3.1-flash-image-preview"; return modelId; } function normalizeNewApiWGGeminiBaseUrl(baseUrl: string): string { return baseUrl.replace(/\/v1$/, "/v1beta"); } function getStringCandidate(value: unknown): string | null { if (typeof value !== "string" || value.length === 0) return null; if (value.startsWith("data:image/")) return value; if (/^https?:\/\/\S+\.(png|jpe?g|webp)(\?\S*)?$/i.test(value)) return value; return null; } function getImageUrlCandidate(value: unknown): string | null { if (typeof value !== "string" || value.length === 0) return null; if (value.startsWith("data:image/")) return value; if (value.startsWith("http://") || value.startsWith("https://")) return value; return null; } function isHttpUrl(value: string): boolean { return value.startsWith("http://") || value.startsWith("https://"); } function extractImageFromText(text: string): string | null { const dataUrl = text.match(/data:image\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/i)?.[0]; if (dataUrl) return dataUrl; const markdownUrl = text.match(/!\[[^\]]*]\((https?:\/\/[^)\s]+)\)/i)?.[1]; if (markdownUrl) return markdownUrl; const plainUrl = text.match(/https?:\/\/\S+/i)?.[0]; if (plainUrl) return plainUrl; try { return extractNewApiWGImage(JSON.parse(text)); } catch { return null; } } function extractNewApiWGImage(value: unknown): string | null { const direct = getStringCandidate(value); if (direct) return direct; if (!value || typeof value !== "object") return null; if (Array.isArray(value)) { for (const item of value) { const found = extractNewApiWGImage(item); if (found) return found; } return null; } const record = value as Record; const b64 = record.b64_json ?? record.base64 ?? record.image_base64; if (typeof b64 === "string" && b64.length > 0) { return b64.startsWith("data:") ? b64 : `data:image/png;base64,${b64}`; } const nestedImageUrl = record.image_url; if (nestedImageUrl && typeof nestedImageUrl === "object") { const url = (nestedImageUrl as Record).url; const found = getImageUrlCandidate(url); if (found) return found; } const candidates = [ record.url, record.image, record.image_url, record.output_image_url, record.result_url, record.content, record.text, ]; for (const candidate of candidates) { if (typeof candidate === "string") { const found = getImageUrlCandidate(candidate) ?? extractImageFromText(candidate); if (found) return found; } else { const found = extractNewApiWGImage(candidate); if (found) return found; } } for (const nested of Object.values(record)) { const found = extractNewApiWGImage(nested); if (found) return found; } return null; } const MISSING_TEMPORARY_REFERENCE_IMAGE_ERROR = "Local temporary reference image was not found. Please retry the generation."; function isTemporaryImageRouteUrl(image: string): boolean { let parsed: URL; try { parsed = new URL(image); } catch { return false; } return ( (parsed.protocol === "http:" || parsed.protocol === "https:") && !parsed.search && !parsed.hash && /^\/api\/images\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(parsed.pathname) ); } function prepareNewApiWGTemporaryMediaUrl(image: string): string { const temporaryDataUrl = resolveTemporaryImageUrlToDataUrl(image); if (temporaryDataUrl) return temporaryDataUrl; if (isTemporaryImageRouteUrl(image)) { throw new Error(MISSING_TEMPORARY_REFERENCE_IMAGE_ERROR); } return image; } function prepareNewApiWGTemporaryMediaInputs( inputs: Record ): Record { const prepared: Record = {}; for (const [key, value] of Object.entries(inputs)) { prepared[key] = Array.isArray(value) ? value.map(prepareNewApiWGTemporaryMediaUrl) : prepareNewApiWGTemporaryMediaUrl(value); } return prepared; } function prepareOpenAIChatImageUrl(image: string): string { return prepareNewApiWGTemporaryMediaUrl(image); } async function generateImageViaNewApiWGChat( apiKey: string, baseUrl: string, input: GenerationInput, prompt: string, loginToken?: string | null ): Promise { let images = input.images || []; try { images = images.map(prepareOpenAIChatImageUrl); } catch (error) { return { success: false, error: error instanceof Error ? error.message : "Failed to prepare reference image", }; } const content: string | Array<{ type: string; text?: string; image_url?: { url: string } }> = images.length > 0 ? [ { type: "text", text: prompt }, ...images.map((image) => ({ type: "image_url", image_url: { url: image } })), ] : prompt; const { response, data } = await fetchNewApiWGJsonWithRetry(`${baseUrl}/chat/completions`, { method: "POST", headers: buildGatewayHeaders(apiKey, loginToken, { "Content-Type": "application/json" }), body: JSON.stringify({ model: input.model.id, messages: [{ role: "user", content }], stream: false, ...dynamicInputsWithoutPrompt(input), ...normalizeOpenAIChatImageParameters(input.model.id, input.parameters), }), }, input.model.name, "NewApiWG chat image API error"); if (!response.ok) { const message = getNewApiWGErrorMessage(data, `NewApiWG chat image API error: ${response.status}`); return { success: false, error: normalizeNewApiWGError(message, input.model.name) }; } const image = extractNewApiWGImage(data); if (!image) { const text = data && typeof data === "object" ? firstString((data as Record).choices) : undefined; return { success: false, error: typeof text === "string" && text.length > 0 ? `NewApiWG image response did not include an image: ${text.slice(0, 240)}` : "NewApiWG image response did not include an image", }; } if (image.startsWith("http")) { return { success: true, outputs: [{ type: "image", data: "", url: image }] }; } return { success: true, outputs: [{ type: "image", data: image }] }; } function dataUrlToInlineData(image: string): { mimeType: string; data: string } | null { const match = image.match(/^data:([^;,]+);base64,(.+)$/); if (!match) return null; return { mimeType: match[1], data: match[2] }; } function isLoopbackTemporaryImageUrl(image: string): boolean { let parsed: URL; try { parsed = new URL(image); } catch { return false; } const hostname = parsed.hostname.toLowerCase(); const isLoopbackHost = hostname === "localhost" || hostname === "127.0.0.1" || hostname === "[::1]" || hostname === "::1"; return ( (parsed.protocol === "http:" || parsed.protocol === "https:") && isLoopbackHost && isTemporaryImageRouteUrl(image) ); } async function imageUrlToInlineData(image: string): Promise<{ mimeType: string; data: string } | null> { if (!isHttpUrl(image)) return null; const temporaryDataUrl = resolveTemporaryImageUrlToDataUrl(image); const temporaryInlineData = temporaryDataUrl ? dataUrlToInlineData(temporaryDataUrl) : null; if (temporaryInlineData) return temporaryInlineData; const isTrustedTemporaryUrl = isLoopbackTemporaryImageUrl(image); const urlCheck = validateMediaUrl(image); if (!urlCheck.valid && !isTrustedTemporaryUrl) { throw new Error(`Blocked reference image URL: ${urlCheck.error}`); } const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), REFERENCE_IMAGE_FETCH_TIMEOUT_MS); try { const response = await fetch(image, { signal: controller.signal }); if (!response.ok) throw new Error(`Failed to fetch reference image: ${response.status}`); const contentLength = response.headers.get("content-length"); if (contentLength && Number(contentLength) > MAX_REFERENCE_IMAGE_BYTES) { throw new Error(`Reference image exceeds maximum size ${MAX_REFERENCE_IMAGE_BYTES} bytes`); } const contentType = response.headers.get("content-type")?.split(";")[0]?.trim(); if (contentType && !contentType.startsWith("image/")) { throw new Error(`Reference URL did not return an image: ${contentType}`); } const buffer = Buffer.from(await response.arrayBuffer()); if (buffer.byteLength > MAX_REFERENCE_IMAGE_BYTES) { throw new Error(`Reference image exceeds maximum size ${MAX_REFERENCE_IMAGE_BYTES} bytes`); } return { mimeType: contentType || "image/png", data: buffer.toString("base64") }; } finally { clearTimeout(timeoutId); } } async function imageToInlineData(image: string): Promise<{ mimeType: string; data: string } | null> { return dataUrlToInlineData(image) ?? imageUrlToInlineData(image); } function extractGeminiNativeImage(value: unknown): string | null { const direct = extractNewApiWGImage(value); if (direct) return direct; if (!value || typeof value !== "object") return null; if (Array.isArray(value)) { for (const item of value) { const found = extractGeminiNativeImage(item); if (found) return found; } return null; } const record = value as Record; const inlineData = record.inlineData ?? record.inline_data; if (inlineData && typeof inlineData === "object") { const inline = inlineData as Record; if (typeof inline.data === "string" && inline.data.length > 0) { const mimeType = typeof inline.mimeType === "string" ? inline.mimeType : typeof inline.mime_type === "string" ? inline.mime_type : "image/png"; return inline.data.startsWith("data:") ? inline.data : `data:${mimeType};base64,${inline.data}`; } } const fileData = record.fileData ?? record.file_data; if (fileData && typeof fileData === "object") { const file = fileData as Record; const fileUri = file.fileUri ?? file.file_uri ?? file.uri ?? file.url; const found = getImageUrlCandidate(fileUri); if (found) return found; } for (const nested of Object.values(record)) { const found = extractGeminiNativeImage(nested); if (found) return found; } return null; } async function generateImageViaNewApiWGGemini( apiKey: string, baseUrl: string, input: GenerationInput, prompt: string, loginToken?: string | null ): Promise { const parts: Array<{ text: string } | { inlineData: { mimeType: string; data: string } }> = [{ text: prompt }]; for (const image of input.images || []) { let inlineData: { mimeType: string; data: string } | null; try { inlineData = await imageToInlineData(image); } catch (error) { return { success: false, error: error instanceof Error ? error.message : "Failed to prepare reference image", }; } if (inlineData) parts.push({ inlineData }); } const aspectRatio = typeof input.parameters?.aspectRatio === "string" ? input.parameters.aspectRatio : typeof input.parameters?.aspect_ratio === "string" ? input.parameters.aspect_ratio : "1:1"; const imageSize = typeof input.parameters?.imageSize === "string" ? input.parameters.imageSize : typeof input.parameters?.resolution === "string" ? input.parameters.resolution : "2K"; const modelName = toGeminiNativeModelName(input.model.id); const { response, data } = await fetchNewApiWGJsonWithRetry(`${normalizeNewApiWGGeminiBaseUrl(baseUrl)}/models/${encodeURIComponent(modelName)}:generateContent`, { method: "POST", headers: buildGatewayHeaders(apiKey, loginToken, { "Content-Type": "application/json" }), body: JSON.stringify({ contents: [{ parts }], generationConfig: { responseModalities: ["IMAGE"], imageConfig: { aspectRatio, imageSize, }, responseFormat: { image: { aspectRatio, imageSize, }, }, }, ...dynamicInputsWithoutPrompt(input), }), }, input.model.name, "NewApiWG Gemini image API error"); if (!response.ok) { const message = getNewApiWGErrorMessage(data, `NewApiWG Gemini image API error: ${response.status}`); return { success: false, error: normalizeNewApiWGError(message, input.model.name) }; } const image = extractGeminiNativeImage(data); if (!image) return { success: false, error: "NewApiWG Gemini image response did not include an image" }; return image.startsWith("http") ? { success: true, outputs: [{ type: "image", data: "", url: image }] } : { success: true, outputs: [{ type: "image", data: image }] }; } export async function generateWithNewApiWG( apiKey: string, baseUrl: string | null, input: GenerationInput, loginToken?: string | null ): Promise { const normalizedBaseUrl = normalizeNewApiWGBaseUrl(baseUrl); if (isDoubaoChatModelName(`${input.model.id} ${input.model.name}`)) { return { success: false, error: `${input.model.name}: this is a text/chat model, not an image, video, audio, or 3D generation model. Choose a generation model such as PopiArt Nano Pro, Seedream, or Seedance.`, }; } const isVideo = input.model.capabilities.some((cap) => cap.includes("video")); const isAudio = !isVideo && input.model.capabilities.includes("text-to-audio"); const prompt = promptFromInput(input); if (isAudio) { const audioUrl = `${normalizedBaseUrl}/audio/speech`; const audioInit: RequestInit = { method: "POST", headers: buildGatewayHeaders(apiKey, loginToken, { "Content-Type": "application/json" }), body: JSON.stringify({ model: input.model.id, input: prompt, voice: typeof input.parameters?.voice === "string" ? input.parameters.voice : "alloy", ...dynamicInputsWithoutPrompt(input), ...(input.parameters || {}), }), }; const context = { modelName: input.model.name, }; const startedAt = Date.now(); logGatewayRequest("Calling NewApiWG audio API", audioUrl, audioInit, context); const response = await fetch(audioUrl, audioInit); await logGatewayResponse("NewApiWG audio API response", audioUrl, audioInit, response, context, startedAt); if (!response.ok) { const text = await response.text().catch(() => ""); let message = text || `NewApiWG audio API error: ${response.status}`; try { message = getNewApiWGErrorMessage(JSON.parse(text), message); } catch { // Keep the plain text response. } return { success: false, error: normalizeNewApiWGError(message, input.model.name) }; } const contentType = response.headers.get("content-type") || "audio/mpeg"; const buffer = Buffer.from(await response.arrayBuffer()); return { success: true, outputs: [{ type: "audio", data: `data:${contentType};base64,${buffer.toString("base64")}` }], }; } if (!isVideo && !isAudio && isGeminiNativeImageModel(input.model.id)) { return generateImageViaNewApiWGGemini(apiKey, normalizedBaseUrl, input, prompt, loginToken); } if (!isVideo && !isAudio && isOpenAIChatImageModel(input.model.id)) { return generateImageViaNewApiWGChat(apiKey, normalizedBaseUrl, input, prompt, loginToken); } const endpoint = `${normalizedBaseUrl}/${isVideo ? "video" : "images"}/generations`; const imageSize = resolveNewApiWGImageSize(input.parameters, input.model.id); const imageParameters = normalizeNewApiWGImageGenerationParameters(input.parameters); const rawDynamicInputs = dynamicInputsWithoutPrompt(input); const normalizedVideoInputs = isVideo ? normalizeNewApiWGVideoInputs(input, rawDynamicInputs) : null; let normalizedDynamicInputs: Record; let mediaInputs: Record; try { normalizedDynamicInputs = prepareNewApiWGTemporaryMediaInputs( normalizedVideoInputs?.dynamicInputs ?? rawDynamicInputs ); mediaInputs = prepareNewApiWGTemporaryMediaInputs( normalizedVideoInputs?.mediaInputs ?? buildNewApiWGImageInputs(input) ); } catch (error) { return { success: false, error: error instanceof Error ? error.message : "Failed to prepare reference media", }; } const { response, data } = await fetchNewApiWGJsonWithRetry(endpoint, { method: "POST", headers: buildGatewayHeaders(apiKey, loginToken, { "Content-Type": "application/json" }), body: JSON.stringify({ model: input.model.id, prompt, n: 1, ...normalizedDynamicInputs, ...mediaInputs, ...(!isVideo ? { size: imageSize, response_format: "b64_json" } : {}), ...(isVideo ? sanitizeNewApiWGVideoParameters(input.model, input.parameters) : imageParameters), }), }, input.model.name, `NewApiWG ${isVideo ? "video" : "image"} API error`); if (!response.ok) { const message = getNewApiWGErrorMessage(data, `NewApiWG ${isVideo ? "video" : "image"} API error: ${response.status}`); return { success: false, error: normalizeNewApiWGError(message, input.model.name) }; } if (isVideo) { const immediateUrl = getNewApiWGVideoUrl(data); if (immediateUrl) { return { success: true, outputs: [{ type: "video", data: "", url: immediateUrl }] }; } const taskId = getNewApiWGTaskId(data); if (taskId) { return pollNewApiWGVideo(apiKey, normalizedBaseUrl, taskId, loginToken); } } const dataRecord = data && typeof data === "object" ? data as Record : {}; const dataItems = Array.isArray(dataRecord.data) ? dataRecord.data : []; const imageItems = Array.isArray(dataRecord.images) ? dataRecord.images : []; const videoItems = Array.isArray(dataRecord.videos) ? dataRecord.videos : []; const outputItems = Array.isArray(dataRecord.output) ? dataRecord.output : []; const first = dataItems[0] ?? imageItems[0] ?? videoItems[0] ?? outputItems[0] ?? data; const firstRecord = first && typeof first === "object" ? first as Record : {}; const b64 = firstRecord.b64_json || firstRecord.base64 || firstRecord.image_base64; const url = firstRecord.url || firstRecord.image_url || firstRecord.video_url || firstString(first); if (typeof b64 === "string" && b64.length > 0) { const mime = isVideo ? "video/mp4" : "image/png"; const prefix = b64.startsWith("data:") ? "" : `data:${mime};base64,`; return { success: true, outputs: [{ type: isVideo ? "video" : "image", data: `${prefix}${b64}` }] }; } if (typeof url === "string" && url.length > 0) { if (url.startsWith("data:")) { return { success: true, outputs: [{ type: isVideo ? "video" : "image", data: url }] }; } if (isHttpUrl(url)) { return { success: true, outputs: [{ type: isVideo ? "video" : "image", data: "", url }] }; } } return { success: false, error: `No ${isVideo ? "video" : "image"} returned from NewApiWG` }; }