diff --git a/src/app/api/popi/media/upload/route.ts b/src/app/api/popi/media/upload/route.ts index 6ed268c3..dc0ced94 100644 --- a/src/app/api/popi/media/upload/route.ts +++ b/src/app/api/popi/media/upload/route.ts @@ -3,6 +3,7 @@ import { requireLogin } from "@/app/api/_auth"; import { ApiError } from "@/app/api/_errors"; const POPI_MEDIA_UPLOAD_BASE64_PATH = "/api_client/media/uploadFileBase64"; +const POPI_MEDIA_UPLOAD_PATH = "/api_client/media/upload"; type PopiserverResponse = { status?: string; @@ -20,6 +21,15 @@ type PopiMedia = { thumbUrl?: string; }; +type PopiMediaUploadData = + | PopiMedia + | PopiMedia[] + | { + urls?: string[]; + list?: PopiMedia[]; + files?: PopiMedia[]; + }; + function getPopiBaseUrl(request: NextRequest): string { const baseUrl = process.env.POPIART_API_BASE_URL || @@ -55,9 +65,98 @@ function mediaResourceUrl(media: PopiMedia): string | null { return media.url || media.originUrl || media.path || media.originPath || null; } +function uploadedMediaUrls(data: PopiMediaUploadData | undefined): string[] { + if (!data) return []; + if (Array.isArray(data)) { + return data.map(mediaResourceUrl).filter((url): url is string => Boolean(url)); + } + if ("urls" in data && Array.isArray(data.urls)) { + return data.urls.filter((url): url is string => typeof url === "string" && url.length > 0); + } + const list = ("list" in data && Array.isArray(data.list)) + ? data.list + : ("files" in data && Array.isArray(data.files)) + ? data.files + : null; + if (list) { + return list.map(mediaResourceUrl).filter((url): url is string => Boolean(url)); + } + const single = mediaResourceUrl(data); + return single ? [single] : []; +} + +async function uploadMultipartToPopiserver(request: NextRequest, token: string) { + const formData = await request.formData(); + const files = formData.getAll("file").filter((file): file is File => file instanceof File); + if (files.length === 0) { + return NextResponse.json( + { success: false, error: "Missing media file" }, + { status: 400 } + ); + } + + const upstreamFormData = new FormData(); + if (files.length > 1) { + upstreamFormData.append("multiple", "true"); + } + files.forEach((file) => { + upstreamFormData.append("file", file, file.name || "canvas-media"); + }); + + const url = new URL(getPopiBaseUrl(request) + POPI_MEDIA_UPLOAD_PATH); + if (files.length > 1) { + url.searchParams.set("multiple", "true"); + } + + const upstream = await fetch(url, { + method: "POST", + headers: { + Accept: "application/json", + Authorization: `Bearer ${token}`, + token, + }, + body: upstreamFormData, + cache: "no-store", + }); + + const payload = (await upstream.json().catch(() => null)) as PopiserverResponse | null; + if (!upstream.ok) { + return NextResponse.json( + { success: false, error: payload?.message || "Popiserver media upload failed." }, + { status: upstream.status } + ); + } + if (!payload || payload.status !== "0000" || !payload.data) { + return NextResponse.json( + { success: false, error: payload?.message || "Popiserver media upload returned an error." }, + { status: 502 } + ); + } + + const urls = uploadedMediaUrls(payload.data); + if (urls.length === 0) { + return NextResponse.json( + { success: false, error: "Popiserver media upload response is missing resource URL." }, + { status: 502 } + ); + } + + return NextResponse.json({ + success: true, + url: urls[0], + urls, + media: payload.data, + }); +} + export async function POST(request: NextRequest) { try { const login = requireLogin(request); + const contentType = request.headers.get("content-type") || ""; + if (contentType.includes("multipart/form-data")) { + return uploadMultipartToPopiserver(request, login.token); + } + const body = (await request.json().catch(() => null)) as { data?: unknown; fileName?: unknown; diff --git a/src/store/execution/generateAudioExecutor.ts b/src/store/execution/generateAudioExecutor.ts index 06273829..8a5770cc 100644 --- a/src/store/execution/generateAudioExecutor.ts +++ b/src/store/execution/generateAudioExecutor.ts @@ -10,6 +10,7 @@ import { buildGenerateHeaders } from "@/store/utils/buildApiHeaders"; import { pollGenerateTask } from "./pollTaskCompletion"; import { runWithFallback } from "./runWithFallback"; import type { NodeExecutionContext } from "./types"; +import { uploadPopiserverDynamicInputsForGeneration } from "@/utils/temporaryImageUpload"; export interface GenerateAudioOptions { /** When true, falls back to stored inputPrompt if no connections provide it. */ @@ -74,13 +75,16 @@ export async function executeGenerateAudio( const runOnce = async (modelToUse: SelectedModel, parametersOverride?: Record): Promise => { const provider = modelToUse.provider; const headers = buildGenerateHeaders(provider, providerSettings); + const preparedDynamicInputs = provider === "popiserver" + ? await uploadPopiserverDynamicInputsForGeneration(dynamicInputs) + : { dynamicInputs, cleanupIds: [] }; const requestPayload = { images: [], prompt: text, selectedModel: modelToUse, parameters: parametersOverride ?? nodeData.parameters, - dynamicInputs, + dynamicInputs: preparedDynamicInputs.dynamicInputs, mediaType: "audio" as const, }; diff --git a/src/store/execution/generateVideoExecutor.ts b/src/store/execution/generateVideoExecutor.ts index 1d446983..4d07ed11 100644 --- a/src/store/execution/generateVideoExecutor.ts +++ b/src/store/execution/generateVideoExecutor.ts @@ -27,6 +27,8 @@ import { } from "@/utils/videoGenerationSettings"; import { cleanupTemporaryImages, + uploadPopiserverDynamicInputsForGeneration, + uploadPopiserverImagesForGeneration, uploadTemporaryDynamicInputsForGeneration, uploadTemporaryImagesForGeneration, } from "@/utils/temporaryImageUpload"; @@ -89,6 +91,27 @@ function mergeConnectedVideosIntoDynamicInputs( }; } +const hasDynamicAudioInput = (dynamicInputs: Record): boolean => + Object.entries(dynamicInputs).some(([key, value]) => { + if (!key.toLowerCase().includes("audio")) return false; + return Array.isArray(value) ? value.length > 0 : Boolean(value); + }); + +function mergeConnectedAudioIntoDynamicInputs( + dynamicInputs: Record, + audio: string[] +): Record { + if (audio.length === 0 || hasDynamicAudioInput(dynamicInputs)) { + return dynamicInputs; + } + + return { + ...dynamicInputs, + audio: audio[0], + audios: audio, + }; +} + export async function executeGenerateVideo( ctx: NodeExecutionContext, options: GenerateVideoOptions = {} @@ -110,7 +133,10 @@ export async function executeGenerateVideo( const { useStoredFallback = false } = options; const { images: connectedImages, videos: connectedVideos, text: connectedText, audio: connectedAudio, dynamicInputs } = getConnectedInputs(node.id); - const requestDynamicInputs = mergeConnectedVideosIntoDynamicInputs(dynamicInputs, connectedVideos); + const requestDynamicInputs = mergeConnectedAudioIntoDynamicInputs( + mergeConnectedVideosIntoDynamicInputs(dynamicInputs, connectedVideos), + connectedAudio + ); // Get fresh node data from store const freshNode = getFreshNode(node.id); @@ -211,12 +237,16 @@ export async function executeGenerateVideo( let temporaryMediaIds: string[] = []; try { - const preparedImages = provider === "newapiwg" - ? await uploadTemporaryImagesForGeneration(images) - : { images, cleanupIds: [] }; - const preparedDynamicInputs = provider === "newapiwg" - ? await uploadTemporaryDynamicInputsForGeneration(requestDynamicInputs) - : { dynamicInputs: requestDynamicInputs, cleanupIds: [] }; + const preparedImages = provider === "popiserver" + ? await uploadPopiserverImagesForGeneration(images) + : provider === "newapiwg" + ? await uploadTemporaryImagesForGeneration(images) + : { images, cleanupIds: [] }; + const preparedDynamicInputs = provider === "popiserver" + ? await uploadPopiserverDynamicInputsForGeneration(requestDynamicInputs) + : provider === "newapiwg" + ? await uploadTemporaryDynamicInputsForGeneration(requestDynamicInputs) + : { dynamicInputs: requestDynamicInputs, cleanupIds: [] }; temporaryMediaIds = [ ...preparedImages.cleanupIds, ...preparedDynamicInputs.cleanupIds, diff --git a/src/store/execution/nanoBananaExecutor.ts b/src/store/execution/nanoBananaExecutor.ts index 7890ece0..5c01ed64 100644 --- a/src/store/execution/nanoBananaExecutor.ts +++ b/src/store/execution/nanoBananaExecutor.ts @@ -29,6 +29,8 @@ import { import { isHttpMediaUrl } from "@/utils/mediaResolver"; import { cleanupTemporaryImages, + uploadPopiserverDynamicInputsForGeneration, + uploadPopiserverImagesForGeneration, uploadTemporaryDynamicInputsForGeneration, uploadTemporaryImagesForGeneration, } from "@/utils/temporaryImageUpload"; @@ -144,12 +146,16 @@ export async function executeNanoBanana( let temporaryImageIds: string[] = []; try { - const preparedImages = provider === "newapiwg" - ? await uploadTemporaryImagesForGeneration(images) - : { images, cleanupIds: [] }; - const preparedDynamicInputs = provider === "newapiwg" - ? await uploadTemporaryDynamicInputsForGeneration(sanitizedDynamicInputs) - : { dynamicInputs: sanitizedDynamicInputs, cleanupIds: [] }; + const preparedImages = provider === "popiserver" + ? await uploadPopiserverImagesForGeneration(images) + : provider === "newapiwg" + ? await uploadTemporaryImagesForGeneration(images) + : { images, cleanupIds: [] }; + const preparedDynamicInputs = provider === "popiserver" + ? await uploadPopiserverDynamicInputsForGeneration(sanitizedDynamicInputs) + : provider === "newapiwg" + ? await uploadTemporaryDynamicInputsForGeneration(sanitizedDynamicInputs) + : { dynamicInputs: sanitizedDynamicInputs, cleanupIds: [] }; temporaryImageIds = [ ...preparedImages.cleanupIds, ...preparedDynamicInputs.cleanupIds, diff --git a/src/utils/temporaryImageUpload.ts b/src/utils/temporaryImageUpload.ts index 2563ac2c..71186563 100644 --- a/src/utils/temporaryImageUpload.ts +++ b/src/utils/temporaryImageUpload.ts @@ -39,6 +39,22 @@ function dataUrlToBlob(dataUrl: string): Blob { return new Blob([bytes], { type: mimeType }); } +async function localMediaToBlob(media: string): Promise { + if (media.startsWith("data:")) { + return dataUrlToBlob(media); + } + + if (media.startsWith("blob:")) { + const response = await fetch(media); + if (!response.ok) { + throw new Error(`Failed to read local media blob: HTTP ${response.status}`); + } + return response.blob(); + } + + throw new Error("Unsupported local media URL"); +} + function shouldUploadTemporaryMedia(media: string): boolean { return ( (media.startsWith("data:image/") || media.startsWith("data:video/")) && @@ -46,6 +62,15 @@ function shouldUploadTemporaryMedia(media: string): boolean { ); } +function shouldUploadPopiserverMedia(media: string): boolean { + return ( + media.startsWith("data:image/") || + media.startsWith("data:video/") || + media.startsWith("data:audio/") || + media.startsWith("blob:") + ); +} + function extensionFromMime(mimeType: string | null): string { if (mimeType === "image/jpeg") return "jpg"; if (mimeType === "image/webp") return "webp"; @@ -54,6 +79,12 @@ function extensionFromMime(mimeType: string | null): string { if (mimeType === "video/webm") return "webm"; if (mimeType === "video/quicktime") return "mov"; if (mimeType?.startsWith("video/")) return "mp4"; + if (mimeType === "audio/mpeg" || mimeType === "audio/mp3") return "mp3"; + if (mimeType === "audio/wav" || mimeType === "audio/x-wav") return "wav"; + if (mimeType === "audio/ogg") return "ogg"; + if (mimeType === "audio/flac") return "flac"; + if (mimeType === "audio/aac") return "aac"; + if (mimeType?.startsWith("audio/")) return "mp3"; return "png"; } @@ -84,6 +115,28 @@ async function uploadTemporaryMedia(media: string): Promise<{ url: string; id: s return { url: sameOriginUrl, id: result.id }; } +async function uploadPopiserverMedia(media: string): Promise { + const blob = await localMediaToBlob(media); + const mimeType = dataUrlMimeType(media) || blob.type || null; + const formData = new FormData(); + formData.append("file", blob, `reference.${extensionFromMime(mimeType)}`); + + const response = await fetch("/api/popi/media/upload", { + method: "POST", + body: formData, + }); + + const result = await response.json().catch(() => null) as + | { success?: boolean; url?: string; error?: string } + | null; + + if (!response.ok || !result?.success || !result.url) { + throw new Error(result?.error || "Failed to upload media to PopiServer"); + } + + return result.url; +} + export async function uploadTemporaryMediaForGeneration( mediaItems: string[] ): Promise { @@ -108,6 +161,27 @@ export async function uploadTemporaryImagesForGeneration( return { images: prepared.media, cleanupIds: prepared.cleanupIds }; } +export async function uploadPopiserverMediaForGeneration( + mediaItems: string[] +): Promise { + const preparedMedia = await Promise.all( + mediaItems.map(async (media) => { + if (!shouldUploadPopiserverMedia(media)) return media; + return uploadPopiserverMedia(media); + }) + ); + + return { media: preparedMedia, cleanupIds: [] }; +} + +export async function uploadPopiserverImagesForGeneration( + images: string[] +): Promise { + const prepared = await uploadPopiserverMediaForGeneration(images); + + return { images: prepared.media, cleanupIds: [] }; +} + export async function uploadTemporaryDynamicInputsForGeneration( dynamicInputs: Record ): Promise { @@ -141,6 +215,33 @@ export async function uploadTemporaryDynamicInputsForGeneration( return { dynamicInputs: Object.fromEntries(entries), cleanupIds: [...new Set(cleanupIds)] }; } +export async function uploadPopiserverDynamicInputsForGeneration( + dynamicInputs: Record +): Promise { + const uploadCache = new Map>(); + const prepareMedia = async (media: string): Promise => { + if (!shouldUploadPopiserverMedia(media)) return media; + let upload = uploadCache.get(media); + if (!upload) { + upload = uploadPopiserverMedia(media); + uploadCache.set(media, upload); + } + return upload; + }; + + const entries = await Promise.all( + Object.entries(dynamicInputs).map(async ([key, value]) => { + if (typeof value === "string") { + return [key, await prepareMedia(value)] as const; + } + + return [key, await Promise.all(value.map(prepareMedia))] as const; + }) + ); + + return { dynamicInputs: Object.fromEntries(entries), cleanupIds: [] }; +} + export async function cleanupTemporaryImages(ids: string[]): Promise { if (ids.length === 0) return;