增加本地资源上传popiserver的能力

src/app/api/popi/media/upload/route.ts

@@ -3,6 +3,7 @@ import { requireLogin } from "@/app/api/_auth";
 import { ApiError } from "@/app/api/_errors";
 
 const POPI_MEDIA_UPLOAD_BASE64_PATH = "/api_client/media/uploadFileBase64";
+const POPI_MEDIA_UPLOAD_PATH = "/api_client/media/upload";
 
 type PopiserverResponse<T> = {
   status?: string;
@@ -20,6 +21,15 @@ type PopiMedia = {
   thumbUrl?: string;
 };
 
+type PopiMediaUploadData =
+  | PopiMedia
+  | PopiMedia[]
+  | {
+      urls?: string[];
+      list?: PopiMedia[];
+      files?: PopiMedia[];
+    };
+
 function getPopiBaseUrl(request: NextRequest): string {
   const baseUrl =
     process.env.POPIART_API_BASE_URL ||
@@ -55,9 +65,98 @@ function mediaResourceUrl(media: PopiMedia): string | null {
   return media.url || media.originUrl || media.path || media.originPath || null;
 }
 
+function uploadedMediaUrls(data: PopiMediaUploadData | undefined): string[] {
+  if (!data) return [];
+  if (Array.isArray(data)) {
+    return data.map(mediaResourceUrl).filter((url): url is string => Boolean(url));
+  }
+  if ("urls" in data && Array.isArray(data.urls)) {
+    return data.urls.filter((url): url is string => typeof url === "string" && url.length > 0);
+  }
+  const list = ("list" in data && Array.isArray(data.list))
+    ? data.list
+    : ("files" in data && Array.isArray(data.files))
+      ? data.files
+      : null;
+  if (list) {
+    return list.map(mediaResourceUrl).filter((url): url is string => Boolean(url));
+  }
+  const single = mediaResourceUrl(data);
+  return single ? [single] : [];
+}
+
+async function uploadMultipartToPopiserver(request: NextRequest, token: string) {
+  const formData = await request.formData();
+  const files = formData.getAll("file").filter((file): file is File => file instanceof File);
+  if (files.length === 0) {
+    return NextResponse.json(
+      { success: false, error: "Missing media file" },
+      { status: 400 }
+    );
+  }
+
+  const upstreamFormData = new FormData();
+  if (files.length > 1) {
+    upstreamFormData.append("multiple", "true");
+  }
+  files.forEach((file) => {
+    upstreamFormData.append("file", file, file.name || "canvas-media");
+  });
+
+  const url = new URL(getPopiBaseUrl(request) + POPI_MEDIA_UPLOAD_PATH);
+  if (files.length > 1) {
+    url.searchParams.set("multiple", "true");
+  }
+
+  const upstream = await fetch(url, {
+    method: "POST",
+    headers: {
+      Accept: "application/json",
+      Authorization: `Bearer ${token}`,
+      token,
+    },
+    body: upstreamFormData,
+    cache: "no-store",
+  });
+
+  const payload = (await upstream.json().catch(() => null)) as PopiserverResponse<PopiMediaUploadData> | null;
+  if (!upstream.ok) {
+    return NextResponse.json(
+      { success: false, error: payload?.message || "Popiserver media upload failed." },
+      { status: upstream.status }
+    );
+  }
+  if (!payload || payload.status !== "0000" || !payload.data) {
+    return NextResponse.json(
+      { success: false, error: payload?.message || "Popiserver media upload returned an error." },
+      { status: 502 }
+    );
+  }
+
+  const urls = uploadedMediaUrls(payload.data);
+  if (urls.length === 0) {
+    return NextResponse.json(
+      { success: false, error: "Popiserver media upload response is missing resource URL." },
+      { status: 502 }
+    );
+  }
+
+  return NextResponse.json({
+    success: true,
+    url: urls[0],
+    urls,
+    media: payload.data,
+  });
+}
+
 export async function POST(request: NextRequest) {
   try {
     const login = requireLogin(request);
+    const contentType = request.headers.get("content-type") || "";
+    if (contentType.includes("multipart/form-data")) {
+      return uploadMultipartToPopiserver(request, login.token);
+    }
+
     const body = (await request.json().catch(() => null)) as {
       data?: unknown;
       fileName?: unknown;

src/store/execution/generateAudioExecutor.ts

@@ -10,6 +10,7 @@ import { buildGenerateHeaders } from "@/store/utils/buildApiHeaders";
 import { pollGenerateTask } from "./pollTaskCompletion";
 import { runWithFallback } from "./runWithFallback";
 import type { NodeExecutionContext } from "./types";
+import { uploadPopiserverDynamicInputsForGeneration } from "@/utils/temporaryImageUpload";
 
 export interface GenerateAudioOptions {
   /** When true, falls back to stored inputPrompt if no connections provide it. */
@@ -74,13 +75,16 @@ export async function executeGenerateAudio(
   const runOnce = async (modelToUse: SelectedModel, parametersOverride?: Record<string, unknown>): Promise<void> => {
     const provider = modelToUse.provider;
     const headers = buildGenerateHeaders(provider, providerSettings);
+    const preparedDynamicInputs = provider === "popiserver"
+      ? await uploadPopiserverDynamicInputsForGeneration(dynamicInputs)
+      : { dynamicInputs, cleanupIds: [] };
 
     const requestPayload = {
       images: [],
       prompt: text,
       selectedModel: modelToUse,
       parameters: parametersOverride ?? nodeData.parameters,
-      dynamicInputs,
+      dynamicInputs: preparedDynamicInputs.dynamicInputs,
       mediaType: "audio" as const,
     };
 

src/store/execution/generateVideoExecutor.ts

@@ -27,6 +27,8 @@ import {
 } from "@/utils/videoGenerationSettings";
 import {
   cleanupTemporaryImages,
+  uploadPopiserverDynamicInputsForGeneration,
+  uploadPopiserverImagesForGeneration,
   uploadTemporaryDynamicInputsForGeneration,
   uploadTemporaryImagesForGeneration,
 } from "@/utils/temporaryImageUpload";
@@ -89,6 +91,27 @@ function mergeConnectedVideosIntoDynamicInputs(
   };
 }
 
+const hasDynamicAudioInput = (dynamicInputs: Record<string, string | string[]>): boolean =>
+  Object.entries(dynamicInputs).some(([key, value]) => {
+    if (!key.toLowerCase().includes("audio")) return false;
+    return Array.isArray(value) ? value.length > 0 : Boolean(value);
+  });
+
+function mergeConnectedAudioIntoDynamicInputs(
+  dynamicInputs: Record<string, string | string[]>,
+  audio: string[]
+): Record<string, string | string[]> {
+  if (audio.length === 0 || hasDynamicAudioInput(dynamicInputs)) {
+    return dynamicInputs;
+  }
+
+  return {
+    ...dynamicInputs,
+    audio: audio[0],
+    audios: audio,
+  };
+}
+
 export async function executeGenerateVideo(
   ctx: NodeExecutionContext,
   options: GenerateVideoOptions = {}
@@ -110,7 +133,10 @@ export async function executeGenerateVideo(
   const { useStoredFallback = false } = options;
 
   const { images: connectedImages, videos: connectedVideos, text: connectedText, audio: connectedAudio, dynamicInputs } = getConnectedInputs(node.id);
-  const requestDynamicInputs = mergeConnectedVideosIntoDynamicInputs(dynamicInputs, connectedVideos);
+  const requestDynamicInputs = mergeConnectedAudioIntoDynamicInputs(
+    mergeConnectedVideosIntoDynamicInputs(dynamicInputs, connectedVideos),
+    connectedAudio
+  );
 
   // Get fresh node data from store
   const freshNode = getFreshNode(node.id);
@@ -211,10 +237,14 @@ export async function executeGenerateVideo(
     let temporaryMediaIds: string[] = [];
 
     try {
-      const preparedImages = provider === "newapiwg"
+      const preparedImages = provider === "popiserver"
+        ? await uploadPopiserverImagesForGeneration(images)
+        : provider === "newapiwg"
           ? await uploadTemporaryImagesForGeneration(images)
           : { images, cleanupIds: [] };
-      const preparedDynamicInputs = provider === "newapiwg"
+      const preparedDynamicInputs = provider === "popiserver"
+        ? await uploadPopiserverDynamicInputsForGeneration(requestDynamicInputs)
+        : provider === "newapiwg"
           ? await uploadTemporaryDynamicInputsForGeneration(requestDynamicInputs)
           : { dynamicInputs: requestDynamicInputs, cleanupIds: [] };
       temporaryMediaIds = [

src/store/execution/nanoBananaExecutor.ts

@@ -29,6 +29,8 @@ import {
 import { isHttpMediaUrl } from "@/utils/mediaResolver";
 import {
   cleanupTemporaryImages,
+  uploadPopiserverDynamicInputsForGeneration,
+  uploadPopiserverImagesForGeneration,
   uploadTemporaryDynamicInputsForGeneration,
   uploadTemporaryImagesForGeneration,
 } from "@/utils/temporaryImageUpload";
@@ -144,10 +146,14 @@ export async function executeNanoBanana(
     let temporaryImageIds: string[] = [];
 
     try {
-      const preparedImages = provider === "newapiwg"
+      const preparedImages = provider === "popiserver"
+        ? await uploadPopiserverImagesForGeneration(images)
+        : provider === "newapiwg"
           ? await uploadTemporaryImagesForGeneration(images)
           : { images, cleanupIds: [] };
-      const preparedDynamicInputs = provider === "newapiwg"
+      const preparedDynamicInputs = provider === "popiserver"
+        ? await uploadPopiserverDynamicInputsForGeneration(sanitizedDynamicInputs)
+        : provider === "newapiwg"
           ? await uploadTemporaryDynamicInputsForGeneration(sanitizedDynamicInputs)
           : { dynamicInputs: sanitizedDynamicInputs, cleanupIds: [] };
       temporaryImageIds = [

src/utils/temporaryImageUpload.ts

@@ -39,6 +39,22 @@ function dataUrlToBlob(dataUrl: string): Blob {
   return new Blob([bytes], { type: mimeType });
 }
 
+async function localMediaToBlob(media: string): Promise<Blob> {
+  if (media.startsWith("data:")) {
+    return dataUrlToBlob(media);
+  }
+
+  if (media.startsWith("blob:")) {
+    const response = await fetch(media);
+    if (!response.ok) {
+      throw new Error(`Failed to read local media blob: HTTP ${response.status}`);
+    }
+    return response.blob();
+  }
+
+  throw new Error("Unsupported local media URL");
+}
+
 function shouldUploadTemporaryMedia(media: string): boolean {
   return (
     (media.startsWith("data:image/") || media.startsWith("data:video/")) &&
@@ -46,6 +62,15 @@ function shouldUploadTemporaryMedia(media: string): boolean {
   );
 }
 
+function shouldUploadPopiserverMedia(media: string): boolean {
+  return (
+    media.startsWith("data:image/") ||
+    media.startsWith("data:video/") ||
+    media.startsWith("data:audio/") ||
+    media.startsWith("blob:")
+  );
+}
+
 function extensionFromMime(mimeType: string | null): string {
   if (mimeType === "image/jpeg") return "jpg";
   if (mimeType === "image/webp") return "webp";
@@ -54,6 +79,12 @@ function extensionFromMime(mimeType: string | null): string {
   if (mimeType === "video/webm") return "webm";
   if (mimeType === "video/quicktime") return "mov";
   if (mimeType?.startsWith("video/")) return "mp4";
+  if (mimeType === "audio/mpeg" || mimeType === "audio/mp3") return "mp3";
+  if (mimeType === "audio/wav" || mimeType === "audio/x-wav") return "wav";
+  if (mimeType === "audio/ogg") return "ogg";
+  if (mimeType === "audio/flac") return "flac";
+  if (mimeType === "audio/aac") return "aac";
+  if (mimeType?.startsWith("audio/")) return "mp3";
   return "png";
 }
 
@@ -84,6 +115,28 @@ async function uploadTemporaryMedia(media: string): Promise<{ url: string; id: s
   return { url: sameOriginUrl, id: result.id };
 }
 
+async function uploadPopiserverMedia(media: string): Promise<string> {
+  const blob = await localMediaToBlob(media);
+  const mimeType = dataUrlMimeType(media) || blob.type || null;
+  const formData = new FormData();
+  formData.append("file", blob, `reference.${extensionFromMime(mimeType)}`);
+
+  const response = await fetch("/api/popi/media/upload", {
+    method: "POST",
+    body: formData,
+  });
+
+  const result = await response.json().catch(() => null) as
+    | { success?: boolean; url?: string; error?: string }
+    | null;
+
+  if (!response.ok || !result?.success || !result.url) {
+    throw new Error(result?.error || "Failed to upload media to PopiServer");
+  }
+
+  return result.url;
+}
+
 export async function uploadTemporaryMediaForGeneration(
   mediaItems: string[]
 ): Promise<UploadedTemporaryMedia> {
@@ -108,6 +161,27 @@ export async function uploadTemporaryImagesForGeneration(
   return { images: prepared.media, cleanupIds: prepared.cleanupIds };
 }
 
+export async function uploadPopiserverMediaForGeneration(
+  mediaItems: string[]
+): Promise<UploadedTemporaryMedia> {
+  const preparedMedia = await Promise.all(
+    mediaItems.map(async (media) => {
+      if (!shouldUploadPopiserverMedia(media)) return media;
+      return uploadPopiserverMedia(media);
+    })
+  );
+
+  return { media: preparedMedia, cleanupIds: [] };
+}
+
+export async function uploadPopiserverImagesForGeneration(
+  images: string[]
+): Promise<UploadedTemporaryImages> {
+  const prepared = await uploadPopiserverMediaForGeneration(images);
+
+  return { images: prepared.media, cleanupIds: [] };
+}
+
 export async function uploadTemporaryDynamicInputsForGeneration(
   dynamicInputs: Record<string, string | string[]>
 ): Promise<UploadedTemporaryDynamicInputs> {
@@ -141,6 +215,33 @@ export async function uploadTemporaryDynamicInputsForGeneration(
   return { dynamicInputs: Object.fromEntries(entries), cleanupIds: [...new Set(cleanupIds)] };
 }
 
+export async function uploadPopiserverDynamicInputsForGeneration(
+  dynamicInputs: Record<string, string | string[]>
+): Promise<UploadedTemporaryDynamicInputs> {
+  const uploadCache = new Map<string, Promise<string>>();
+  const prepareMedia = async (media: string): Promise<string> => {
+    if (!shouldUploadPopiserverMedia(media)) return media;
+    let upload = uploadCache.get(media);
+    if (!upload) {
+      upload = uploadPopiserverMedia(media);
+      uploadCache.set(media, upload);
+    }
+    return upload;
+  };
+
+  const entries = await Promise.all(
+    Object.entries(dynamicInputs).map(async ([key, value]) => {
+      if (typeof value === "string") {
+        return [key, await prepareMedia(value)] as const;
+      }
+
+      return [key, await Promise.all(value.map(prepareMedia))] as const;
+    })
+  );
+
+  return { dynamicInputs: Object.fromEntries(entries), cleanupIds: [] };
+}
+
 export async function cleanupTemporaryImages(ids: string[]): Promise<void> {
   if (ids.length === 0) return;