Browse Source

音频字段修改, 视频连接bug,图片预览修改

feature/260608
TianYun 1 month ago
parent
commit
bd512b3b7b
  1. 6
      src/app/api/_popiserverModels.ts
  2. 38
      src/app/api/generate/providers/__tests__/newapiwg.test.ts
  3. 88
      src/app/api/generate/providers/__tests__/popiserver.test.ts
  4. 56
      src/app/api/generate/providers/newapiwg.ts
  5. 53
      src/app/api/generate/providers/popiserver.ts
  6. 63
      src/app/api/popi/character/list/route.ts
  7. 19
      src/app/globals.css
  8. 207
      src/components/CharacterPickerModal.tsx
  9. 57
      src/components/FloatingActionBar.tsx
  10. 106
      src/components/WorkflowCanvas.tsx
  11. 1774
      src/components/composer/GenerationComposer.tsx
  12. 30
      src/i18n/index.tsx
  13. 46
      src/store/execution/__tests__/generateVideoExecutor.test.ts
  14. 125
      src/store/execution/generateVideoExecutor.ts
  15. 101
      src/utils/generationInputSupport.ts

6
src/app/api/_popiserverModels.ts

@ -255,11 +255,11 @@ export function buildPopiModelInputs(model: PopiAIModel): ModelInput[] {
if (model.isSupportAudios) {
inputs.push({
name: "audios",
name: "voices",
type: "audio",
required: true,
label: "Audios",
description: "Input audios",
label: "Voices",
description: "Input voices",
isArray: true,
});
}

38
src/app/api/generate/providers/__tests__/newapiwg.test.ts

@ -478,10 +478,14 @@ describe("NewApiWG generation payloads", () => {
it("passes connected media inputs with NewApiWG gateway field names", async () => {
const result = await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({
images: ["data:image/png;base64,abc"],
parameters: { duration: 5 },
parameters: {
duration: 5,
audios: ["https://cdn.example.com/legacy-param-audio.mp3"],
},
dynamicInputs: {
image: "data:image/png;base64,abc",
reference_audio_urls: ["data:audio/mpeg;base64,audio"],
voices: ["https://cdn.example.com/input-audio.mp3"],
reference_audio_urls: ["https://cdn.example.com/legacy-audio.mp3"],
},
}));
@ -489,8 +493,9 @@ describe("NewApiWG generation payloads", () => {
expect(capturedBody).not.toBeNull();
expect(capturedBody!.prompt).toBe("a dancing robot");
expect(capturedBody!.images).toEqual(["data:image/png;base64,abc"]);
expect(capturedBody!.audios).toEqual(["data:audio/mpeg;base64,audio"]);
expect(capturedBody!.voices).toEqual(["https://cdn.example.com/input-audio.mp3"]);
expect(capturedBody!.image).toBeUndefined();
expect(capturedBody!.audios).toBeUndefined();
expect(capturedBody!.reference_audio_urls).toBeUndefined();
expect(capturedBody!.duration).toBe(5);
});
@ -516,7 +521,7 @@ describe("NewApiWG generation payloads", () => {
name: "viduq3-turbo",
description: null,
provider: "newapiwg",
capabilities: ["text-to-video", "image-to-video"],
capabilities: ["text-to-video", "image-to-video", "video-to-video"],
},
dynamicInputs: {
reference_video_urls: ["data:video/mp4;base64,reference"],
@ -534,6 +539,31 @@ describe("NewApiWG generation payloads", () => {
expect(capturedBody!.video_urls).toBeUndefined();
});
it("drops video references when the NewApiWG model does not support video inputs", async () => {
const result = await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({
model: {
id: "text-video-only",
name: "Text Video Only",
description: null,
provider: "newapiwg",
capabilities: ["text-to-video"],
},
dynamicInputs: {
videos: ["https://cdn.example.com/reference.mp4"],
},
parameters: {
referenceSubjectList: [
{ id: 1, type: "video", url: "https://cdn.example.com/reference.mp4", name: "视频1" },
],
},
}));
expect(result.success).toBe(true);
expect(capturedBody).not.toBeNull();
expect(capturedBody!.videos).toBeUndefined();
expect(capturedBody!.referenceSubjectList).toBeUndefined();
});
it("resolves temporary local media inputs before NewApiWG video generation", async () => {
const imageId = storeImage("data:image/png;base64,dmlkZW8tcmVm");
try {

88
src/app/api/generate/providers/__tests__/popiserver.test.ts

@ -271,7 +271,7 @@ describe("popiserver generation provider", () => {
});
});
it("uploads task input media in one batch before creating the task", async () => {
it("uploads task image and video inputs before creating the task and keeps voices as URLs", async () => {
vi.mocked(global.fetch)
.mockResolvedValueOnce({
ok: true,
@ -281,7 +281,6 @@ describe("popiserver generation provider", () => {
data: [
{ url: "https://cdn.popi.art/input-image.png" },
{ url: "https://cdn.popi.art/input-video.mp4" },
{ url: "https://cdn.popi.art/input-audio.mp3" },
],
}),
} as Response)
@ -295,10 +294,20 @@ describe("popiserver generation provider", () => {
} as Response);
const input = makeInput();
input.model.metadata = {
...input.model.metadata,
isSupportAudios: true,
isSupportVideos: true,
};
input.images = ["data:image/png;base64,aW1hZ2U="];
input.parameters = {
...input.parameters,
audios: ["https://cdn.popi.art/legacy-param-audio.mp3"],
};
input.dynamicInputs = {
videos: ["data:video/mp4;base64,dmlkZW8="],
audios: ["data:audio/mpeg;base64,YXVkaW8="],
voices: ["https://cdn.popi.art/input-audio.mp3"],
audio_url: "https://cdn.popi.art/legacy-audio-url.mp3",
};
const result = await submitPopiTask(makeRequest(), "login-token", input);
@ -312,14 +321,16 @@ describe("popiserver generation provider", () => {
expect(uploadInit?.body).toBeInstanceOf(FormData);
const uploadBody = uploadInit?.body as FormData;
expect(uploadBody.get("multiple")).toBe("true");
expect(uploadBody.getAll("file")).toHaveLength(3);
expect(uploadBody.getAll("file")).toHaveLength(2);
const [createUrl, createInit] = vi.mocked(global.fetch).mock.calls[1];
expect(String(createUrl)).toContain("/api_client/anime/task/create");
const createBody = JSON.parse(String(createInit?.body));
expect(createBody.images).toEqual(["https://cdn.popi.art/input-image.png"]);
expect(createBody.videos).toEqual(["https://cdn.popi.art/input-video.mp4"]);
expect(createBody.audios).toEqual(["https://cdn.popi.art/input-audio.mp3"]);
expect(createBody.voices).toEqual(["https://cdn.popi.art/input-audio.mp3"]);
expect(createBody.audios).toBeUndefined();
expect(createBody.audio_url).toBeUndefined();
});
it("accepts single-object media upload responses and reuses duplicate inputs", async () => {
@ -389,6 +400,10 @@ describe("popiserver generation provider", () => {
} as Response);
const input = makeInput();
input.model.metadata = {
...input.model.metadata,
isSupportVideos: true,
};
input.images = [
"https://already.cdn/source.png",
"data:image/png;base64,aW1hZ2U=",
@ -440,13 +455,15 @@ describe("popiserver generation provider", () => {
aiModelId: 29,
aiModelCode: "seedance 2.0",
aiModelCodeAlias: "doubao-seedance-2-0-fast-260128",
isSupportAudios: true,
isSupportVideos: true,
},
};
input.prompt = "@图1的人物参考@视频2的人物的动作跳舞";
input.images = ["https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg"];
input.dynamicInputs = {
videos: ["https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4"],
audios: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"],
voices: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"],
};
input.parameters = {
subType: 203,
@ -503,7 +520,7 @@ describe("popiserver generation provider", () => {
chatPrompt: "@图1的人物参考@视频2的人物的动作跳舞",
images: ["https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg"],
videos: ["https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4"],
audios: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"],
voices: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"],
referenceSubjectList: [
{
id: 1,
@ -528,6 +545,63 @@ describe("popiserver generation provider", () => {
});
});
it("drops video references when the PopiServer model does not support video inputs", async () => {
vi.mocked(global.fetch).mockResolvedValue({
ok: true,
json: () => Promise.resolve({
status: "0000",
message: "ok",
data: { id: 1195 },
}),
} as Response);
const input = makeInput();
input.model.metadata = {
...input.model.metadata,
isSupportVideos: false,
};
input.dynamicInputs = {
videos: ["https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4"],
};
input.parameters = {
...input.parameters,
referenceSubjectList: [
{
id: 1,
type: "image",
url: "stale-image-url",
name: "@图1",
},
{
id: 2,
type: "video",
url: "stale-video-url",
name: "@视频2",
},
],
};
await submitPopiTask(makeRequest(), "login-token", input);
const [, init] = vi.mocked(global.fetch).mock.calls[0];
const body = JSON.parse(String(init?.body));
expect(body.videos).toBeUndefined();
expect(body.referenceSubjectList).toEqual([
{
id: 1,
type: "image",
url: "https://example.com/1.png",
name: "图1",
},
{
id: 2,
type: "image",
url: "https://example.com/2.png",
name: "图2",
},
]);
});
it("extracts completed image task output from task list", async () => {
vi.mocked(global.fetch).mockResolvedValue({
ok: true,

56
src/app/api/generate/providers/newapiwg.ts

@ -4,6 +4,8 @@ import { resolveTemporaryImageUrlToDataUrl } from "@/lib/images/temporaryUrls";
import { validateMediaUrl } from "@/utils/urlValidation";
import { buildGatewayHeaders } from "@/app/api/_auth";
import { logGatewayRequest, logGatewayResponse } from "@/app/api/_gatewayLogging";
import type { SelectedModel } from "@/types";
import { filterReferenceSubjectList, modelSupportsReferenceMedia } from "@/utils/generationInputSupport";
type NewApiWGModelObject = {
id?: string;
@ -819,6 +821,10 @@ const NEWAPIWG_VIDEO_INPUTS = new Set([
]);
const NEWAPIWG_VIDEO_AUDIO_INPUTS = new Set([
"voices",
]);
const NEWAPIWG_LEGACY_VIDEO_AUDIO_INPUTS = new Set([
"audio",
"audios",
"audio_url",
@ -874,13 +880,17 @@ function normalizeNewApiWGVideoInputs(
const normalizedDynamicInputs: Record<string, string | string[]> = {};
const images = [...(input.images || [])];
const videos: string[] = [];
const audios: string[] = [];
const voices: string[] = [];
const shouldUseViduReferenceVideoField = isViduVideoModel(input.model.id);
const selectedModel = toSelectedModel(input.model);
const supportsVideoReference = modelSupportsReferenceMedia(selectedModel, "video", undefined, input.parameters);
const supportsAudioReference = modelSupportsReferenceMedia(selectedModel, "audio", undefined, input.parameters);
for (const [key, value] of Object.entries(dynamicInputs)) {
if (NEWAPIWG_VIDEO_IMAGE_INPUTS.has(key)) {
images.push(...asStringArray(value));
} else if (NEWAPIWG_VIDEO_INPUTS.has(key)) {
if (!supportsVideoReference) continue;
const videoValues = asStringArray(value);
if (shouldUseViduReferenceVideoField) {
appendDynamicInputArray(normalizedDynamicInputs, "reference_video_urls", videoValues);
@ -888,7 +898,10 @@ function normalizeNewApiWGVideoInputs(
videos.push(...videoValues);
}
} else if (NEWAPIWG_VIDEO_AUDIO_INPUTS.has(key)) {
audios.push(...asStringArray(value));
if (!supportsAudioReference) continue;
voices.push(...asStringArray(value));
} else if (NEWAPIWG_LEGACY_VIDEO_AUDIO_INPUTS.has(key)) {
continue;
} else {
normalizedDynamicInputs[key] = value;
}
@ -897,14 +910,47 @@ function normalizeNewApiWGVideoInputs(
const mediaInputs: Record<string, string[]> = {};
const uniqueImages = uniqueStrings(images);
const uniqueVideos = uniqueStrings(videos);
const uniqueAudios = uniqueStrings(audios);
const uniqueVoices = uniqueStrings(voices);
if (uniqueImages.length > 0) mediaInputs.images = uniqueImages;
if (uniqueVideos.length > 0) mediaInputs.videos = uniqueVideos;
if (uniqueAudios.length > 0) mediaInputs.audios = uniqueAudios;
if (uniqueVoices.length > 0) mediaInputs.voices = uniqueVoices;
return { dynamicInputs: normalizedDynamicInputs, mediaInputs };
}
function toSelectedModel(model: GenerationInput["model"]): SelectedModel {
return {
provider: model.provider,
modelId: model.id,
displayName: model.name,
capabilities: model.capabilities,
metadata: model.metadata,
};
}
function sanitizeNewApiWGVideoParameters(
model: GenerationInput["model"],
parameters: Record<string, unknown> | undefined
): Record<string, unknown> | undefined {
if (!parameters) return undefined;
const normalized = { ...parameters };
for (const key of NEWAPIWG_LEGACY_VIDEO_AUDIO_INPUTS) {
delete normalized[key];
}
if (Object.prototype.hasOwnProperty.call(normalized, "referenceSubjectList")) {
normalized.referenceSubjectList = filterReferenceSubjectList(
normalized.referenceSubjectList,
toSelectedModel(model),
undefined,
parameters
);
if (Array.isArray(normalized.referenceSubjectList) && normalized.referenceSubjectList.length === 0) {
delete normalized.referenceSubjectList;
}
}
return Object.keys(normalized).length > 0 ? normalized : undefined;
}
function buildNewApiWGImageInputs(input: GenerationInput): Record<string, string | string[]> {
if (!input.images?.length) return {};
return { image: input.images.length === 1 ? input.images[0] : input.images };
@ -1587,7 +1633,7 @@ export async function generateWithNewApiWG(
...normalizedDynamicInputs,
...mediaInputs,
...(!isVideo ? { size: imageSize, response_format: "b64_json" } : {}),
...(isVideo ? (input.parameters || {}) : imageParameters),
...(isVideo ? sanitizeNewApiWGVideoParameters(input.model, input.parameters) : imageParameters),
}),
}, input.model.name, `NewApiWG ${isVideo ? "video" : "image"} API error`);

53
src/app/api/generate/providers/popiserver.ts

@ -8,6 +8,7 @@ import {
} from "@/app/api/_popiserverModels";
import { ApiError } from "@/app/api/_errors";
import { logGatewayRequest, logGatewayResponse } from "@/app/api/_gatewayLogging";
import { modelSupportsReferenceMedia } from "@/utils/generationInputSupport";
const POPISERVER_TASK_CREATE_PATH = "/api_client/anime/task/create";
const POPISERVER_TASK_LIST_PATH = "/api_client/anime/task/list";
@ -34,14 +35,21 @@ const POPISERVER_VIDEO_DYNAMIC_KEYS = [
"video_urls",
];
const POPISERVER_AUDIO_DYNAMIC_KEYS = [
const POPISERVER_VOICE_DYNAMIC_KEYS = [
"voices",
];
const POPISERVER_LEGACY_AUDIO_KEYS = [
"audio",
"audios",
"audio_url",
"audio_urls",
"reference_audio",
"reference_audio_url",
"reference_audio_urls",
];
const POPISERVER_VOICE_DYNAMIC_KEYS = [
const POPISERVER_VOICE_ID_DYNAMIC_KEYS = [
"voiceId",
"voice_id",
];
@ -49,7 +57,7 @@ const POPISERVER_VOICE_DYNAMIC_KEYS = [
const POPISERVER_MEDIA_DYNAMIC_KEYS = [
...POPISERVER_IMAGE_DYNAMIC_KEYS,
...POPISERVER_VIDEO_DYNAMIC_KEYS,
...POPISERVER_AUDIO_DYNAMIC_KEYS,
...POPISERVER_VOICE_DYNAMIC_KEYS,
];
type PopiserverResponse<T> = {
@ -465,10 +473,18 @@ function normalizeReferenceSubjectName(name: string): string {
return name.replace(/^@+/, "");
}
function removeLegacyAudioParameters(parameters: Record<string, unknown>): Record<string, unknown> {
const normalized = { ...parameters };
POPISERVER_LEGACY_AUDIO_KEYS.forEach((key) => {
delete normalized[key];
});
return normalized;
}
function buildReferenceSubjectList(
images: string[],
videos: string[],
audios: string[],
voices: string[],
parameters: Record<string, unknown>
): ReferenceSubject[] {
const metadata = referenceSubjectMetadata(parameters);
@ -502,7 +518,7 @@ function buildReferenceSubjectList(
});
});
audios.forEach((url, index) => {
voices.forEach((url, index) => {
const id = subjects.length + 1;
const meta = metadataByType.audio[index];
subjects.push({
@ -517,7 +533,16 @@ function buildReferenceSubjectList(
}
function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | null): Record<string, unknown> {
const parameters = input.parameters || {};
const rawParameters = removeLegacyAudioParameters(input.parameters || {});
const parameters = { ...rawParameters };
delete parameters.referenceSubjectList;
const selectedModel = {
provider: "popiserver" as const,
modelId: input.model.id,
displayName: input.model.name,
capabilities: input.model.capabilities,
metadata: input.model.metadata,
};
const type = asNumber(parameters.type) ?? inferTaskType(input);
const subType = asNumber(parameters.subType) ?? inferTaskSubType(input, type);
const aspectRatio = resolveAspectRatio(parameters);
@ -538,13 +563,17 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu
...(input.images || []),
...collectDynamicMedia(input.dynamicInputs, POPISERVER_IMAGE_DYNAMIC_KEYS),
];
const videos = collectDynamicMedia(input.dynamicInputs, POPISERVER_VIDEO_DYNAMIC_KEYS);
const audios = collectDynamicMedia(input.dynamicInputs, POPISERVER_AUDIO_DYNAMIC_KEYS);
const videos = modelSupportsReferenceMedia(selectedModel, "video", undefined, rawParameters)
? collectDynamicMedia(input.dynamicInputs, POPISERVER_VIDEO_DYNAMIC_KEYS)
: [];
const voices = modelSupportsReferenceMedia(selectedModel, "audio", undefined, rawParameters)
? collectDynamicMedia(input.dynamicInputs, POPISERVER_VOICE_DYNAMIC_KEYS)
: [];
const referenceSubjectList = type === 1 || type === 2
? buildReferenceSubjectList([...new Set(images)], [...new Set(videos)], [...new Set(audios)], parameters)
? buildReferenceSubjectList([...new Set(images)], [...new Set(videos)], [...new Set(voices)], rawParameters)
: [];
const voiceId =
firstDynamicString(input.dynamicInputs, POPISERVER_VOICE_DYNAMIC_KEYS) ??
firstDynamicString(input.dynamicInputs, POPISERVER_VOICE_ID_DYNAMIC_KEYS) ??
asString(parameters.voiceId) ??
"";
@ -566,7 +595,7 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu
return {
...parameters,
...commonBody,
...(audios.length > 0 ? { audios: [...new Set(audios)] } : {}),
...(voices.length > 0 ? { voices: [...new Set(voices)] } : {}),
...(voiceId ? { voiceId } : {}),
};
}
@ -576,7 +605,7 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu
...commonBody,
...(images.length > 0 ? { images: [...new Set(images)] } : {}),
...(videos.length > 0 ? { videos: [...new Set(videos)] } : {}),
...(audios.length > 0 ? { audios: [...new Set(audios)] } : {}),
...(voices.length > 0 ? { voices: [...new Set(voices)] } : {}),
...(referenceSubjectList.length > 0 ? { referenceSubjectList } : {}),
styleId: asNumber(parameters.styleId) ?? 0,
width: asNumber(parameters.width) ?? dimensions.width,

63
src/app/api/popi/character/list/route.ts

@ -0,0 +1,63 @@
import { NextRequest, NextResponse } from "next/server";
import { requireLogin } from "@/app/api/_auth";
import { ApiError } from "@/app/api/_errors";
const POPI_CHARACTER_LIST_PATH = "/api_client/anime/character/list";
type PopiserverResponse<T> = {
status?: string;
message?: string;
data?: T;
};
function getPopiBaseUrl(request: NextRequest): string {
const baseUrl =
process.env.POPIART_API_BASE_URL ||
process.env.NEXT_PUBLIC_APP_URL ||
request.nextUrl.origin;
return baseUrl.replace(/\/+$/, "");
}
export async function GET(request: NextRequest) {
try {
const login = requireLogin(request);
const upstreamUrl = new URL(getPopiBaseUrl(request) + POPI_CHARACTER_LIST_PATH);
request.nextUrl.searchParams.forEach((value, key) => {
upstreamUrl.searchParams.set(key, value);
});
if (!upstreamUrl.searchParams.has("keyword")) {
upstreamUrl.searchParams.set("keyword", "");
}
const upstream = await fetch(upstreamUrl, {
method: "GET",
headers: {
Accept: "application/json",
Authorization: `Bearer ${login.token}`,
token: login.token,
},
cache: "no-store",
});
const payload = (await upstream.json().catch(() => null)) as PopiserverResponse<unknown> | null;
if (!upstream.ok) {
throw ApiError.provider(payload?.message || "Popiserver character list failed.", upstream.status);
}
if (!payload || payload.status !== "0000") {
throw ApiError.provider(payload?.message || "Popiserver character list returned an error.", 502);
}
return NextResponse.json({
success: true,
data: payload.data,
});
} catch (error) {
if (error instanceof ApiError) {
return NextResponse.json({ success: false, error: error.message }, { status: error.status });
}
return NextResponse.json(
{ success: false, error: error instanceof Error ? error.message : "Character list failed." },
{ status: 500 }
);
}
}

19
src/app/globals.css

@ -62,6 +62,25 @@ body {
pointer-events: none;
}
.canvas-default-cursor .react-flow,
.canvas-default-cursor .react-flow__pane {
cursor: default !important;
}
.canvas-pan-ready .react-flow,
.canvas-pan-ready .react-flow__pane,
.canvas-pan-ready .react-flow__node,
.canvas-pan-ready .react-flow__node * {
cursor: grab !important;
}
.canvas-pan-active .react-flow,
.canvas-pan-active .react-flow__pane,
.canvas-pan-active .react-flow__node,
.canvas-pan-active .react-flow__node * {
cursor: grabbing !important;
}
/* Remove default React Flow styling for output nodes to match custom nodes */
.react-flow__node-output {
border: none !important;

207
src/components/CharacterPickerModal.tsx

@ -0,0 +1,207 @@
"use client";
import { useCallback, useEffect, useMemo, useState } from "react";
import { createPortal } from "react-dom";
import { Pagination } from "antd";
import { authFetch } from "@/utils/authFetch";
import { useI18n } from "@/i18n";
export interface CharacterItem {
id: number;
title?: string;
description?: string;
avatar?: string;
image?: string;
characterName?: string;
defaultPrompt?: string;
}
interface CharacterPickerModalProps {
onClose: () => void;
onSelect: (character: CharacterItem) => void;
}
interface CharacterListPayload {
list?: CharacterItem[];
pageInfo?: {
page?: number;
pageSize?: number;
pageCount?: number;
total?: number;
};
}
interface CharacterListResponse {
success?: boolean;
data?: CharacterListPayload;
error?: string;
}
const PAGE_SIZE = 20;
function getCharacterTitle(character: CharacterItem): string {
return character.title?.trim() || `#${character.id}`;
}
export function CharacterPickerModal({ onClose, onSelect }: CharacterPickerModalProps) {
const { t } = useI18n();
const [page, setPage] = useState(1);
const [items, setItems] = useState<CharacterItem[]>([]);
const [total, setTotal] = useState(0);
const [isLoading, setIsLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const loadCharacters = useCallback(async (signal: AbortSignal) => {
setIsLoading(true);
setError(null);
try {
const params = new URLSearchParams({
keyword: "",
page: String(page),
pageSize: String(PAGE_SIZE),
});
const response = await authFetch(`/api/popi/character/list?${params.toString()}`, { signal });
const payload = (await response.json().catch(() => null)) as CharacterListResponse | null;
if (!response.ok || !payload?.success) {
throw new Error(payload?.error || t("characterPicker.loadFailed"));
}
const pageInfo = payload.data?.pageInfo;
setItems(payload.data?.list || []);
setTotal(pageInfo?.total ?? (pageInfo?.pageCount ?? 1) * (pageInfo?.pageSize ?? PAGE_SIZE));
} catch (err) {
if (err instanceof DOMException && err.name === "AbortError") return;
setError(err instanceof Error ? err.message : t("characterPicker.loadFailed"));
setItems([]);
setTotal(0);
} finally {
if (!signal.aborted) setIsLoading(false);
}
}, [page, t]);
useEffect(() => {
const controller = new AbortController();
void loadCharacters(controller.signal);
return () => controller.abort();
}, [loadCharacters]);
useEffect(() => {
const handleKeyDown = (event: KeyboardEvent) => {
if (event.key === "Escape") onClose();
};
document.addEventListener("keydown", handleKeyDown);
return () => document.removeEventListener("keydown", handleKeyDown);
}, [onClose]);
const content = useMemo(() => {
if (isLoading) {
return (
<div className="flex h-80 items-center justify-center text-sm text-neutral-400">
{t("characterPicker.loading")}
</div>
);
}
if (error) {
return (
<div className="flex h-80 flex-col items-center justify-center gap-3 text-sm text-neutral-400">
<span>{error}</span>
<button
type="button"
onClick={() => {
const controller = new AbortController();
void loadCharacters(controller.signal);
}}
className="rounded-md border border-neutral-600 bg-neutral-800 px-3 py-1.5 text-neutral-200 hover:bg-neutral-700"
>
{t("characterPicker.retry")}
</button>
</div>
);
}
const selectableItems = items.filter((item) => item.image && (item.avatar || item.image));
if (selectableItems.length === 0) {
return (
<div className="flex h-80 items-center justify-center text-sm text-neutral-500">
{t("characterPicker.empty")}
</div>
);
}
return (
<div className="grid max-h-[68vh] grid-cols-2 gap-3 overflow-y-auto pr-1 sm:grid-cols-3 lg:grid-cols-4">
{selectableItems.map((character) => {
const title = getCharacterTitle(character);
const previewUrl = character.avatar || character.image;
if (!previewUrl || !character.image) return null;
return (
<button
key={character.id}
type="button"
onClick={() => onSelect(character)}
className="group overflow-hidden rounded-lg border border-neutral-700 bg-neutral-900 text-left transition-colors hover:border-neutral-500 hover:bg-neutral-800"
>
<div className="relative aspect-square bg-neutral-950">
<img src={previewUrl} alt={title} className="h-full w-full object-cover" />
</div>
<div className="flex min-h-12 flex-col justify-center gap-0.5 px-2 py-1.5">
<span className="truncate text-xs text-neutral-300">{title}</span>
<span className="truncate text-[10px] text-neutral-500">
{character.description?.trim() || t("characterPicker.character")}
</span>
</div>
</button>
);
})}
</div>
);
}, [error, isLoading, items, loadCharacters, onSelect, t]);
if (typeof document === "undefined") return null;
return createPortal(
<div
role="dialog"
aria-modal="true"
aria-label={t("characterPicker.title")}
className="fixed inset-0 z-[1000] flex items-center justify-center bg-black/70 p-5"
onClick={onClose}
>
<div
className="flex w-full max-w-4xl flex-col rounded-xl border border-neutral-700 bg-neutral-950 p-4 shadow-2xl"
onClick={(event) => event.stopPropagation()}
>
<div className="mb-4 flex items-center justify-between gap-3">
<h2 className="text-base font-medium text-neutral-100">{t("characterPicker.title")}</h2>
<button
type="button"
aria-label={t("common.close")}
onClick={onClose}
className="flex h-8 w-8 items-center justify-center rounded-md text-neutral-400 hover:bg-neutral-800 hover:text-neutral-100"
>
<svg className="h-4 w-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={2}>
<path strokeLinecap="round" strokeLinejoin="round" d="M6 18 18 6M6 6l12 12" />
</svg>
</button>
</div>
{content}
<div className="mt-4 flex justify-end">
<Pagination
size="small"
current={page}
total={total}
pageSize={PAGE_SIZE}
showSizeChanger={false}
hideOnSinglePage
onChange={setPage}
/>
</div>
</div>
</div>,
document.body
);
}

57
src/components/FloatingActionBar.tsx

@ -1,7 +1,7 @@
"use client";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { CloseOutlined, PictureOutlined, PlusOutlined } from "@ant-design/icons";
import { PictureOutlined, PlusOutlined, UserOutlined } from "@ant-design/icons";
import { useWorkflowStore } from "@/store/workflowStore";
import { NodeType } from "@/types";
import { useReactFlow } from "@xyflow/react";
@ -10,6 +10,7 @@ import { buildDefaultGenerationNodeData } from "@/utils/defaultGenerationNodeMod
import { GenerateNodeMenu } from "./GenerateNodeMenu";
import { ModelSearchDialog } from "./modals/ModelSearchDialog";
import { AssetPickerModal, PopiAssetItem } from "./AssetPickerModal";
import { CharacterPickerModal, CharacterItem } from "./CharacterPickerModal";
import { getPopiImageAssetPreview, resolvePopiImageAsset } from "@/lib/popiAssetApi";
import { useFTUXStore, TutorialStep } from "@/store/ftuxStore";
@ -198,6 +199,59 @@ function ResourceImageButton() {
);
}
function CharacterButton() {
const { t } = useI18n();
const [isCharacterPickerOpen, setIsCharacterPickerOpen] = useState(false);
const addNode = useWorkflowStore((state) => state.addNode);
const selectSingleNode = useWorkflowStore((state) => state.selectSingleNode);
const { screenToFlowPosition } = useReactFlow();
const handleSelectCharacter = (character: CharacterItem) => {
if (!character.image) return;
const center = getPaneCenter();
const position = screenToFlowPosition({
x: center.x + Math.random() * 100 - 50,
y: center.y + Math.random() * 100 - 50,
});
const title = character.title?.trim() || `character-${character.id}`;
const nodeId = addNode("smartImage", position, {
image: character.image,
imageRef: undefined,
previewImage: character.avatar || character.image,
assetDetailLoading: false,
assetDetailError: null,
filename: title,
dimensions: null,
});
selectSingleNode(nodeId);
setIsCharacterPickerOpen(false);
};
return (
<>
<button
onClick={() => setIsCharacterPickerOpen(true)}
title={t("characterPicker.open")}
aria-label={t("characterPicker.open")}
className="flex h-8 w-8 items-center justify-center rounded-lg text-neutral-300 transition-colors hover:bg-neutral-700 hover:text-neutral-100"
>
<UserOutlined className="text-[18px]" />
<span className="sr-only">{t("characterPicker.open")}</span>
</button>
{isCharacterPickerOpen && (
<CharacterPickerModal
onClose={() => setIsCharacterPickerOpen(false)}
onSelect={handleSelectCharacter}
/>
)}
</>
);
}
export function FloatingActionBar() {
const { t } = useI18n();
const nodes = useWorkflowStore((state) => state.nodes) ?? [];
@ -331,6 +385,7 @@ export function FloatingActionBar() {
<div className="flex flex-col items-center gap-2 bg-neutral-800/95 rounded-2xl shadow-lg border border-neutral-700/80 px-1.5 py-1.5">
<GenerateComboButton />
<ResourceImageButton />
<CharacterButton />
<button
onClick={() => setModelSearchOpen(true)}

106
src/components/WorkflowCanvas.tsx

@ -393,6 +393,18 @@ export const isPanningRef = { current: false };
/** Shared ref so child components (BaseNode) can skip hover updates during node drags */
export const isDraggingNodeRef = { current: false };
type PanOnDragValue = boolean | number[];
function getCanvasPanOnDrag(disabled: boolean, spacePanActive: boolean): PanOnDragValue {
if (disabled) return false;
if (spacePanActive) return [0, 1];
return [1];
}
function canUseSpacePan(disabled: boolean): boolean {
return !disabled;
}
export function WorkflowCanvas() {
const { t } = useI18n();
const popiProviderMode = isPopiProviderMode();
@ -496,6 +508,8 @@ export function WorkflowCanvas() {
// FTUX tutorial state (client-side only to avoid SSR hydration issues)
const [tutorialActive, setTutorialActive] = useState(false);
const [lockedFeatures, setLockedFeatures] = useState(false);
const [isSpacePanActive, setIsSpacePanActive] = useState(false);
const [isCanvasPanPointerActive, setIsCanvasPanPointerActive] = useState(false);
useEffect(() => {
// Subscribe to FTUX store on client-side only
@ -512,6 +526,71 @@ export function WorkflowCanvas() {
return unsubscribe;
}, []);
const canvasPanDisabled = tutorialActive || isModalOpen;
const spacePanEnabled = canUseSpacePan(canvasPanDisabled);
const canvasPanOnDrag = useMemo(
() => getCanvasPanOnDrag(canvasPanDisabled, isSpacePanActive),
[canvasPanDisabled, isSpacePanActive]
);
const handleCanvasMouseDownCapture = useCallback((event: ReactMouseEvent<HTMLDivElement>) => {
const canPanWithButton = Array.isArray(canvasPanOnDrag) && canvasPanOnDrag.includes(event.button);
if (!canPanWithButton) return;
const target = event.target;
if (!(target instanceof HTMLElement)) return;
if (!target.closest(".react-flow") || target.closest(".nopan")) return;
setIsCanvasPanPointerActive(true);
event.preventDefault();
}, [canvasPanOnDrag]);
useEffect(() => {
if (!spacePanEnabled && isSpacePanActive) {
setIsSpacePanActive(false);
setIsCanvasPanPointerActive(false);
}
}, [isSpacePanActive, spacePanEnabled]);
useEffect(() => {
const handleSpacePanKeyDown = (event: KeyboardEvent) => {
if (!spacePanEnabled || event.code !== "Space" || isLocalKeyboardTarget(event.target)) return;
event.preventDefault();
setIsSpacePanActive(true);
};
const handleSpacePanKeyUp = (event: KeyboardEvent) => {
if (event.code === "Space") {
setIsSpacePanActive(false);
setIsCanvasPanPointerActive(false);
}
};
const handlePointerUp = () => {
setIsCanvasPanPointerActive(false);
};
const handleWindowBlur = () => {
setIsSpacePanActive(false);
setIsCanvasPanPointerActive(false);
};
window.addEventListener("keydown", handleSpacePanKeyDown);
window.addEventListener("keyup", handleSpacePanKeyUp);
window.addEventListener("mouseup", handlePointerUp);
window.addEventListener("pointerup", handlePointerUp);
window.addEventListener("pointercancel", handlePointerUp);
window.addEventListener("dragend", handlePointerUp);
window.addEventListener("contextmenu", handlePointerUp);
window.addEventListener("blur", handleWindowBlur);
return () => {
window.removeEventListener("keydown", handleSpacePanKeyDown);
window.removeEventListener("keyup", handleSpacePanKeyUp);
window.removeEventListener("mouseup", handlePointerUp);
window.removeEventListener("pointerup", handlePointerUp);
window.removeEventListener("pointercancel", handlePointerUp);
window.removeEventListener("dragend", handlePointerUp);
window.removeEventListener("contextmenu", handlePointerUp);
window.removeEventListener("blur", handleWindowBlur);
};
}, [spacePanEnabled]);
// Detect if canvas is empty for showing quickstart
const isCanvasEmpty = nodes.length === 0;
@ -2676,7 +2755,12 @@ export function WorkflowCanvas() {
return (
<div
ref={reactFlowWrapper}
className={`flex-1 bg-canvas-bg relative ${isDragOver ? "ring-2 ring-inset ring-blue-500" : ""}`}
className={`canvas-default-cursor flex-1 bg-canvas-bg relative ${
isSpacePanActive ? "canvas-pan-ready" : ""
} ${isCanvasPanPointerActive ? "canvas-pan-active" : ""} ${
isDragOver ? "ring-2 ring-inset ring-blue-500" : ""
}`}
onMouseDownCapture={handleCanvasMouseDownCapture}
onClickCapture={handleQuickAddClickCapture}
onDoubleClickCapture={handleCanvasDoubleClickCapture}
onDragOver={handleDragOver}
@ -2733,15 +2817,9 @@ export function WorkflowCanvas() {
fitViewOptions={FIT_VIEW_OPTIONS}
deleteKeyCode={["Backspace", "Delete"]}
multiSelectionKeyCode="Shift"
selectionOnDrag={tutorialActive ? false : !isModalOpen}
selectionOnDrag={tutorialActive ? false : !isModalOpen && !isSpacePanActive}
selectionKeyCode={null}
panOnDrag={
tutorialActive
? false
: isModalOpen
? false
: false
}
panOnDrag={canvasPanOnDrag}
selectNodesOnDrag={false}
nodeDragThreshold={5}
nodeClickDistance={5}
@ -2751,14 +2829,8 @@ export function WorkflowCanvas() {
minZoom={0.1}
maxZoom={4}
defaultViewport={{ x: 0, y: 0, zoom: 1 }}
panActivationKeyCode={
tutorialActive
? null
: isModalOpen
? null
: "Space"
}
nodesDraggable={!isModalOpen}
panActivationKeyCode={null}
nodesDraggable={!isModalOpen && !isSpacePanActive}
nodesConnectable={!isModalOpen}
elementsSelectable={!isModalOpen}
className="bg-neutral-900"

1774
src/components/composer/GenerationComposer.tsx

File diff suppressed because it is too large

30
src/i18n/index.tsx

@ -382,6 +382,13 @@ const en = {
"assetPicker.retry": "Retry",
"assetPicker.asset": "Asset",
"assetPicker.page": "Page {page} / {pageCount}",
"characterPicker.open": "Choose character",
"characterPicker.title": "Choose character",
"characterPicker.loading": "Loading characters...",
"characterPicker.empty": "No characters available",
"characterPicker.loadFailed": "Failed to load characters",
"characterPicker.retry": "Retry",
"characterPicker.character": "Character",
"imageInput.replace": "Replace",
"imageInput.crop": "Crop",
"imageInput.draw": "Draw",
@ -694,6 +701,7 @@ const en = {
"composer.reference": "Reference",
"composer.referenceImage": "Reference image {index}",
"composer.removeReferenceImage": "Remove reference image {index}",
"composer.removeConnection": "Remove connection",
"composer.videoSubTypeMultiReference": "Image to video",
"composer.videoSubTypeTextToVideo": "All-purpose reference",
"composer.videoSubTypeFirstLastFrame": "First/last frame",
@ -1343,6 +1351,13 @@ const zhCN: Dictionary = {
"assetPicker.retry": "重试",
"assetPicker.asset": "资产",
"assetPicker.page": "第 {page} / {pageCount} 页",
"characterPicker.open": "选择角色",
"characterPicker.title": "选择角色",
"characterPicker.loading": "正在加载角色...",
"characterPicker.empty": "暂无可用角色",
"characterPicker.loadFailed": "角色加载失败",
"characterPicker.retry": "重试",
"characterPicker.character": "角色",
"imageInput.replace": "替换",
"imageInput.crop": "裁剪",
"imageInput.draw": "绘制",
@ -1655,6 +1670,7 @@ const zhCN: Dictionary = {
"composer.reference": "参考",
"composer.referenceImage": "参考图片 {index}",
"composer.removeReferenceImage": "移除参考图片 {index}",
"composer.removeConnection": "移除连接",
"composer.commands": "指令",
"composer.referenceMaterials": "引用素材",
"composer.commandCharacter": "生成角色图",
@ -2134,6 +2150,13 @@ const zhTW: Dictionary = {
"assetPicker.retry": "重試",
"assetPicker.asset": "資產",
"assetPicker.page": "第 {page} / {pageCount} 頁",
"characterPicker.open": "選擇角色",
"characterPicker.title": "選擇角色",
"characterPicker.loading": "正在載入角色...",
"characterPicker.empty": "暫無可用角色",
"characterPicker.loadFailed": "角色載入失敗",
"characterPicker.retry": "重試",
"characterPicker.character": "角色",
"imageInput.replace": "替換",
"imageInput.crop": "裁剪",
"imageInput.draw": "繪製",
@ -2787,6 +2810,13 @@ const ja: Dictionary = {
"assetPicker.retry": "再試行",
"assetPicker.asset": "アセット",
"assetPicker.page": "{page} / {pageCount} ページ",
"characterPicker.open": "キャラクターを選択",
"characterPicker.title": "キャラクターを選択",
"characterPicker.loading": "キャラクターを読み込み中...",
"characterPicker.empty": "利用可能なキャラクターがありません",
"characterPicker.loadFailed": "キャラクターの読み込みに失敗しました",
"characterPicker.retry": "再試行",
"characterPicker.character": "キャラクター",
"imageInput.replace": "置換",
"imageInput.crop": "切り抜き",
"imageInput.draw": "描画",

46
src/store/execution/__tests__/generateVideoExecutor.test.ts

@ -206,6 +206,52 @@ describe("executeGenerateVideo", () => {
});
});
it("passes connected audio inputs as voices for video generation", async () => {
const node = makeNode();
mockFetch.mockImplementation(async (url: string) => {
if (url === "/api/generate/task/create") {
return {
ok: true,
json: () => Promise.resolve({ success: true, taskId: "task-1" }),
};
}
if (url.startsWith("/api/generate/task/poll")) {
return {
ok: true,
json: () => Promise.resolve({
status: "completed",
result: { success: true, video: "data:video/mp4;base64,output" },
}),
};
}
throw new Error(`Unexpected fetch URL: ${url}`);
});
const ctx = makeCtx(node, {
getConnectedInputs: vi.fn().mockReturnValue({
images: [],
videos: ["https://cdn.example.com/unsupported-video.mp4"],
audio: ["https://cdn.example.com/input-audio.mp3"],
text: null,
dynamicInputs: {
videos: ["https://cdn.example.com/legacy-video.mp4"],
video_url: "https://cdn.example.com/legacy-video-url.mp4",
audios: ["https://cdn.example.com/legacy-audio.mp3"],
audio_url: "https://cdn.example.com/legacy-audio-url.mp3",
motion: "cinematic",
},
easeCurve: null,
}),
});
await executeGenerateVideo(ctx);
const body = JSON.parse(mockFetch.mock.calls[0][1].body);
expect(body.dynamicInputs).toEqual({
voices: ["https://cdn.example.com/input-audio.mp3"],
motion: "cinematic",
});
});
it("should preserve schema-mapped video dynamic input", async () => {
const node = makeNode();
mockFetch.mockResolvedValueOnce({

125
src/store/execution/generateVideoExecutor.ts

@ -32,6 +32,7 @@ import {
} from "@/utils/temporaryImageUpload";
import { mergeConfiguredParametersOverride } from "./generationParameters";
import { buildDefaultParametersFromSchema, mergeSchemaDefaults } from "@/utils/modelSchemaDefaults";
import { modelSupportsReferenceMedia } from "@/utils/generationInputSupport";
export interface GenerateVideoOptions {
/** When true, falls back to stored inputImages/inputPrompt if no connections provide them. */
@ -127,12 +128,60 @@ function buildVideoParameters(
return Object.keys(parameters).length > 0 ? parameters : undefined;
}
function sanitizeReferenceSubjectListParameter(
parameters: Record<string, unknown> | undefined,
nodeData: GenerateVideoNodeData,
modelToUse: SelectedModel
): Record<string, unknown> | undefined {
if (!parameters || !Array.isArray(parameters.referenceSubjectList)) return parameters;
const supportsImage = modelSupportsReferenceMedia(modelToUse, "image", nodeData.inputSchema, parameters);
const supportsVideo = modelSupportsReferenceMedia(modelToUse, "video", nodeData.inputSchema, parameters);
const supportsAudio = modelSupportsReferenceMedia(modelToUse, "audio", nodeData.inputSchema, parameters);
const referenceSubjectList = parameters.referenceSubjectList.filter((item) => {
if (!item || typeof item !== "object" || Array.isArray(item)) return false;
const type = (item as Record<string, unknown>).type;
if (type === "image") return supportsImage;
if (type === "video") return supportsVideo;
if (type === "audio") return supportsAudio;
return true;
});
return referenceSubjectList.length > 0
? { ...parameters, referenceSubjectList }
: Object.fromEntries(Object.entries(parameters).filter(([key]) => key !== "referenceSubjectList"));
}
const hasDynamicVideoInput = (dynamicInputs: Record<string, string | string[]>): boolean =>
Object.entries(dynamicInputs).some(([key, value]) => {
if (!key.toLowerCase().includes("video")) return false;
return Array.isArray(value) ? value.length > 0 : Boolean(value);
});
const LEGACY_VIDEO_AUDIO_DYNAMIC_KEYS = new Set([
"audio",
"audios",
"audio_url",
"audio_urls",
"reference_audio",
"reference_audio_url",
"reference_audio_urls",
]);
function removeLegacyVideoAudioDynamicInputs(
dynamicInputs: Record<string, string | string[]>
): Record<string, string | string[]> {
return Object.fromEntries(
Object.entries(dynamicInputs).filter(([key]) => !LEGACY_VIDEO_AUDIO_DYNAMIC_KEYS.has(key))
);
}
function removeVideoDynamicInputs(
dynamicInputs: Record<string, string | string[]>
): Record<string, string | string[]> {
return Object.fromEntries(
Object.entries(dynamicInputs).filter(([key]) => !key.toLowerCase().includes("video"))
);
}
function mergeConnectedVideosIntoDynamicInputs(
dynamicInputs: Record<string, string | string[]>,
videos: string[]
@ -148,24 +197,23 @@ function mergeConnectedVideosIntoDynamicInputs(
};
}
const hasDynamicAudioInput = (dynamicInputs: Record<string, string | string[]>): boolean =>
const hasDynamicVoiceInput = (dynamicInputs: Record<string, string | string[]>): boolean =>
Object.entries(dynamicInputs).some(([key, value]) => {
if (!key.toLowerCase().includes("audio")) return false;
if (key !== "voices") return false;
return Array.isArray(value) ? value.length > 0 : Boolean(value);
});
function mergeConnectedAudioIntoDynamicInputs(
function mergeConnectedVoicesIntoDynamicInputs(
dynamicInputs: Record<string, string | string[]>,
audio: string[]
voices: string[]
): Record<string, string | string[]> {
if (audio.length === 0 || hasDynamicAudioInput(dynamicInputs)) {
if (voices.length === 0 || hasDynamicVoiceInput(dynamicInputs)) {
return dynamicInputs;
}
return {
...dynamicInputs,
audio: audio[0],
audios: audio,
voices,
};
}
@ -194,22 +242,30 @@ export async function executeGenerateVideo(
audio: connectedAudio,
dynamicInputs,
} = getConnectedInputs(node.id);
let requestDynamicInputs = mergeConnectedAudioIntoDynamicInputs(
mergeConnectedVideosIntoDynamicInputs(dynamicInputs, connectedVideos),
connectedAudio
);
const baseDynamicInputs = removeLegacyVideoAudioDynamicInputs(dynamicInputs);
// Get fresh node data from store
const freshNode = getFreshNode(node.id);
const nodeData = (freshNode?.data || node.data) as GenerateVideoNodeData;
const localVideo = (nodeData as GenerateVideoNodeData & { video?: string | null }).video;
if (connectedVideos.length === 0 && localVideo) {
requestDynamicInputs = mergeConnectedAudioIntoDynamicInputs(
mergeConnectedVideosIntoDynamicInputs(dynamicInputs, [localVideo]),
const nodeConfig = readVideoGenerationConfig(nodeData, createNewApiWGDefaultVideoModel());
const initialSupportsVideoReference = nodeConfig.selectedModel?.modelId
? modelSupportsReferenceMedia(nodeConfig.selectedModel, "video", nodeData.inputSchema, nodeConfig.parameters)
: false;
const mediaDynamicInputs = initialSupportsVideoReference
? baseDynamicInputs
: removeVideoDynamicInputs(baseDynamicInputs);
const effectiveConnectedVideos = initialSupportsVideoReference ? connectedVideos : [];
let requestDynamicInputs = mergeConnectedVoicesIntoDynamicInputs(
mergeConnectedVideosIntoDynamicInputs(mediaDynamicInputs, effectiveConnectedVideos),
connectedAudio
);
if (initialSupportsVideoReference && connectedVideos.length === 0 && localVideo) {
requestDynamicInputs = mergeConnectedVoicesIntoDynamicInputs(
mergeConnectedVideosIntoDynamicInputs(baseDynamicInputs, [localVideo]),
connectedAudio
);
}
const nodeConfig = readVideoGenerationConfig(nodeData, createNewApiWGDefaultVideoModel());
if (!isValidVideoDurationSeconds(nodeConfig.durationSeconds)) {
const error = "Video duration must be at least 4 seconds";
@ -222,16 +278,16 @@ export async function executeGenerateVideo(
let text: string | null;
if (useStoredFallback) {
const effectiveConnectedVideos = connectedVideos.length > 0
? connectedVideos
: localVideo
const effectiveFallbackVideos = effectiveConnectedVideos.length > 0
? effectiveConnectedVideos
: initialSupportsVideoReference && localVideo
? [localVideo]
: [];
images = connectedImages.length > 0 ? connectedImages : nodeConfig.inputImages;
text = buildPromptFromConnectedTextItems(connectedTextItems, connectedText, nodeConfig.prompt);
const hasPrompt = text || requestDynamicInputs.prompt || requestDynamicInputs.negative_prompt;
const hasAudio = connectedAudio.length > 0;
const hasVideo = effectiveConnectedVideos.length > 0 || hasDynamicVideoInput(requestDynamicInputs);
const hasVideo = effectiveFallbackVideos.length > 0 || hasDynamicVideoInput(requestDynamicInputs);
if (!hasPrompt && images.length === 0 && !hasVideo && !hasAudio) {
updateNodeData(node.id, {
status: "error",
@ -244,7 +300,7 @@ export async function executeGenerateVideo(
text = buildPromptFromConnectedTextItems(connectedTextItems, connectedText, nodeConfig.prompt);
const hasPrompt = text || requestDynamicInputs.prompt || requestDynamicInputs.negative_prompt;
const hasAudio = connectedAudio.length > 0;
const hasVideo = connectedVideos.length > 0 || Boolean(localVideo) || hasDynamicVideoInput(requestDynamicInputs);
const hasVideo = effectiveConnectedVideos.length > 0 || (initialSupportsVideoReference && Boolean(localVideo)) || hasDynamicVideoInput(requestDynamicInputs);
if (!hasPrompt && images.length === 0 && !hasVideo && !hasAudio) {
updateNodeData(node.id, {
status: "error",
@ -309,13 +365,38 @@ export async function executeGenerateVideo(
const provider = modelToUse.provider;
const headers = buildGenerateHeaders(provider, providerSettings);
try {
const preparedInputs = await uploadPopiserverGenerationInputsForGeneration(images, requestDynamicInputs);
const runSupportsVideoReference = modelSupportsReferenceMedia(
modelToUse,
"video",
nodeData.inputSchema,
parametersOverride ?? nodeConfig.parameters
);
const runVideoInputs = runSupportsVideoReference
? connectedVideos.length > 0
? connectedVideos
: localVideo
? [localVideo]
: []
: [];
const runBaseDynamicInputs = runSupportsVideoReference
? baseDynamicInputs
: removeVideoDynamicInputs(baseDynamicInputs);
const runDynamicInputs = mergeConnectedVoicesIntoDynamicInputs(
mergeConnectedVideosIntoDynamicInputs(runBaseDynamicInputs, runVideoInputs),
connectedAudio
);
const preparedInputs = await uploadPopiserverGenerationInputsForGeneration(images, runDynamicInputs);
const parameters = sanitizeReferenceSubjectListParameter(
buildVideoParameters(nodeData, nodeConfig, modelToUse, preparedInputs.images.length, parametersOverride),
nodeData,
modelToUse
);
const requestPayload = {
images: preparedInputs.images,
prompt: text || "",
selectedModel: modelToUse,
parameters: buildVideoParameters(nodeData, nodeConfig, modelToUse, preparedInputs.images.length, parametersOverride),
parameters,
dynamicInputs: preparedInputs.dynamicInputs,
mediaType: "video" as const,
};

101
src/utils/generationInputSupport.ts

@ -0,0 +1,101 @@
import type { ModelInputDef, SelectedModel } from "@/types";
export type ReferenceMediaType = "image" | "video" | "audio";
const VIDEO_INPUT_PARAMETER_NAMES = new Set([
"video",
"videos",
"video_url",
"video_urls",
"reference_video",
"reference_video_url",
"reference_video_urls",
]);
const AUDIO_INPUT_PARAMETER_NAMES = new Set([
"voice",
"voices",
"audio",
"audios",
"audio_url",
"audio_urls",
"reference_audio",
"reference_audio_url",
"reference_audio_urls",
]);
const IMAGE_INPUT_PARAMETER_NAMES = new Set([
"image",
"images",
"image_url",
"image_urls",
"first_frame",
"first_frame_url",
"last_frame",
"last_frame_url",
"tail_image_url",
"reference_image",
"reference_image_url",
"reference_image_urls",
]);
function hasSchemaInput(inputSchema: ModelInputDef[] | undefined, mediaType: ReferenceMediaType): boolean {
return Boolean(inputSchema?.some((input) => input.type === mediaType));
}
function hasNamedParameterInput(parameters: Record<string, unknown> | undefined, names: Set<string>): boolean {
if (!parameters) return false;
return Object.keys(parameters).some((key) => names.has(key));
}
function metadataBoolean(model: SelectedModel, key: string): boolean | null {
const value = model.metadata?.[key];
return typeof value === "boolean" ? value : null;
}
export function modelSupportsReferenceMedia(
model: SelectedModel,
mediaType: ReferenceMediaType,
inputSchema?: ModelInputDef[],
parameters?: Record<string, unknown>
): boolean {
if (hasSchemaInput(inputSchema, mediaType)) return true;
if (mediaType === "image") {
const supportImages = metadataBoolean(model, "isSupportImages");
if (supportImages !== null) return supportImages;
if (hasNamedParameterInput(parameters, IMAGE_INPUT_PARAMETER_NAMES)) return true;
return Boolean(model.capabilities?.some((capability) =>
capability === "image-to-image" ||
capability === "image-to-video" ||
capability === "image-to-3d"
));
}
if (mediaType === "video") {
const supportVideos = metadataBoolean(model, "isSupportVideos");
if (supportVideos !== null) return supportVideos;
if (hasNamedParameterInput(parameters, VIDEO_INPUT_PARAMETER_NAMES)) return true;
return Boolean(model.capabilities?.includes("video-to-video"));
}
const supportAudios = metadataBoolean(model, "isSupportAudios");
if (supportAudios !== null) return supportAudios;
if (hasNamedParameterInput(parameters, AUDIO_INPUT_PARAMETER_NAMES)) return true;
return Boolean(model.capabilities?.includes("audio-to-video"));
}
export function filterReferenceSubjectList(
referenceSubjectList: unknown,
model: SelectedModel,
inputSchema?: ModelInputDef[],
parameters?: Record<string, unknown>
): unknown {
if (!Array.isArray(referenceSubjectList)) return referenceSubjectList;
return referenceSubjectList.filter((item) => {
if (!item || typeof item !== "object" || Array.isArray(item)) return false;
const type = (item as Record<string, unknown>).type;
if (type !== "image" && type !== "video" && type !== "audio") return true;
return modelSupportsReferenceMedia(model, type, inputSchema, parameters);
});
}
Loading…
Cancel
Save