From 49398b24f8c4fce64b64fea94e882df7a09f09df Mon Sep 17 00:00:00 2001 From: TianYun Date: Mon, 1 Jun 2026 16:42:12 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E8=A7=A3=E9=99=A4=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E5=A4=A7=E5=B0=8F=E9=99=90=E5=88=B6=20=E6=9C=80=E5=A4=A7?= =?UTF-8?q?=E4=B8=8A=E4=BC=A022m=E7=9A=84=E5=9B=BE=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- next.config.ts | 1 + src/components/nodes/AnnotationNode.tsx | 8 ++++---- src/components/nodes/ImageInputNode.tsx | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/next.config.ts b/next.config.ts index 969b7994..2160d996 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,6 +4,7 @@ const nextConfig: NextConfig = { reactStrictMode: true, output: "standalone", experimental: { + proxyClientMaxBodySize: "32mb", serverActions: { bodySizeLimit: "100mb", // Increased for large media files }, diff --git a/src/components/nodes/AnnotationNode.tsx b/src/components/nodes/AnnotationNode.tsx index ecd25c98..ad8b9efa 100644 --- a/src/components/nodes/AnnotationNode.tsx +++ b/src/components/nodes/AnnotationNode.tsx @@ -32,10 +32,10 @@ export function AnnotationNode({ id, data, selected }: NodeProps 10 * 1024 * 1024) { - alert(t("upload.imageTooLarge")); - return; - } + // if (file.size > 10 * 1024 * 1024) { + // alert(t("upload.imageTooLarge")); + // return; + // } const reader = new FileReader(); reader.onload = (event) => { diff --git a/src/components/nodes/ImageInputNode.tsx b/src/components/nodes/ImageInputNode.tsx index d57a2a59..7e489f8b 100644 --- a/src/components/nodes/ImageInputNode.tsx +++ b/src/components/nodes/ImageInputNode.tsx @@ -135,10 +135,10 @@ export function ImageInputNode({ id, data, selected }: NodeProps 10 * 1024 * 1024) { - alert(t("upload.imageTooLarge")); - return; - } + // if (file.size > 10 * 1024 * 1024) { + // alert(t("upload.imageTooLarge")); + // return; + // } const reader = new FileReader(); reader.onload = (event) => { From febb5b9ffbd9d692f3d5c59e472ac06f1b7eb3f9 Mon Sep 17 00:00:00 2001 From: TianYun Date: Wed, 3 Jun 2026 16:04:55 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=8E=A5=E5=85=A5=20=E5=85=A8=E8=83=BD?= =?UTF-8?q?=E5=8F=82=E8=80=83=20=E9=A6=96=E5=B0=BE=E5=B8=A7=20=E5=9B=BE?= =?UTF-8?q?=E7=94=9F=E8=A7=86=E9=A2=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../providers/__tests__/popiserver.test.ts | 110 +++++ src/app/api/generate/providers/popiserver.ts | 98 +++- .../__tests__/GenerationComposer.test.tsx | 153 ++++++ .../composer/GenerationComposer.tsx | 461 ++++++++++++++---- src/i18n/index.tsx | 17 + 5 files changed, 751 insertions(+), 88 deletions(-) diff --git a/src/app/api/generate/providers/__tests__/popiserver.test.ts b/src/app/api/generate/providers/__tests__/popiserver.test.ts index 3615da63..ced0dc5a 100644 --- a/src/app/api/generate/providers/__tests__/popiserver.test.ts +++ b/src/app/api/generate/providers/__tests__/popiserver.test.ts @@ -202,6 +202,7 @@ describe("popiserver generation provider", () => { model: "", origin: "canvas", aiPlatform: "GATEWAY", + aiModelName: "Speech 2.8 HD", aiModelId: 30, aiModelCode: "speech-2.8-hd", aiModelCodeAlias: "speech-2.8-hd", @@ -358,6 +359,115 @@ describe("popiserver generation provider", () => { ]); }); + it("builds referenceSubjectList from final image and video URLs", async () => { + vi.mocked(global.fetch).mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ + status: "0000", + message: "ok", + data: { id: 1194 }, + }), + } as Response); + + const input = makeInput(); + input.model = { + ...input.model, + id: "29", + name: "seedance 2.0-fast", + metadata: { + type: 2, + subType: 203, + aiModelId: 29, + aiModelCode: "seedance 2.0", + aiModelCodeAlias: "doubao-seedance-2-0-fast-260128", + }, + }; + input.prompt = "@图1的人物参考@视频2的人物的动作跳舞"; + input.images = ["https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg"]; + input.dynamicInputs = { + videos: ["https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4"], + audios: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"], + }; + input.parameters = { + subType: 203, + aiModelName: "seedance 2.0-fast", + aiPlatform: "seedance 2.0", + model: "seedance 2.0", + ratio: "16:9", + resolution: "480P", + duration: 5, + batchSize: 1, + referenceSubjectList: [ + { + id: 1, + type: "image", + url: "stale-image-url", + name: "@图1", + }, + { + id: 2, + type: "video", + url: "stale-video-url", + name: "@视频2", + duration: 5.033, + }, + { + id: 3, + type: "audio", + url: "stale-audio-url", + name: "@音频3", + }, + ], + }; + + const result = await submitPopiTask(makeRequest(), "login-token", input); + + expect(result.taskId).toBe("1194"); + const [, init] = vi.mocked(global.fetch).mock.calls[0]; + const body = JSON.parse(String(init?.body)); + expect(body).toMatchObject({ + subType: 203, + aiModelName: "seedance 2.0-fast", + aiPlatform: "seedance 2.0", + aiModelId: 29, + model: "seedance 2.0", + aiModelCode: "seedance 2.0", + aiModelCodeAlias: "doubao-seedance-2-0-fast-260128", + type: 2, + ratio: "16:9", + resolution: "480P", + duration: 5, + batchSize: 1, + width: 1280, + height: 720, + chatPrompt: "@图1的人物参考@视频2的人物的动作跳舞", + images: ["https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg"], + videos: ["https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4"], + audios: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"], + referenceSubjectList: [ + { + id: 1, + type: "image", + url: "https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg", + name: "@图1", + }, + { + id: 2, + type: "video", + url: "https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4", + name: "@视频2", + duration: 5.033, + }, + { + id: 3, + type: "audio", + url: "https://statictest.popi.art/media/audio/2026/0603/5470.mp3", + name: "@音频3", + }, + ], + }); + }); + it("extracts completed image task output from task list", async () => { vi.mocked(global.fetch).mockResolvedValue({ ok: true, diff --git a/src/app/api/generate/providers/popiserver.ts b/src/app/api/generate/providers/popiserver.ts index 895bd8b3..69ddbfdc 100644 --- a/src/app/api/generate/providers/popiserver.ts +++ b/src/app/api/generate/providers/popiserver.ts @@ -102,6 +102,14 @@ type PopiTask = { resultList?: PopiTaskResult[] | null; }; +type ReferenceSubject = { + id: number; + type: "image" | "video" | "audio"; + url: string; + name: string; + duration?: number; +}; + export type PopiTaskPollResult = | { status: "processing" } | { status: "failed"; error: string } @@ -422,6 +430,87 @@ function buildPopiChatPrompt( ].join("\n"); } +function referenceSubjectMetadata(parameters: Record): ReferenceSubject[] { + const value = parameters.referenceSubjectList; + if (!Array.isArray(value)) return []; + + return value.flatMap((item) => { + if (!item || typeof item !== "object" || Array.isArray(item)) return []; + const subject = item as Record; + const type = + subject.type === "video" + ? "video" + : subject.type === "image" + ? "image" + : subject.type === "audio" + ? "audio" + : null; + if (!type) return []; + const id = asNumber(subject.id); + const name = asString(subject.name); + const url = asString(subject.url); + const duration = asNumber(subject.duration); + return [{ + id: id ?? 0, + type, + url: url ?? "", + name: name ?? "", + ...(duration !== null ? { duration } : {}), + }]; + }); +} + +function buildReferenceSubjectList( + images: string[], + videos: string[], + audios: string[], + parameters: Record +): ReferenceSubject[] { + const metadata = referenceSubjectMetadata(parameters); + const metadataByType = { + image: metadata.filter((item) => item.type === "image"), + video: metadata.filter((item) => item.type === "video"), + audio: metadata.filter((item) => item.type === "audio"), + }; + const subjects: ReferenceSubject[] = []; + + images.forEach((url, index) => { + const id = subjects.length + 1; + const meta = metadataByType.image[index]; + subjects.push({ + id, + type: "image", + url, + name: meta?.name || `@图${id}`, + }); + }); + + videos.forEach((url, index) => { + const id = subjects.length + 1; + const meta = metadataByType.video[index]; + subjects.push({ + id, + type: "video", + url, + name: meta?.name || `@视频${id}`, + ...(meta?.duration ? { duration: meta.duration } : {}), + }); + }); + + audios.forEach((url, index) => { + const id = subjects.length + 1; + const meta = metadataByType.audio[index]; + subjects.push({ + id, + type: "audio", + url, + name: meta?.name || `@音频${id}`, + }); + }); + + return subjects; +} + function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | null): Record { const parameters = input.parameters || {}; const type = asNumber(parameters.type) ?? inferTaskType(input); @@ -446,6 +535,9 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu ]; const videos = collectDynamicMedia(input.dynamicInputs, POPISERVER_VIDEO_DYNAMIC_KEYS); const audios = collectDynamicMedia(input.dynamicInputs, POPISERVER_AUDIO_DYNAMIC_KEYS); + const referenceSubjectList = type === 2 + ? buildReferenceSubjectList([...new Set(images)], [...new Set(videos)], [...new Set(audios)], parameters) + : []; const voiceId = firstDynamicString(input.dynamicInputs, POPISERVER_VOICE_DYNAMIC_KEYS) ?? asString(parameters.voiceId) ?? @@ -454,9 +546,10 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu const commonBody = { type, chatPrompt: buildPopiChatPrompt(input.prompt, parameters, aspectRatio, modelCode, modelAlias), - model: "", + model: asString(parameters.model) ?? "", origin: "canvas", - aiPlatform: "GATEWAY", + aiPlatform: asString(parameters.aiPlatform) ?? "GATEWAY", + aiModelName: asString(parameters.aiModelName) ?? input.model.name ?? modelDetail?.name ?? modelCode, aiModelId, aiModelCode: modelCode, aiModelCodeAlias: modelAlias, @@ -479,6 +572,7 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu ...(images.length > 0 ? { images: [...new Set(images)] } : {}), ...(videos.length > 0 ? { videos: [...new Set(videos)] } : {}), ...(audios.length > 0 ? { audios: [...new Set(audios)] } : {}), + ...(referenceSubjectList.length > 0 ? { referenceSubjectList } : {}), styleId: asNumber(parameters.styleId) ?? 0, width: asNumber(parameters.width) ?? dimensions.width, height: asNumber(parameters.height) ?? dimensions.height, diff --git a/src/components/__tests__/GenerationComposer.test.tsx b/src/components/__tests__/GenerationComposer.test.tsx index 44691cc7..d3a1b904 100644 --- a/src/components/__tests__/GenerationComposer.test.tsx +++ b/src/components/__tests__/GenerationComposer.test.tsx @@ -308,6 +308,17 @@ function textEdge(source: string, target: string): WorkflowEdge { } as WorkflowEdge; } +function imageEdge(source: string, target: string, index: number, targetHandle = `image-${index}`): WorkflowEdge { + return { + id: `edge-${source}-${target}-image-${index}`, + source, + sourceHandle: "image", + target, + targetHandle, + data: { createdAt: index }, + } as WorkflowEdge; +} + function videoEdge(source: string, target: string, index: number, targetHandle = `video-${index}`): WorkflowEdge { return { id: `edge-${source}-${target}-video-${index}`, @@ -319,6 +330,17 @@ function videoEdge(source: string, target: string, index: number, targetHandle = } as WorkflowEdge; } +function audioEdge(source: string, target: string, index: number, targetHandle = `audio-${index}`): WorkflowEdge { + return { + id: `edge-${source}-${target}-audio-${index}`, + source, + sourceHandle: "audio", + target, + targetHandle, + data: { createdAt: index }, + } as WorkflowEdge; +} + describe("GenerationComposer", () => { beforeEach(() => { vi.clearAllMocks(); @@ -1025,6 +1047,137 @@ describe("GenerationComposer", () => { expect(data.soundEnabled).toBe(false); }); + it("shows connected image, video, and audio materials in the @ menu", async () => { + useWorkflowStore.setState({ + nodes: [ + imageNode("img-ref", false, { outputImage: "https://static.popi.art/ref.jpg" }), + videoNode("vid-ref", false, { + outputVideo: "https://static.popi.art/ref.mp4", + durationSeconds: 5.033, + }), + audioNode("aud-ref", false, { + outputAudio: "https://static.popi.art/ref.mp3", + }), + videoNode("vid-1", true, { + inputPrompt: "make it", + }), + ], + edges: [ + imageEdge("img-ref", "vid-1", 0), + videoEdge("vid-ref", "vid-1", 1, "video"), + audioEdge("aud-ref", "vid-1", 2, "audio"), + ], + }); + render(); + + const prompt = screen.getByPlaceholderText(/Describe what you want to generate/) as HTMLTextAreaElement; + fireEvent.change(prompt, { target: { value: "make it @" } }); + + await waitFor(() => { + expect(screen.getByText("图1")).toBeInTheDocument(); + expect(screen.getByText("视频2")).toBeInTheDocument(); + expect(screen.getByText("音频3")).toBeInTheDocument(); + }); + fireEvent.click(screen.getByText("音频3")); + + expect(prompt).toHaveValue("make it @音频3 "); + fireEvent.click(screen.getByLabelText("Generate")); + + await waitFor(() => { + expect(useWorkflowStore.getState().regenerateNode).toHaveBeenCalledWith("vid-1"); + }); + const data = useWorkflowStore.getState().nodes[3].data as GenerateVideoNodeData; + expect(data.parameters?.referenceSubjectList).toEqual([ + { + id: 1, + type: "image", + url: "https://static.popi.art/ref.jpg", + name: "@图1", + }, + { + id: 2, + type: "video", + url: "https://static.popi.art/ref.mp4", + name: "@视频2", + duration: 5.033, + }, + { + id: 3, + type: "audio", + url: "https://static.popi.art/ref.mp3", + name: "@音频3", + }, + ]); + }); + + it("does not allow first/last frame mode unless exactly two images are connected", async () => { + useWorkflowStore.setState({ + nodes: [ + imageNode("img-a", false, { outputImage: "https://static.popi.art/a.jpg" }), + imageNode("img-b", false, { outputImage: "https://static.popi.art/b.jpg" }), + imageNode("img-c", false, { outputImage: "https://static.popi.art/c.jpg" }), + videoNode("vid-1", true, { + parameters: {}, + }), + ], + edges: [ + imageEdge("img-a", "vid-1", 0), + imageEdge("img-b", "vid-1", 1), + imageEdge("img-c", "vid-1", 2), + ], + }); + render(); + + fireEvent.click(screen.getByText("First/last frame")); + + await waitFor(() => { + const data = useWorkflowStore.getState().nodes[3].data as GenerateVideoNodeData; + expect(data.parameters?.subType).not.toBe(204); + }); + }); + + it("labels two first/last frame images without changing their reference order", async () => { + useWorkflowStore.setState({ + nodes: [ + imageNode("img-first", false, { outputImage: "https://static.popi.art/first.jpg" }), + imageNode("img-last", false, { outputImage: "https://static.popi.art/last.jpg" }), + videoNode("vid-1", true, { + inputPrompt: "animate", + parameters: { subType: 204 }, + }), + ], + edges: [ + imageEdge("img-first", "vid-1", 0), + imageEdge("img-last", "vid-1", 1), + ], + }); + render(); + + expect(screen.getByText("First")).toBeInTheDocument(); + expect(screen.getByText("Last")).toBeInTheDocument(); + + fireEvent.click(screen.getByLabelText("Generate")); + + await waitFor(() => { + expect(useWorkflowStore.getState().regenerateNode).toHaveBeenCalledWith("vid-1"); + }); + const data = useWorkflowStore.getState().nodes[2].data as GenerateVideoNodeData; + expect(data.parameters?.referenceSubjectList).toEqual([ + { + id: 1, + type: "image", + url: "https://static.popi.art/first.jpg", + name: "@图1", + }, + { + id: 2, + type: "image", + url: "https://static.popi.art/last.jpg", + name: "@图2", + }, + ]); + }); + it("writes selected video quick settings to the node immediately", () => { useWorkflowStore.setState({ nodes: [videoNode("vid-1", true, { diff --git a/src/components/composer/GenerationComposer.tsx b/src/components/composer/GenerationComposer.tsx index 81a3d661..0efcddf2 100644 --- a/src/components/composer/GenerationComposer.tsx +++ b/src/components/composer/GenerationComposer.tsx @@ -13,6 +13,7 @@ import { } from "react"; import { createPortal } from "react-dom"; import { useReactFlow, useViewport, ViewportPortal } from "@xyflow/react"; +import { message, Segmented } from "antd"; import { useProviderApiKeys, useWorkflowStore } from "@/store/workflowStore"; import { useModelStore } from "@/store/modelStore"; import { @@ -79,6 +80,17 @@ type EditableNodeType = "nanoBanana" | "generateVideo" | "generateAudio"; type DraftField = GenerationConfigField; type ComposerDraft = GenerationNodeConfig; type VideoStitchLoopCount = 1 | 2 | 3; +type VideoGenerationSubType = 202 | 203 | 204; +type ComposerReferenceMaterialType = "image" | "video" | "audio"; + +interface ComposerReferenceMaterial { + id: number; + type: ComposerReferenceMaterialType; + url: string; + name: string; + duration?: number; + removableIndex?: number; +} interface ComposerContext { mode: ComposerMode; @@ -127,6 +139,9 @@ const COMPOSER_COLLAPSED_WIDTH = 560; const COMPOSER_EXPANDED_HEIGHT_ESTIMATE = 300; const COMPOSER_COLLAPSED_HEIGHT_ESTIMATE = 80; const DEFAULT_IMAGE_COUNT_OPTIONS: ImageGenerationCount[] = [1, 2, 4]; +const VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE = 202; +const VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO = 203; +const VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME = 204; interface SchemaSelectOption { key: string; @@ -161,6 +176,23 @@ function getSchemaParameterDefault(schema: ModelParameter[] | undefined, name: s return param?.default ?? param?.options?.[0]?.value ?? param?.enum?.[0]; } +function isVideoGenerationSubType(value: unknown): value is VideoGenerationSubType { + return ( + value === VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE || + value === VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO || + value === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME + ); +} + +function videoGenerationSubTypeFromUnknown(value: unknown): VideoGenerationSubType | null { + const parsed = finiteIntegerFromUnknown(value); + return isVideoGenerationSubType(parsed) ? parsed : null; +} + +function canUseFirstLastFrameSubType(imageCount: number): boolean { + return imageCount === 2; +} + function getSchemaSelectOptions(param: ModelParameter): SchemaSelectOption[] { return param.options?.map((option, index) => ({ key: String(option.value ?? index), @@ -543,6 +575,74 @@ function getConnectedVideoDurationSeconds( return getNodeVideoDurationSeconds(nodes.find((node) => node.id === videoEdge.source)); } +function getConnectedVideoDurationList( + targetNodeId: string, + nodes: WorkflowNode[], + edges: Array<{ source: string; target: string; sourceHandle?: string | null; targetHandle?: string | null }> +): number[] { + return edges + .filter((edge) => + edge.target === targetNodeId && + (isVideoHandle(edge.targetHandle) || isVideoHandle(edge.sourceHandle)) + ) + .map((edge) => getNodeVideoDurationSeconds(nodes.find((node) => node.id === edge.source))); +} + +function buildComposerReferenceMaterials( + images: string[], + videos: string[], + audio: string[], + videoDurations: number[] = [], + removableImages = false +): ComposerReferenceMaterial[] { + const materials: ComposerReferenceMaterial[] = []; + + images.forEach((url, index) => { + const id = materials.length + 1; + materials.push({ + id, + type: "image", + url, + name: `@图${id}`, + ...(removableImages ? { removableIndex: index } : {}), + }); + }); + + videos.forEach((url, index) => { + const id = materials.length + 1; + const duration = videoDurations[index]; + materials.push({ + id, + type: "video", + url, + name: `@视频${id}`, + ...(duration && duration > 0 ? { duration } : {}), + }); + }); + + audio.forEach((url) => { + const id = materials.length + 1; + materials.push({ + id, + type: "audio", + url, + name: `@音频${id}`, + }); + }); + + return materials; +} + +function buildReferenceSubjectListParameter(materials: ComposerReferenceMaterial[]): Array> { + return materials.map((material) => ({ + id: material.id, + type: material.type, + url: material.url, + name: material.name, + ...(material.duration ? { duration: material.duration } : {}), + })); +} + function hasAnyCapability(model: SelectedModel, capabilities: string[]): boolean { return Boolean(model.capabilities?.some((capability) => capabilities.includes(capability))); } @@ -927,6 +1027,11 @@ export function GenerationComposer() { return getConnectedVideoDurationSeconds(context.node.id, nodes, edges); }, [context, edges, nodes]); + const connectedVideoDurations = useMemo(() => { + if (context.mode !== "node-edit" || !context.node) return []; + return getConnectedVideoDurationList(context.node.id, nodes, edges); + }, [context, edges, nodes]); + const estimatedReferenceImageCount = context.mode === "node-edit" && (connectedInputs?.images.length ?? 0) > 0 ? connectedInputs?.images.length ?? 0 : draft.inputImages.length; @@ -1265,12 +1370,16 @@ export function GenerationComposer() { const hasReferenceImageForSubmit = (context.mode === "empty-create" || context.mode === "node-edit") && ((connectedInputs?.images.length ?? 0) > 0 || draft.inputImages.length > 0); + const hasReferenceVideoForSubmit = + context.mode === "node-edit" && (connectedInputs?.videos.length ?? 0) > 0; + const hasReferenceAudioForSubmit = + context.mode === "node-edit" && (connectedInputs?.audio.length ?? 0) > 0; const submitNodeType = context.mode === "empty-create" ? inferNodeTypeFromModel(draft.selectedModel) : context.nodeType; const hasRequiredPromptOrReference = submitNodeType === "generateAudio" ? promptForSubmit.length > 0 - : (promptForSubmit.length > 0 || hasReferenceImageForSubmit); + : (promptForSubmit.length > 0 || hasReferenceImageForSubmit || hasReferenceVideoForSubmit || hasReferenceAudioForSubmit); const canCreateFromSelectedImageInput = context.mode === "unsupported-node" && Boolean(selectedImageInputImage) && @@ -1298,9 +1407,21 @@ export function GenerationComposer() { canCreateFromSelectedImageInput; const connectedReferenceImages = context.mode === "node-edit" ? connectedInputs?.images ?? [] : []; + const connectedReferenceVideos = context.mode === "node-edit" ? connectedInputs?.videos ?? [] : []; + const connectedReferenceAudio = context.mode === "node-edit" ? connectedInputs?.audio ?? [] : []; const referenceImages = connectedReferenceImages.length > 0 ? connectedReferenceImages : draft.inputImages; + const canUseFirstLastFrame = canUseFirstLastFrameSubType(referenceImages.length); + const referenceMaterials = useMemo(() => buildComposerReferenceMaterials( + referenceImages, + connectedReferenceVideos, + connectedReferenceAudio, + connectedVideoDurations, + connectedReferenceImages.length === 0 + ), [connectedReferenceAudio, connectedReferenceImages.length, connectedReferenceVideos, connectedVideoDurations, referenceImages]); const previewReferenceImage = - referencePreviewIndex === null ? null : referenceImages[referencePreviewIndex] ?? null; + referencePreviewIndex === null ? null : referenceMaterials[referencePreviewIndex]?.type === "image" + ? referenceMaterials[referencePreviewIndex]?.url ?? null + : null; const canEditReferenceImages = (context.mode === "empty-create" || context.mode === "node-edit") && connectedReferenceImages.length === 0; @@ -1501,6 +1622,37 @@ export function GenerationComposer() { [updateNodeData] ); + const isVideoSubTypeSelectorVisible = context.mode === "node-edit" && context.nodeType === "generateVideo"; + const currentVideoSubType = videoGenerationSubTypeFromUnknown(draft.parameters?.subType); + const selectedVideoSubType = currentVideoSubType ?? VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE; + const isFirstLastFrameSelected = + selectedVideoSubType === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME && canUseFirstLastFrame; + const videoSubTypeOptions = useMemo( + () => [ + { + label: t("composer.videoSubTypeMultiReference"), + value: VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE, + }, + { + label: t("composer.videoSubTypeFirstLastFrame"), + value: VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME, + disabled: !canUseFirstLastFrame, + }, + { + label: t("composer.videoSubTypeTextToVideo"), + value: VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO, + }, + ], + [canUseFirstLastFrame, t] + ); + + useEffect(() => { + if (!isVideoSubTypeSelectorVisible) return; + if (currentVideoSubType !== VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME) return; + if (canUseFirstLastFrame) return; + updateSchemaDraftParameter("subType", VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE); + }, [canUseFirstLastFrame, currentVideoSubType, isVideoSubTypeSelectorVisible, updateSchemaDraftParameter]); + useEffect(() => { if ( isProcessNode || @@ -1706,10 +1858,10 @@ export function GenerationComposer() { ); useEffect(() => { - if (referencePreviewIndex !== null && referencePreviewIndex >= referenceImages.length) { + if (referencePreviewIndex !== null && referencePreviewIndex >= referenceMaterials.length) { setReferencePreviewIndex(null); } - }, [referenceImages.length, referencePreviewIndex]); + }, [referenceMaterials.length, referencePreviewIndex]); useEffect(() => { if (!previewReferenceImage) return; @@ -1741,6 +1893,43 @@ export function GenerationComposer() { if (!canSubmit) return; const fallbackPrompt = buildReferenceOnlyPrompt(); const submitPrompt = promptForSubmit || fallbackPrompt; + const activeDraft = draftRef.current; + const submitVideoNodeType = context.mode === "empty-create" + ? inferNodeTypeFromModel(activeDraft.selectedModel) + : context.nodeType; + const explicitSubmitVideoSubType = videoGenerationSubTypeFromUnknown(activeDraft.parameters?.subType); + const submitVideoSubType = explicitSubmitVideoSubType ?? VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE; + const hasSubmitReferenceImage = + (connectedInputs?.images.length ?? 0) > 0 || activeDraft.inputImages.length > 0; + const submitReferenceImageCount = connectedInputs?.images.length && connectedInputs.images.length > 0 + ? connectedInputs.images.length + : activeDraft.inputImages.length; + const referenceSubjectList = buildReferenceSubjectListParameter(referenceMaterials); + const shouldWriteVideoSubType = context.mode === "empty-create" || explicitSubmitVideoSubType !== null; + const videoParameters = submitVideoNodeType === "generateVideo" + ? { + ...activeDraft.parameters, + ...(shouldWriteVideoSubType ? { subType: submitVideoSubType } : {}), + ...(referenceSubjectList.length > 0 ? { referenceSubjectList } : {}), + } + : activeDraft.parameters; + if ( + submitVideoNodeType === "generateVideo" && + explicitSubmitVideoSubType !== null && + explicitSubmitVideoSubType !== VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO && + !hasSubmitReferenceImage + ) { + message.warning(t("composer.videoSubTypeImageRequired")); + return; + } + if ( + submitVideoNodeType === "generateVideo" && + explicitSubmitVideoSubType === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME && + !canUseFirstLastFrameSubType(submitReferenceImageCount) + ) { + message.warning(t("composer.videoSubTypeFirstLastFrameImageCountRequired")); + return; + } if (context.mode === "unsupported-node" && context.node?.type === "imageInput") { const imageData = context.node.data as ImageInputNodeData; @@ -1767,14 +1956,15 @@ export function GenerationComposer() { } if (context.mode === "empty-create") { - const nodeType = inferNodeTypeFromModel(draftRef.current.selectedModel); + const nodeType = submitVideoNodeType; if (!nodeType) return; const nextNodeId = addNode( nodeType, buildTargetPosition(), buildInitialDataForNode(nodeType, { - ...draftRef.current, + ...activeDraft, + parameters: nodeType === "generateVideo" ? videoParameters : activeDraft.parameters, prompt: submitPrompt, }) ); @@ -1786,6 +1976,18 @@ export function GenerationComposer() { } if (context.mode === "node-edit" && context.node) { + if (submitVideoNodeType === "generateVideo") { + const nextDraft = { + ...draftRef.current, + parameters: videoParameters, + }; + const nextDirtyFields = new Set(dirtyFieldsRef.current); + nextDirtyFields.add("parameters"); + draftRef.current = nextDraft; + dirtyFieldsRef.current = nextDirtyFields; + setDraft(nextDraft); + setDirtyFields(nextDirtyFields); + } if (!promptForSubmit && !promptReadOnly) { const nextDraft = { ...draftRef.current, prompt: submitPrompt }; const nextDirtyFields = new Set(dirtyFieldsRef.current); @@ -1811,12 +2013,15 @@ export function GenerationComposer() { buildTargetPosition, canSubmit, clearDirtyFields, + connectedInputs, context, flushCurrentDraft, promptForSubmit, promptReadOnly, + referenceMaterials, regenerateNode, selectSingleNode, + t, ]); const insertCommand = useCallback((value: string) => { @@ -1995,13 +2200,31 @@ export function GenerationComposer() { ) : ( <> -
-
- {t("composer.console")} - {modeLabel} -
+
+ {isVideoSubTypeSelectorVisible && ( + { + const nextSubType = videoGenerationSubTypeFromUnknown(value); + if (!nextSubType) return; + if (nextSubType === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME && !canUseFirstLastFrame) { + message.warning(t("composer.videoSubTypeFirstLastFrameImageCountRequired")); + return; + } + updateSchemaDraftParameter("subType", nextSubType); + }} + className="nodrag nopan nowheel max-w-full text-xs" + /> + )} - {isProcessNode ? ( +
+
+ {t("composer.console")} + {modeLabel} +
+ + {isProcessNode ? ( isVideoStitchNode ? (
@@ -2024,82 +2247,127 @@ export function GenerationComposer() {
) ) : ( - <> - - - {referenceImages.length > 0 && ( -
- {referenceImages.map((referenceImage, imageIndex) => ( -
- + + {referenceMaterials.length > 0 && ( +
+ {referenceMaterials.map((material, materialIndex) => ( +
- - - - - - - - - - - - {imageIndex + 1} - - {canEditReferenceImages && ( - )} -
- ))} -
- )} - + + {isFirstLastFrameSelected && material.type === "image" + ? material.id === 1 + ? t("composer.firstFrame") + : t("composer.lastFrame") + : material.id} + + {canEditReferenceImages && material.type === "image" && material.removableIndex !== undefined && ( + + )} +
+ ))} +
+ )} +
)} -
- - - - - - +
+ + + + + + +
@@ -2129,17 +2397,38 @@ export function GenerationComposer() {
) : (
- {referenceImages.length > 0 ? ( - referenceImages.map((referenceImage, imageIndex) => { - const referenceLabel = `图${imageIndex + 1}`; + {referenceMaterials.length > 0 ? ( + referenceMaterials.map((material) => { + const referenceLabel = material.name.replace(/^@/, ""); return ( ); diff --git a/src/i18n/index.tsx b/src/i18n/index.tsx index d293b5b8..d637ea80 100644 --- a/src/i18n/index.tsx +++ b/src/i18n/index.tsx @@ -601,6 +601,13 @@ const en = { "composer.reference": "Reference", "composer.referenceImage": "Reference image {index}", "composer.removeReferenceImage": "Remove reference image {index}", + "composer.videoSubTypeMultiReference": "Image to video", + "composer.videoSubTypeTextToVideo": "All-purpose reference", + "composer.videoSubTypeFirstLastFrame": "First/last frame", + "composer.videoSubTypeImageRequired": "Image to video or first/last frame requires at least one reference image.", + "composer.videoSubTypeFirstLastFrameImageCountRequired": "First/last frame requires exactly two reference images.", + "composer.firstFrame": "First", + "composer.lastFrame": "Last", "composer.commands": "Commands", "composer.referenceMaterials": "Reference assets", "composer.commandCharacter": "Character image", @@ -1587,6 +1594,16 @@ const zhCN: Dictionary = { "cost.viewDetails": "查看费用详情", }; +Object.assign(zhCN, { + "composer.videoSubTypeFirstLastFrameImageCountRequired": "\u9996\u5c3e\u5e27\u9700\u8981\u6070\u597d\u4e24\u5f20\u53c2\u8003\u56fe\u7247\u3002", + "composer.firstFrame": "\u9996\u5e27", + "composer.lastFrame": "\u5c3e\u5e27", + "composer.videoSubTypeMultiReference": "图生视频", + "composer.videoSubTypeTextToVideo": "全能参考", + "composer.videoSubTypeFirstLastFrame": "首尾帧", + "composer.videoSubTypeImageRequired": "图生视频或首尾帧至少需要一张参考图片。", +}); + const zhTW: Dictionary = { ...zhCN, ...zhTWAuth,