diff --git a/src/app/api/generate/providers/__tests__/popiserver.test.ts b/src/app/api/generate/providers/__tests__/popiserver.test.ts index 3615da63..ced0dc5a 100644 --- a/src/app/api/generate/providers/__tests__/popiserver.test.ts +++ b/src/app/api/generate/providers/__tests__/popiserver.test.ts @@ -202,6 +202,7 @@ describe("popiserver generation provider", () => { model: "", origin: "canvas", aiPlatform: "GATEWAY", + aiModelName: "Speech 2.8 HD", aiModelId: 30, aiModelCode: "speech-2.8-hd", aiModelCodeAlias: "speech-2.8-hd", @@ -358,6 +359,115 @@ describe("popiserver generation provider", () => { ]); }); + it("builds referenceSubjectList from final image and video URLs", async () => { + vi.mocked(global.fetch).mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ + status: "0000", + message: "ok", + data: { id: 1194 }, + }), + } as Response); + + const input = makeInput(); + input.model = { + ...input.model, + id: "29", + name: "seedance 2.0-fast", + metadata: { + type: 2, + subType: 203, + aiModelId: 29, + aiModelCode: "seedance 2.0", + aiModelCodeAlias: "doubao-seedance-2-0-fast-260128", + }, + }; + input.prompt = "@图1的人物参考@视频2的人物的动作跳舞"; + input.images = ["https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg"]; + input.dynamicInputs = { + videos: ["https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4"], + audios: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"], + }; + input.parameters = { + subType: 203, + aiModelName: "seedance 2.0-fast", + aiPlatform: "seedance 2.0", + model: "seedance 2.0", + ratio: "16:9", + resolution: "480P", + duration: 5, + batchSize: 1, + referenceSubjectList: [ + { + id: 1, + type: "image", + url: "stale-image-url", + name: "@图1", + }, + { + id: 2, + type: "video", + url: "stale-video-url", + name: "@视频2", + duration: 5.033, + }, + { + id: 3, + type: "audio", + url: "stale-audio-url", + name: "@音频3", + }, + ], + }; + + const result = await submitPopiTask(makeRequest(), "login-token", input); + + expect(result.taskId).toBe("1194"); + const [, init] = vi.mocked(global.fetch).mock.calls[0]; + const body = JSON.parse(String(init?.body)); + expect(body).toMatchObject({ + subType: 203, + aiModelName: "seedance 2.0-fast", + aiPlatform: "seedance 2.0", + aiModelId: 29, + model: "seedance 2.0", + aiModelCode: "seedance 2.0", + aiModelCodeAlias: "doubao-seedance-2-0-fast-260128", + type: 2, + ratio: "16:9", + resolution: "480P", + duration: 5, + batchSize: 1, + width: 1280, + height: 720, + chatPrompt: "@图1的人物参考@视频2的人物的动作跳舞", + images: ["https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg"], + videos: ["https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4"], + audios: ["https://statictest.popi.art/media/audio/2026/0603/5470.mp3"], + referenceSubjectList: [ + { + id: 1, + type: "image", + url: "https://statictest.popi.art/media/ani/character/2026/0512/4540.jpg", + name: "@图1", + }, + { + id: 2, + type: "video", + url: "https://statictest.popi.art/media/aiGen/2026/0603/5470.mp4", + name: "@视频2", + duration: 5.033, + }, + { + id: 3, + type: "audio", + url: "https://statictest.popi.art/media/audio/2026/0603/5470.mp3", + name: "@音频3", + }, + ], + }); + }); + it("extracts completed image task output from task list", async () => { vi.mocked(global.fetch).mockResolvedValue({ ok: true, diff --git a/src/app/api/generate/providers/popiserver.ts b/src/app/api/generate/providers/popiserver.ts index 06dd9aed..72feca36 100644 --- a/src/app/api/generate/providers/popiserver.ts +++ b/src/app/api/generate/providers/popiserver.ts @@ -103,6 +103,14 @@ type PopiTask = { resultList?: PopiTaskResult[] | null; }; +type ReferenceSubject = { + id: number; + type: "image" | "video" | "audio"; + url: string; + name: string; + duration?: number; +}; + export type PopiTaskPollResult = | { status: "processing" } | { status: "failed"; error: string } @@ -423,6 +431,87 @@ function buildPopiChatPrompt( ].join("\n"); } +function referenceSubjectMetadata(parameters: Record): ReferenceSubject[] { + const value = parameters.referenceSubjectList; + if (!Array.isArray(value)) return []; + + return value.flatMap((item) => { + if (!item || typeof item !== "object" || Array.isArray(item)) return []; + const subject = item as Record; + const type = + subject.type === "video" + ? "video" + : subject.type === "image" + ? "image" + : subject.type === "audio" + ? "audio" + : null; + if (!type) return []; + const id = asNumber(subject.id); + const name = asString(subject.name); + const url = asString(subject.url); + const duration = asNumber(subject.duration); + return [{ + id: id ?? 0, + type, + url: url ?? "", + name: name ?? "", + ...(duration !== null ? { duration } : {}), + }]; + }); +} + +function buildReferenceSubjectList( + images: string[], + videos: string[], + audios: string[], + parameters: Record +): ReferenceSubject[] { + const metadata = referenceSubjectMetadata(parameters); + const metadataByType = { + image: metadata.filter((item) => item.type === "image"), + video: metadata.filter((item) => item.type === "video"), + audio: metadata.filter((item) => item.type === "audio"), + }; + const subjects: ReferenceSubject[] = []; + + images.forEach((url, index) => { + const id = subjects.length + 1; + const meta = metadataByType.image[index]; + subjects.push({ + id, + type: "image", + url, + name: meta?.name || `@图${id}`, + }); + }); + + videos.forEach((url, index) => { + const id = subjects.length + 1; + const meta = metadataByType.video[index]; + subjects.push({ + id, + type: "video", + url, + name: meta?.name || `@视频${id}`, + ...(meta?.duration ? { duration: meta.duration } : {}), + }); + }); + + audios.forEach((url, index) => { + const id = subjects.length + 1; + const meta = metadataByType.audio[index]; + subjects.push({ + id, + type: "audio", + url, + name: meta?.name || `@音频${id}`, + }); + }); + + return subjects; +} + function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | null): Record { const parameters = input.parameters || {}; const type = asNumber(parameters.type) ?? inferTaskType(input); @@ -447,6 +536,9 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu ]; const videos = collectDynamicMedia(input.dynamicInputs, POPISERVER_VIDEO_DYNAMIC_KEYS); const audios = collectDynamicMedia(input.dynamicInputs, POPISERVER_AUDIO_DYNAMIC_KEYS); + const referenceSubjectList = type === 2 + ? buildReferenceSubjectList([...new Set(images)], [...new Set(videos)], [...new Set(audios)], parameters) + : []; const voiceId = firstDynamicString(input.dynamicInputs, POPISERVER_VOICE_DYNAMIC_KEYS) ?? asString(parameters.voiceId) ?? @@ -455,9 +547,10 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu const commonBody = { type, chatPrompt: buildPopiChatPrompt(input.prompt, parameters, aspectRatio, modelCode, modelAlias), - model: "", + model: asString(parameters.model) ?? "", origin: "canvas", - aiPlatform: "GATEWAY", + aiPlatform: asString(parameters.aiPlatform) ?? "GATEWAY", + aiModelName: asString(parameters.aiModelName) ?? input.model.name ?? modelDetail?.name ?? modelCode, aiModelId, aiModelCode: modelCode, aiModelCodeAlias: modelAlias, @@ -480,6 +573,7 @@ function buildPopiTaskBody(input: GenerationInput, modelDetail: PopiAIModel | nu ...(images.length > 0 ? { images: [...new Set(images)] } : {}), ...(videos.length > 0 ? { videos: [...new Set(videos)] } : {}), ...(audios.length > 0 ? { audios: [...new Set(audios)] } : {}), + ...(referenceSubjectList.length > 0 ? { referenceSubjectList } : {}), styleId: asNumber(parameters.styleId) ?? 0, width: asNumber(parameters.width) ?? dimensions.width, height: asNumber(parameters.height) ?? dimensions.height, diff --git a/src/components/__tests__/GenerationComposer.test.tsx b/src/components/__tests__/GenerationComposer.test.tsx index 089fffc2..6b21f888 100644 --- a/src/components/__tests__/GenerationComposer.test.tsx +++ b/src/components/__tests__/GenerationComposer.test.tsx @@ -308,6 +308,17 @@ function textEdge(source: string, target: string): WorkflowEdge { } as WorkflowEdge; } +function imageEdge(source: string, target: string, index: number, targetHandle = `image-${index}`): WorkflowEdge { + return { + id: `edge-${source}-${target}-image-${index}`, + source, + sourceHandle: "image", + target, + targetHandle, + data: { createdAt: index }, + } as WorkflowEdge; +} + function videoEdge(source: string, target: string, index: number, targetHandle = `video-${index}`): WorkflowEdge { return { id: `edge-${source}-${target}-video-${index}`, @@ -319,6 +330,17 @@ function videoEdge(source: string, target: string, index: number, targetHandle = } as WorkflowEdge; } +function audioEdge(source: string, target: string, index: number, targetHandle = `audio-${index}`): WorkflowEdge { + return { + id: `edge-${source}-${target}-audio-${index}`, + source, + sourceHandle: "audio", + target, + targetHandle, + data: { createdAt: index }, + } as WorkflowEdge; +} + describe("GenerationComposer", () => { beforeEach(() => { vi.clearAllMocks(); @@ -1025,6 +1047,137 @@ describe("GenerationComposer", () => { expect(data.soundEnabled).toBe(false); }); + it("shows connected image, video, and audio materials in the @ menu", async () => { + useWorkflowStore.setState({ + nodes: [ + imageNode("img-ref", false, { outputImage: "https://static.popi.art/ref.jpg" }), + videoNode("vid-ref", false, { + outputVideo: "https://static.popi.art/ref.mp4", + durationSeconds: 5.033, + }), + audioNode("aud-ref", false, { + outputAudio: "https://static.popi.art/ref.mp3", + }), + videoNode("vid-1", true, { + inputPrompt: "make it", + }), + ], + edges: [ + imageEdge("img-ref", "vid-1", 0), + videoEdge("vid-ref", "vid-1", 1, "video"), + audioEdge("aud-ref", "vid-1", 2, "audio"), + ], + }); + render(); + + const prompt = screen.getByPlaceholderText(/Describe what you want to generate/) as HTMLTextAreaElement; + fireEvent.change(prompt, { target: { value: "make it @" } }); + + await waitFor(() => { + expect(screen.getByText("图1")).toBeInTheDocument(); + expect(screen.getByText("视频2")).toBeInTheDocument(); + expect(screen.getByText("音频3")).toBeInTheDocument(); + }); + fireEvent.click(screen.getByText("音频3")); + + expect(prompt).toHaveValue("make it @音频3 "); + fireEvent.click(screen.getByLabelText("Generate")); + + await waitFor(() => { + expect(useWorkflowStore.getState().regenerateNode).toHaveBeenCalledWith("vid-1"); + }); + const data = useWorkflowStore.getState().nodes[3].data as GenerateVideoNodeData; + expect(data.parameters?.referenceSubjectList).toEqual([ + { + id: 1, + type: "image", + url: "https://static.popi.art/ref.jpg", + name: "@图1", + }, + { + id: 2, + type: "video", + url: "https://static.popi.art/ref.mp4", + name: "@视频2", + duration: 5.033, + }, + { + id: 3, + type: "audio", + url: "https://static.popi.art/ref.mp3", + name: "@音频3", + }, + ]); + }); + + it("does not allow first/last frame mode unless exactly two images are connected", async () => { + useWorkflowStore.setState({ + nodes: [ + imageNode("img-a", false, { outputImage: "https://static.popi.art/a.jpg" }), + imageNode("img-b", false, { outputImage: "https://static.popi.art/b.jpg" }), + imageNode("img-c", false, { outputImage: "https://static.popi.art/c.jpg" }), + videoNode("vid-1", true, { + parameters: {}, + }), + ], + edges: [ + imageEdge("img-a", "vid-1", 0), + imageEdge("img-b", "vid-1", 1), + imageEdge("img-c", "vid-1", 2), + ], + }); + render(); + + fireEvent.click(screen.getByText("First/last frame")); + + await waitFor(() => { + const data = useWorkflowStore.getState().nodes[3].data as GenerateVideoNodeData; + expect(data.parameters?.subType).not.toBe(204); + }); + }); + + it("labels two first/last frame images without changing their reference order", async () => { + useWorkflowStore.setState({ + nodes: [ + imageNode("img-first", false, { outputImage: "https://static.popi.art/first.jpg" }), + imageNode("img-last", false, { outputImage: "https://static.popi.art/last.jpg" }), + videoNode("vid-1", true, { + inputPrompt: "animate", + parameters: { subType: 204 }, + }), + ], + edges: [ + imageEdge("img-first", "vid-1", 0), + imageEdge("img-last", "vid-1", 1), + ], + }); + render(); + + expect(screen.getByText("First")).toBeInTheDocument(); + expect(screen.getByText("Last")).toBeInTheDocument(); + + fireEvent.click(screen.getByLabelText("Generate")); + + await waitFor(() => { + expect(useWorkflowStore.getState().regenerateNode).toHaveBeenCalledWith("vid-1"); + }); + const data = useWorkflowStore.getState().nodes[2].data as GenerateVideoNodeData; + expect(data.parameters?.referenceSubjectList).toEqual([ + { + id: 1, + type: "image", + url: "https://static.popi.art/first.jpg", + name: "@图1", + }, + { + id: 2, + type: "image", + url: "https://static.popi.art/last.jpg", + name: "@图2", + }, + ]); + }); + it("writes selected video quick settings to the node immediately", () => { useWorkflowStore.setState({ nodes: [videoNode("vid-1", true, { diff --git a/src/components/composer/GenerationComposer.tsx b/src/components/composer/GenerationComposer.tsx index 47ee7627..7b76be9f 100644 --- a/src/components/composer/GenerationComposer.tsx +++ b/src/components/composer/GenerationComposer.tsx @@ -13,6 +13,7 @@ import { } from "react"; import { createPortal } from "react-dom"; import { useReactFlow, useViewport, ViewportPortal } from "@xyflow/react"; +import { message, Segmented } from "antd"; import { useProviderApiKeys, useWorkflowStore } from "@/store/workflowStore"; import { useModelStore } from "@/store/modelStore"; import { @@ -84,6 +85,17 @@ type EditableNodeType = "smartImage" | "smartVideo" | "nanoBanana" | "generateVi type DraftField = GenerationConfigField; type ComposerDraft = GenerationNodeConfig; type VideoStitchLoopCount = 1 | 2 | 3; +type VideoGenerationSubType = 202 | 203 | 204; +type ComposerReferenceMaterialType = "image" | "video" | "audio"; + +interface ComposerReferenceMaterial { + id: number; + type: ComposerReferenceMaterialType; + url: string; + name: string; + duration?: number; + removableIndex?: number; +} interface ComposerContext { mode: ComposerMode; @@ -130,6 +142,9 @@ const COMPOSER_VIEWPORT_MARGIN = 16; const COMPOSER_EXPANDED_WIDTH = 640; const COMPOSER_COLLAPSED_WIDTH = 560; const DEFAULT_IMAGE_COUNT_OPTIONS: ImageGenerationCount[] = [1, 2, 4]; +const VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE = 202; +const VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO = 203; +const VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME = 204; interface SchemaSelectOption { key: string; @@ -164,6 +179,47 @@ function getSchemaParameterDefault(schema: ModelParameter[] | undefined, name: s return param?.default ?? param?.options?.[0]?.value ?? param?.enum?.[0]; } +function getDraftFieldForSchemaParameter(name: string): DraftField | null { + if (isResolutionParameterName(name)) return "resolution"; + if (isAspectRatioParameterName(name)) return "aspectRatio"; + if (isDurationParameterName(name)) return "durationSeconds"; + return null; +} + +function hasPersistedSchemaParameterValue( + data: NanoBananaNodeData | GenerateVideoNodeData | GenerateAudioNodeData | undefined, + name: string +): boolean { + if (!data) return false; + if (Object.prototype.hasOwnProperty.call(data.parameters ?? {}, name)) return true; + if (Object.prototype.hasOwnProperty.call(data.config?.parameters ?? {}, name)) return true; + + const field = getDraftFieldForSchemaParameter(name); + return Boolean( + field && + Object.prototype.hasOwnProperty.call(data, field) && + data.config && + Object.prototype.hasOwnProperty.call(data.config, field) + ); +} + +function isVideoGenerationSubType(value: unknown): value is VideoGenerationSubType { + return ( + value === VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE || + value === VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO || + value === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME + ); +} + +function videoGenerationSubTypeFromUnknown(value: unknown): VideoGenerationSubType | null { + const parsed = finiteIntegerFromUnknown(value); + return isVideoGenerationSubType(parsed) ? parsed : null; +} + +function canUseFirstLastFrameSubType(imageCount: number): boolean { + return imageCount === 2; +} + function getSchemaSelectOptions(param: ModelParameter): SchemaSelectOption[] { return param.options?.map((option, index) => ({ key: String(option.value ?? index), @@ -546,6 +602,74 @@ function getConnectedVideoDurationSeconds( return getNodeVideoDurationSeconds(nodes.find((node) => node.id === videoEdge.source)); } +function getConnectedVideoDurationList( + targetNodeId: string, + nodes: WorkflowNode[], + edges: Array<{ source: string; target: string; sourceHandle?: string | null; targetHandle?: string | null }> +): number[] { + return edges + .filter((edge) => + edge.target === targetNodeId && + (isVideoHandle(edge.targetHandle) || isVideoHandle(edge.sourceHandle)) + ) + .map((edge) => getNodeVideoDurationSeconds(nodes.find((node) => node.id === edge.source))); +} + +function buildComposerReferenceMaterials( + images: string[], + videos: string[], + audio: string[], + videoDurations: number[] = [], + removableImages = false +): ComposerReferenceMaterial[] { + const materials: ComposerReferenceMaterial[] = []; + + images.forEach((url, index) => { + const id = materials.length + 1; + materials.push({ + id, + type: "image", + url, + name: `@图${id}`, + ...(removableImages ? { removableIndex: index } : {}), + }); + }); + + videos.forEach((url, index) => { + const id = materials.length + 1; + const duration = videoDurations[index]; + materials.push({ + id, + type: "video", + url, + name: `@视频${id}`, + ...(duration && duration > 0 ? { duration } : {}), + }); + }); + + audio.forEach((url) => { + const id = materials.length + 1; + materials.push({ + id, + type: "audio", + url, + name: `@音频${id}`, + }); + }); + + return materials; +} + +function buildReferenceSubjectListParameter(materials: ComposerReferenceMaterial[]): Array> { + return materials.map((material) => ({ + id: material.id, + type: material.type, + url: material.url, + name: material.name, + ...(material.duration ? { duration: material.duration } : {}), + })); +} + function hasAnyCapability(model: SelectedModel, capabilities: string[]): boolean { return Boolean(model.capabilities?.some((capability) => capabilities.includes(capability))); } @@ -944,6 +1068,11 @@ export function GenerationComposer() { return getConnectedVideoDurationSeconds(context.node.id, nodes, edges); }, [context, edges, nodes]); + const connectedVideoDurations = useMemo(() => { + if (context.mode !== "node-edit" || !context.node) return []; + return getConnectedVideoDurationList(context.node.id, nodes, edges); + }, [context, edges, nodes]); + const estimatedReferenceImageCount = context.mode === "node-edit" && (connectedInputs?.images.length ?? 0) > 0 ? connectedInputs?.images.length ?? 0 : draft.inputImages.length; @@ -1282,12 +1411,16 @@ export function GenerationComposer() { const hasReferenceImageForSubmit = (context.mode === "empty-create" || context.mode === "node-edit") && ((connectedInputs?.images.length ?? 0) > 0 || draft.inputImages.length > 0); + const hasReferenceVideoForSubmit = + context.mode === "node-edit" && (connectedInputs?.videos.length ?? 0) > 0; + const hasReferenceAudioForSubmit = + context.mode === "node-edit" && (connectedInputs?.audio.length ?? 0) > 0; const submitNodeType = context.mode === "empty-create" ? inferNodeTypeFromModel(draft.selectedModel) : context.nodeType; const hasRequiredPromptOrReference = submitNodeType === "generateAudio" ? promptForSubmit.length > 0 - : (promptForSubmit.length > 0 || hasReferenceImageForSubmit); + : (promptForSubmit.length > 0 || hasReferenceImageForSubmit || hasReferenceVideoForSubmit || hasReferenceAudioForSubmit); const canCreateFromSelectedImageInput = context.mode === "unsupported-node" && Boolean(selectedImageInputImage) && @@ -1315,9 +1448,21 @@ export function GenerationComposer() { canCreateFromSelectedImageInput; const connectedReferenceImages = context.mode === "node-edit" ? connectedInputs?.images ?? [] : []; + const connectedReferenceVideos = context.mode === "node-edit" ? connectedInputs?.videos ?? [] : []; + const connectedReferenceAudio = context.mode === "node-edit" ? connectedInputs?.audio ?? [] : []; const referenceImages = connectedReferenceImages.length > 0 ? connectedReferenceImages : draft.inputImages; + const canUseFirstLastFrame = canUseFirstLastFrameSubType(referenceImages.length); + const referenceMaterials = useMemo(() => buildComposerReferenceMaterials( + referenceImages, + connectedReferenceVideos, + connectedReferenceAudio, + connectedVideoDurations, + connectedReferenceImages.length === 0 + ), [connectedReferenceAudio, connectedReferenceImages.length, connectedReferenceVideos, connectedVideoDurations, referenceImages]); const previewReferenceImage = - referencePreviewIndex === null ? null : referenceImages[referencePreviewIndex] ?? null; + referencePreviewIndex === null ? null : referenceMaterials[referencePreviewIndex]?.type === "image" + ? referenceMaterials[referencePreviewIndex]?.url ?? null + : null; const canEditReferenceImages = (context.mode === "empty-create" || context.mode === "node-edit") && connectedReferenceImages.length === 0; @@ -1470,10 +1615,14 @@ export function GenerationComposer() { value: unknown, patch: Partial = {}, fields: DraftField[] = [], - expectedContextIdentity?: string + expectedContextIdentity?: string, + options: { persistFirstClassParameter?: boolean } = {} ) => { const parameters = updateModelParameterValue(draftRef.current.parameters, name, value, { - persistFirstClassParameter: !isFirstClassGenerationParameterName(name) || fields.length === 0, + persistFirstClassParameter: + options.persistFirstClassParameter || + !isFirstClassGenerationParameterName(name) || + fields.length === 0, }); const nextPatch = { ...patch, parameters }; const updatedFields: DraftField[] = [...fields, "parameters"]; @@ -1509,6 +1658,37 @@ export function GenerationComposer() { [updateNodeData] ); + const isVideoSubTypeSelectorVisible = context.mode === "node-edit" && context.nodeType === "generateVideo"; + const currentVideoSubType = videoGenerationSubTypeFromUnknown(draft.parameters?.subType); + const selectedVideoSubType = currentVideoSubType ?? VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE; + const isFirstLastFrameSelected = + selectedVideoSubType === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME && canUseFirstLastFrame; + const videoSubTypeOptions = useMemo( + () => [ + { + label: t("composer.videoSubTypeMultiReference"), + value: VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE, + }, + { + label: t("composer.videoSubTypeFirstLastFrame"), + value: VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME, + disabled: !canUseFirstLastFrame, + }, + { + label: t("composer.videoSubTypeTextToVideo"), + value: VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO, + }, + ], + [canUseFirstLastFrame, t] + ); + + useEffect(() => { + if (!isVideoSubTypeSelectorVisible) return; + if (currentVideoSubType !== VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME) return; + if (canUseFirstLastFrame) return; + updateSchemaDraftParameter("subType", VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE); + }, [canUseFirstLastFrame, currentVideoSubType, isVideoSubTypeSelectorVisible, updateSchemaDraftParameter]); + useEffect(() => { if ( isProcessNode || @@ -1518,6 +1698,8 @@ export function GenerationComposer() { return; } + const shouldPreservePricedVariantValues = Boolean(draft.selectedModel.pricing?.variants?.length); + for (const select of gatewayDimensionSelects) { if (select.options.length === 0) continue; @@ -1525,39 +1707,44 @@ export function GenerationComposer() { const nextValue = getSchemaParameterDefault(activeParameterSchema, select.parameter.name) ?? select.options[0]?.value; + const hasPersistedValue = hasPersistedSchemaParameterValue(generationNodeData, select.parameter.name); + if (hasPersistedValue) continue; + if (shouldPreservePricedVariantValues && !valueMatchesSchemaOptions(draft.parameters?.[select.parameter.name], optionValues)) { + continue; + } if (isResolutionParameterName(select.parameter.name)) { - if (valueMatchesSchemaOptions(draft.resolution, optionValues)) continue; updateSchemaDraftParameter( select.parameter.name, nextValue, { resolution: String(nextValue) }, ["resolution"], - identity + identity, + { persistFirstClassParameter: activeCapability === "image" } ); return; } if (isAspectRatioParameterName(select.parameter.name)) { - if (valueMatchesSchemaOptions(draft.aspectRatio, optionValues)) continue; updateSchemaDraftParameter( select.parameter.name, nextValue, { aspectRatio: String(nextValue) as AspectRatio }, ["aspectRatio"], - identity + identity, + { persistFirstClassParameter: activeCapability === "image" } ); return; } if (isDurationParameterName(select.parameter.name)) { - if (valueMatchesSchemaOptions(draft.durationSeconds, optionValues)) continue; updateSchemaDraftParameter( select.parameter.name, nextValue, { durationSeconds: normalizeVideoDurationSeconds(nextValue) }, ["durationSeconds"], - identity + identity, + { persistFirstClassParameter: activeCapability === "image" } ); return; } @@ -1568,8 +1755,10 @@ export function GenerationComposer() { draft.aspectRatio, draft.durationSeconds, draft.resolution, + draft.selectedModel.pricing, draftIdentity, gatewayDimensionSelects, + generationNodeData, identity, isProcessNode, updateSchemaDraftParameter, @@ -1714,10 +1903,10 @@ export function GenerationComposer() { ); useEffect(() => { - if (referencePreviewIndex !== null && referencePreviewIndex >= referenceImages.length) { + if (referencePreviewIndex !== null && referencePreviewIndex >= referenceMaterials.length) { setReferencePreviewIndex(null); } - }, [referenceImages.length, referencePreviewIndex]); + }, [referenceMaterials.length, referencePreviewIndex]); useEffect(() => { if (!previewReferenceImage) return; @@ -1749,6 +1938,43 @@ export function GenerationComposer() { if (!canSubmit) return; const fallbackPrompt = buildReferenceOnlyPrompt(); const submitPrompt = promptForSubmit || fallbackPrompt; + const activeDraft = draftRef.current; + const submitVideoNodeType = context.mode === "empty-create" + ? inferNodeTypeFromModel(activeDraft.selectedModel) + : context.nodeType; + const explicitSubmitVideoSubType = videoGenerationSubTypeFromUnknown(activeDraft.parameters?.subType); + const submitVideoSubType = explicitSubmitVideoSubType ?? VIDEO_GENERATION_SUB_TYPE_MULTI_REFERENCE; + const hasSubmitReferenceImage = + (connectedInputs?.images.length ?? 0) > 0 || activeDraft.inputImages.length > 0; + const submitReferenceImageCount = connectedInputs?.images.length && connectedInputs.images.length > 0 + ? connectedInputs.images.length + : activeDraft.inputImages.length; + const referenceSubjectList = buildReferenceSubjectListParameter(referenceMaterials); + const shouldWriteVideoSubType = context.mode === "empty-create" || explicitSubmitVideoSubType !== null; + const videoParameters = submitVideoNodeType === "generateVideo" + ? { + ...activeDraft.parameters, + ...(shouldWriteVideoSubType ? { subType: submitVideoSubType } : {}), + ...(referenceSubjectList.length > 0 ? { referenceSubjectList } : {}), + } + : activeDraft.parameters; + if ( + submitVideoNodeType === "generateVideo" && + explicitSubmitVideoSubType !== null && + explicitSubmitVideoSubType !== VIDEO_GENERATION_SUB_TYPE_TEXT_TO_VIDEO && + !hasSubmitReferenceImage + ) { + message.warning(t("composer.videoSubTypeImageRequired")); + return; + } + if ( + submitVideoNodeType === "generateVideo" && + explicitSubmitVideoSubType === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME && + !canUseFirstLastFrameSubType(submitReferenceImageCount) + ) { + message.warning(t("composer.videoSubTypeFirstLastFrameImageCountRequired")); + return; + } if (context.mode === "unsupported-node" && context.node?.type === "imageInput") { const imageData = context.node.data as ImageInputNodeData; @@ -1775,14 +2001,15 @@ export function GenerationComposer() { } if (context.mode === "empty-create") { - const nodeType = inferNodeTypeFromModel(draftRef.current.selectedModel); + const nodeType = submitVideoNodeType; if (!nodeType) return; const nextNodeId = addNode( nodeType, buildTargetPosition(), buildInitialDataForNode(nodeType, { - ...draftRef.current, + ...activeDraft, + parameters: nodeType === "generateVideo" ? videoParameters : activeDraft.parameters, prompt: submitPrompt, }) ); @@ -1794,6 +2021,18 @@ export function GenerationComposer() { } if (context.mode === "node-edit" && context.node) { + if (submitVideoNodeType === "generateVideo") { + const nextDraft = { + ...draftRef.current, + parameters: videoParameters, + }; + const nextDirtyFields = new Set(dirtyFieldsRef.current); + nextDirtyFields.add("parameters"); + draftRef.current = nextDraft; + dirtyFieldsRef.current = nextDirtyFields; + setDraft(nextDraft); + setDirtyFields(nextDirtyFields); + } if (!promptForSubmit && !promptReadOnly) { const nextDraft = { ...draftRef.current, prompt: submitPrompt }; const nextDirtyFields = new Set(dirtyFieldsRef.current); @@ -1819,12 +2058,15 @@ export function GenerationComposer() { buildTargetPosition, canSubmit, clearDirtyFields, + connectedInputs, context, flushCurrentDraft, promptForSubmit, promptReadOnly, + referenceMaterials, regenerateNode, selectSingleNode, + t, ]); const insertCommand = useCallback((value: string) => { @@ -2003,110 +2245,178 @@ export function GenerationComposer() { ) : ( <> -
- {isProcessNode ? ( - isVideoStitchNode ? ( -
- - {t("composer.videoStitchClipCount", { count: connectedInputs?.videos.length ?? 0 })} - - - - {connectedInputs?.audio.length ? t("composer.videoStitchAudio") : t("composer.videoStitchNoAudio")} - -
- ) : ( -
- - {connectedInputs?.videos.length ? t("composer.easeCurveVideoConnected") : t("composer.easeCurveNoVideo")} - - - - {connectedInputs?.easeCurve ? t("composer.easeCurveInherited") : t("composer.easeCurveLocal")} - -
- ) - ) : ( - <> - - - {referenceImages.length > 0 && ( -
- {referenceImages.map((referenceImage, imageIndex) => ( -
- - - {imageIndex + 1} +
+ {isVideoSubTypeSelectorVisible && ( + { + const nextSubType = videoGenerationSubTypeFromUnknown(value); + if (!nextSubType) return; + if (nextSubType === VIDEO_GENERATION_SUB_TYPE_FIRST_LAST_FRAME && !canUseFirstLastFrame) { + message.warning(t("composer.videoSubTypeFirstLastFrameImageCountRequired")); + return; + } + updateSchemaDraftParameter("subType", nextSubType); + }} + className="nodrag nopan nowheel max-w-full text-xs" + /> + )} + +
+
+ {t("composer.console")} + {modeLabel} +
+ + {isProcessNode ? ( + isVideoStitchNode ? ( +
+ + {t("composer.videoStitchClipCount", { count: connectedInputs?.videos.length ?? 0 })} - {canEditReferenceImages && ( + + + {connectedInputs?.audio.length ? t("composer.videoStitchAudio") : t("composer.videoStitchNoAudio")} + +
+ ) : ( +
+ + {connectedInputs?.videos.length ? t("composer.easeCurveVideoConnected") : t("composer.easeCurveNoVideo")} + + + + {connectedInputs?.easeCurve ? t("composer.easeCurveInherited") : t("composer.easeCurveLocal")} + +
+ ) + ) : ( +
+ + + {referenceMaterials.length > 0 && ( +
+ {referenceMaterials.map((material, materialIndex) => ( +
- )} -
- ))} -
- )} - - )} + + {isFirstLastFrameSelected && material.type === "image" + ? material.id === 1 + ? t("composer.firstFrame") + : t("composer.lastFrame") + : material.id} + + {canEditReferenceImages && material.type === "image" && material.removableIndex !== undefined && ( + + )} +
+ ))} +
+ )} +
+ )} -
- - - - - - -
-
+
+ + + + + + +
+
+
-
+
{assistMenu && (
@@ -2132,17 +2442,38 @@ export function GenerationComposer() {
) : (
- {referenceImages.length > 0 ? ( - referenceImages.map((referenceImage, imageIndex) => { - const referenceLabel = `图${imageIndex + 1}`; + {referenceMaterials.length > 0 ? ( + referenceMaterials.map((material) => { + const referenceLabel = material.name.replace(/^@/, ""); return ( ); @@ -2394,7 +2725,14 @@ export function GenerationComposer() { patch.durationSeconds = normalizeVideoDurationSeconds(option?.value); fields.push("durationSeconds"); } - updateSchemaDraftParameter(parameter.name, option?.value, patch, fields); + updateSchemaDraftParameter( + parameter.name, + option?.value, + patch, + fields, + undefined, + { persistFirstClassParameter: true } + ); }} className="rounded-md border border-transparent bg-transparent px-1.5 py-1 text-neutral-200 outline-none transition-colors hover:border-neutral-700 hover:bg-neutral-700/60" > diff --git a/src/i18n/index.tsx b/src/i18n/index.tsx index 08ec0b71..acfeb07c 100644 --- a/src/i18n/index.tsx +++ b/src/i18n/index.tsx @@ -627,6 +627,13 @@ const en = { "composer.reference": "Reference", "composer.referenceImage": "Reference image {index}", "composer.removeReferenceImage": "Remove reference image {index}", + "composer.videoSubTypeMultiReference": "Image to video", + "composer.videoSubTypeTextToVideo": "All-purpose reference", + "composer.videoSubTypeFirstLastFrame": "First/last frame", + "composer.videoSubTypeImageRequired": "Image to video or first/last frame requires at least one reference image.", + "composer.videoSubTypeFirstLastFrameImageCountRequired": "First/last frame requires exactly two reference images.", + "composer.firstFrame": "First", + "composer.lastFrame": "Last", "composer.commands": "Commands", "composer.referenceMaterials": "Reference assets", "composer.commandCharacter": "Character image", @@ -1639,6 +1646,16 @@ const zhCN: Dictionary = { "cost.viewDetails": "查看费用详情", }; +Object.assign(zhCN, { + "composer.videoSubTypeFirstLastFrameImageCountRequired": "\u9996\u5c3e\u5e27\u9700\u8981\u6070\u597d\u4e24\u5f20\u53c2\u8003\u56fe\u7247\u3002", + "composer.firstFrame": "\u9996\u5e27", + "composer.lastFrame": "\u5c3e\u5e27", + "composer.videoSubTypeMultiReference": "图生视频", + "composer.videoSubTypeTextToVideo": "全能参考", + "composer.videoSubTypeFirstLastFrame": "首尾帧", + "composer.videoSubTypeImageRequired": "图生视频或首尾帧至少需要一张参考图片。", +}); + const zhTW: Dictionary = { ...zhCN, ...zhTWAuth,