Browse Source

线的处理

feature/video-erase
Luckyu_js 2 weeks ago
parent
commit
2fd5805085
  1. 47
      src/components/__tests__/GenerationComposer.test.tsx
  2. 14
      src/components/composer/NodeGenerationComposer.tsx
  3. 21
      src/components/media/CustomVideoPlayer.tsx
  4. 18
      src/components/nodes/GenerateVideoNode.tsx
  5. 11
      src/components/nodes/VideoInputNode.tsx
  6. 100
      src/store/__tests__/workflowStore.integration.test.ts
  7. 90
      src/store/execution/__tests__/batchExecution.test.ts
  8. 73
      src/store/execution/__tests__/derivedImageExecutor.test.ts
  9. 9
      src/store/execution/batchExecution.ts
  10. 49
      src/store/execution/derivedImageExecutor.ts
  11. 21
      src/store/workflowStore.ts
  12. 11
      src/types/nodes.ts
  13. 29
      src/utils/__tests__/nodeConnectionSpec.test.ts
  14. 18
      src/utils/__tests__/smartNodeModes.test.ts
  15. 53
      src/utils/__tests__/videoFrameNode.test.ts
  16. 6
      src/utils/nodeConnectionSpec.ts
  17. 26
      src/utils/smartMediaMode.ts
  18. 30
      src/utils/videoFrameNode.ts

47
src/components/__tests__/GenerationComposer.test.tsx

@ -1215,6 +1215,53 @@ describe("GenerationComposer", () => {
});
});
it("does not show the composer for selected video crop and erase task nodes", () => {
const operations: NonNullable<SmartVideoNodeData["derivedVideo"]>["operation"][] = [
"crop",
"smartErase",
"boxErase",
];
operations.forEach((operation) => {
useWorkflowStore.setState({
nodes: [
videoNode("smartVideo-1", true, {
status: "complete",
outputVideo: `https://example.com/${operation}.mp4`,
derivedVideo: {
sourceNodeId: "video-source-1",
operation,
},
}),
],
});
const { unmount } = render(<NodeGenerationComposer nodeId="smartVideo-1" selected />);
expect(screen.queryByTestId("node-inline-composer")).not.toBeInTheDocument();
unmount();
});
});
it("does not show the composer for captured video frame smart image nodes", () => {
useWorkflowStore.setState({
nodes: [
smartImageNode("smartImage-1", true, {
status: "idle",
image: "https://example.com/video-frame.png",
derivedMedia: {
sourceNodeId: "video-source-1",
operation: "videoFrame",
},
}),
],
});
render(<NodeGenerationComposer nodeId="smartImage-1" selected />);
expect(screen.queryByTestId("node-inline-composer")).not.toBeInTheDocument();
});
it("collapses and expands the bottom composer without losing the selected node context", () => {
useWorkflowStore.setState({ nodes: [imageNode("img-1", true, { inputPrompt: "visible prompt" })] });
render(<GenerationComposer />);

14
src/components/composer/NodeGenerationComposer.tsx

@ -10,7 +10,11 @@ import { useWorkflowStore } from "@/store/workflowStore";
import type { WorkflowNode } from "@/types";
import { deriveComposerContextForNodeId } from "@/utils/composerNodeDescriptor";
import { getNodeImageSource } from "@/utils/getNodeImageSource";
import { isSmartImageAutoTaskNode } from "@/utils/smartMediaMode";
import {
isSmartImageAutoTaskNode,
isSmartVideoAutoTaskNode,
isVideoFrameDerivedMediaNode,
} from "@/utils/smartMediaMode";
import { isHighDefinitionDerivedImageNode } from "@/utils/highDefinitionNodes";
import { isHighDefinitionDerivedVideoNode } from "@/utils/highDefinitionVideoNodes";
import { getNodeVideoSource } from "@/utils/getNodeVideoSource";
@ -37,7 +41,11 @@ function isToolSupported(tool: NodeToolType, node: WorkflowNode | null): boolean
}
function isAutoImageTaskNode(node: WorkflowNode | null): boolean {
return isSmartImageAutoTaskNode(node?.data);
return isSmartImageAutoTaskNode(node?.data) || isVideoFrameDerivedMediaNode(node?.data);
}
function isAutoVideoTaskNode(node: WorkflowNode | null): boolean {
return isSmartVideoAutoTaskNode(node?.data);
}
export function NodeGenerationComposer({
@ -90,7 +98,7 @@ function EditableNodeGenerationComposer({ nodeId, selected }: Pick<NodeGeneratio
isHighDefinitionDerivedImageNode(context.node.data) ||
isHighDefinitionDerivedVideoNode(context.node.data)
));
const hideForAutoTask = isAutoImageTaskNode(context.node) || isHighDefinitionNode;
const hideForAutoTask = isAutoImageTaskNode(context.node) || isAutoVideoTaskNode(context.node) || isHighDefinitionNode;
const isImageInputComposer = context.mode === "unsupported-node" && context.node?.type === "imageInput";
const shouldShowComposer =
selected &&

21
src/components/media/CustomVideoPlayer.tsx

@ -4,11 +4,6 @@ import { Popover } from "antd";
import React, { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { preventVideoDoubleClick } from "@/hooks/usePreventVideoFullscreen";
import { useVideoBlobUrl } from "@/hooks/useVideoBlobUrl";
import {
captureFrameFromUrlAtTime,
captureFrameFromVideoElement,
type CapturedVideoFrame,
} from "@/utils/videoFrameCapture";
interface CustomVideoPlayerProps {
src: string;
@ -25,7 +20,7 @@ interface CustomVideoPlayerProps {
duration: number | null;
dimensions: { width: number; height: number } | null;
}) => void;
onCaptureFrame?: (frame: CapturedVideoFrame & { time: number }) => Promise<void> | void;
onCaptureFrame?: (frame: { time: number; mode: CaptureFrameMode }) => Promise<void> | void;
onError?: () => void;
}
@ -233,21 +228,11 @@ export function CustomVideoPlayer({
: mode === "last"
? Math.max(0, videoDuration - 0.1)
: video.currentTime;
let frame: CapturedVideoFrame;
if (mode === "current") {
try {
frame = await captureFrameFromVideoElement(video);
} catch (error) {
frame = await captureFrameFromUrlAtTime(src, time);
}
} else {
frame = await captureFrameFromUrlAtTime(src, time);
}
await onCaptureFrame({ ...frame, time });
await onCaptureFrame({ time, mode });
} finally {
setIsCaptureLoading(false);
}
}, [duration, isCaptureLoading, onCaptureFrame, src]);
}, [duration, isCaptureLoading, onCaptureFrame]);
const playPauseLabel = useMemo(() => isPlaying ? "暂停" : "播放", [isPlaying]);
const volumeLabel = useMemo(() => {

18
src/components/nodes/GenerateVideoNode.tsx

@ -34,7 +34,7 @@ import { asMediaDimensions, getAutoMediaElementClassName, getAutoMediaFrameClass
import { CustomVideoPlayer } from "@/components/media/CustomVideoPlayer";
import { createVideoFrameSmartImageNode } from "@/utils/videoFrameNode";
import { chooseHighDefinitionVideoModel, createHighDefinitionVideoNode } from "@/utils/highDefinitionVideoNodes";
import { isVideoEraseDerivedNode } from "@/utils/videoEraseNodes";
import { isSmartVideoAutoTaskNode } from "@/utils/smartMediaMode";
import { getActiveNodeTool, useNodeToolStore } from "@/store/nodeToolStore";
import { createVideoEraseActionPanel } from "@/components/media/MediaEditActionDropdown";
import { VideoEraseMenuIcon } from "@/components/media/MediaToolbarIcons";
@ -116,9 +116,9 @@ export function GenerateVideoNodeView({ id, data, selected, suppressEmptyStateDe
useEffect(() => {
if (modelOptions.length === 0) return;
// 擦除派生节点使用内置的擦除模型(可能不在目录列表中),不参与自动选型,
// 否则会被替换成默认视频生成模型、丢失 207/208 路由
if (isVideoEraseDerivedNode(nodeData)) return;
// 视频派生节点使用内置模型或不需要生成模型(裁剪为本地处理),不参与
// 自动选型,否则会被替换成默认视频生成模型、丢失派生语义
if (isSmartVideoAutoTaskNode(nodeData)) return;
const selectedModelExists = modelOptions.some((model) => model.id === nodeConfig.selectedModel?.modelId);
if (nodeConfig.selectedModel?.provider === currentProvider && nodeConfig.selectedModel.modelId && selectedModelExists) return;
const selectedModel = chooseModelFromList(modelOptions, nodeConfig.selectedModel);
@ -334,19 +334,21 @@ export function GenerateVideoNodeView({ id, data, selected, suppressEmptyStateDe
const hasCarouselVideos = (nodeData.videoHistory || []).length > 1;
const handleCaptureFrame = useCallback(async (frame: {
blob: Blob;
dimensions: { width: number; height: number };
time: number;
mode: "first" | "last" | "current";
}) => {
const currentNode = getNodeById(id);
if (!currentNode) return;
await createVideoFrameSmartImageNode({
sourceNode: currentNode,
frame,
time: frame.time,
mode: frame.mode,
addNode,
onConnect,
selectSingleNode,
regenerateNode,
});
}, [addNode, getNodeById, id, onConnect, selectSingleNode]);
}, [addNode, getNodeById, id, onConnect, regenerateNode, selectSingleNode]);
const handleCreateHighDefinitionVideoNode = useCallback(() => {
const currentNode = getNodeById(id);

11
src/components/nodes/VideoInputNode.tsx

@ -87,6 +87,7 @@ export function VideoInputNodeView({ id, data, selected }: VideoInputNodeViewPro
const onConnect = useWorkflowStore((state) => state.onConnect);
const getNodeById = useWorkflowStore((state) => state.getNodeById);
const selectSingleNode = useWorkflowStore((state) => state.selectSingleNode);
const regenerateNode = useWorkflowStore((state) => state.regenerateNode);
const popiserverVideoModels = useModelStore((state) => state.byKind.video);
const fileInputRef = useRef<HTMLInputElement>(null);
const [isPreviewOpen, setIsPreviewOpen] = useState(false);
@ -251,19 +252,21 @@ export function VideoInputNodeView({ id, data, selected }: VideoInputNodeViewPro
const hasVideoAsset = Boolean(nodeData.video || previewVideoPoster);
const handleCaptureFrame = useCallback(async (frame: {
blob: Blob;
dimensions: { width: number; height: number };
time: number;
mode: "first" | "last" | "current";
}) => {
const currentNode = getNodeById(id);
if (!currentNode) return;
await createVideoFrameSmartImageNode({
sourceNode: currentNode,
frame,
time: frame.time,
mode: frame.mode,
addNode,
onConnect,
selectSingleNode,
regenerateNode,
});
}, [addNode, getNodeById, id, onConnect, selectSingleNode]);
}, [addNode, getNodeById, id, onConnect, regenerateNode, selectSingleNode]);
const handleCreateHighDefinitionVideoNode = useCallback(() => {
const currentNode = getNodeById(id);

100
src/store/__tests__/workflowStore.integration.test.ts

@ -18,6 +18,8 @@ import {
} from "@/constants/generationTask";
import type { WorkflowNode, WorkflowEdge } from "@/types";
import { executeDerivedImage } from "../execution/derivedImageExecutor";
import { executeDerivedVideo } from "../execution/derivedVideoExecutor";
import { executeGenerateVideo } from "../execution/generateVideoExecutor";
import { createVideoCropNode } from "@/utils/videoCropNodes";
// Mock the Toast hook
@ -49,6 +51,22 @@ vi.mock("../execution/derivedImageExecutor", async (importOriginal) => {
};
});
vi.mock("../execution/derivedVideoExecutor", async (importOriginal) => {
const actual = await importOriginal<typeof import("../execution/derivedVideoExecutor")>();
return {
...actual,
executeDerivedVideo: vi.fn().mockResolvedValue(undefined),
};
});
vi.mock("../execution/generateVideoExecutor", async (importOriginal) => {
const actual = await importOriginal<typeof import("../execution/generateVideoExecutor")>();
return {
...actual,
executeGenerateVideo: vi.fn().mockResolvedValue(undefined),
};
});
vi.mock("@/lib/popiartTask/runTask", () => ({
runPopiartTask: ({ payload, headers, signal }: {
payload: { mediaType?: string };
@ -2163,6 +2181,88 @@ describe("workflowStore integration tests", () => {
expect(useWorkflowStore.getState().runningGroupIds.size).toBe(0);
});
it("skips video auto task nodes and captured video frame nodes when running a group", async () => {
useWorkflowStore.setState({
nodes: [
{ ...createTestNode("prompt-1", "prompt", { prompt: "generate video" }), groupId: "group-1" },
{
...createTestNode("video-regular-1", "smartVideo", {
status: "idle",
outputVideo: null,
videoHistory: [],
}),
groupId: "group-1",
},
{
...createTestNode("video-crop-1", "smartVideo", {
status: "idle",
derivedVideo: {
sourceNodeId: "video-source-1",
operation: "crop",
task: {
cropRect: { x: 0, y: 0, width: 100, height: 80 },
outputWidth: 100,
outputHeight: 80,
aspectRatio: "original",
},
},
}),
groupId: "group-1",
},
{
...createTestNode("video-erase-1", "smartVideo", {
status: "idle",
derivedVideo: {
sourceNodeId: "video-source-1",
operation: "smartErase",
},
}),
groupId: "group-1",
},
{
...createTestNode("video-box-erase-1", "smartVideo", {
status: "idle",
derivedVideo: {
sourceNodeId: "video-source-1",
operation: "boxErase",
},
}),
groupId: "group-1",
},
{
...createTestNode("video-frame-1", "smartImage", {
status: "idle",
image: "https://example.com/video-frame.png",
derivedMedia: {
sourceNodeId: "video-source-1",
operation: "videoFrame",
},
}),
groupId: "group-1",
},
],
edges: [
createTestEdge("prompt-1", "video-regular-1", "output", "input"),
],
groups: {
"group-1": {
id: "group-1",
name: "Group 1",
color: "neutral",
position: { x: 0, y: 0 },
size: { width: 900, height: 500 },
},
},
});
await useWorkflowStore.getState().runGroup("group-1");
expect(executeGenerateVideo).toHaveBeenCalledTimes(1);
expect(executeDerivedVideo).not.toHaveBeenCalled();
expect(useWorkflowStore.getState().runningNodeIds.size).toBe(0);
expect(useWorkflowStore.getState().runningGroupIds.size).toBe(0);
});
describe("createGroup bounding box calculation", () => {
it("should correctly calculate bounding box for easeCurve nodes (340x260)", () => {
useWorkflowStore.setState({

90
src/store/execution/__tests__/batchExecution.test.ts

@ -0,0 +1,90 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { runBatchIfApplicable } from "../batchExecution";
import { executeGenerateVideo } from "../generateVideoExecutor";
import type { NodeExecutionContext } from "../types";
import type { WorkflowNode } from "@/types";
vi.mock("../nanoBananaExecutor", () => ({
executeNanoBanana: vi.fn(),
}));
vi.mock("../generateVideoExecutor", () => ({
executeGenerateVideo: vi.fn(),
}));
vi.mock("../generateAudioExecutor", () => ({
executeGenerateAudio: vi.fn(),
}));
vi.mock("../llmGenerateExecutor", () => ({
executeLlmGenerate: vi.fn(),
}));
function makeCtx(node: WorkflowNode): NodeExecutionContext {
return {
node,
getConnectedInputs: vi.fn().mockReturnValue({
images: [],
videos: ["https://example.com/source.mp4"],
audio: [],
model3d: null,
text: null,
textItems: [],
batchTextItems: ["one", "two"],
dynamicInputs: {},
easeCurve: null,
}),
updateNodeData: vi.fn(),
updateMediaNodeData: vi.fn(),
getFreshNode: vi.fn().mockReturnValue(node),
getEdges: vi.fn().mockReturnValue([]),
getNodes: vi.fn().mockReturnValue([]),
providerSettings: {} as any,
addIncurredCost: vi.fn(),
appendOutputGalleryImage: vi.fn(),
get: vi.fn(),
};
}
describe("runBatchIfApplicable", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("does not batch video auto task nodes", async () => {
const node = {
id: "video-crop-1",
type: "smartVideo",
position: { x: 0, y: 0 },
data: {
derivedVideo: {
sourceNodeId: "video-source-1",
operation: "crop",
},
},
} as WorkflowNode;
await expect(runBatchIfApplicable(makeCtx(node), { useStoredFallback: true })).resolves.toBe(false);
expect(executeGenerateVideo).not.toHaveBeenCalled();
});
it("does not batch captured video frame smart image nodes", async () => {
const node = {
id: "video-frame-1",
type: "smartImage",
position: { x: 0, y: 0 },
data: {
derivedMedia: {
sourceNodeId: "video-source-1",
operation: "videoFrame",
task: {
time: 1.25,
mode: "current",
},
},
},
} as WorkflowNode;
await expect(runBatchIfApplicable(makeCtx(node), { useStoredFallback: true })).resolves.toBe(false);
});
});

73
src/store/execution/__tests__/derivedImageExecutor.test.ts

@ -4,6 +4,7 @@ import type { NodeExecutionContext } from "../types";
import type { WorkflowNode } from "@/types";
import { createCroppedImageBlob, loadCropImage } from "@/components/media/imageCropUtils";
import { uploadBlobToGatewayMedia } from "@/utils/gatewayMediaUpload";
import { captureFrameFromUrlAtTime } from "@/utils/videoFrameCapture";
vi.mock("@/components/media/imageCropUtils", () => ({
createCroppedImageBlob: vi.fn(),
@ -14,6 +15,10 @@ vi.mock("@/utils/gatewayMediaUpload", () => ({
uploadBlobToGatewayMedia: vi.fn(),
}));
vi.mock("@/utils/videoFrameCapture", () => ({
captureFrameFromUrlAtTime: vi.fn(),
}));
function makeNode(data: Record<string, unknown>): WorkflowNode {
return {
id: "derived-1",
@ -23,12 +28,15 @@ function makeNode(data: Record<string, unknown>): WorkflowNode {
} as WorkflowNode;
}
function makeCtx(node: WorkflowNode, images: string[] = ["source.png"]): NodeExecutionContext {
function makeCtx(
node: WorkflowNode,
inputs: { images?: string[]; videos?: string[] } = { images: ["source.png"] }
): NodeExecutionContext {
return {
node,
getConnectedInputs: vi.fn().mockReturnValue({
images,
videos: [],
images: inputs.images ?? [],
videos: inputs.videos ?? [],
audio: [],
model3d: null,
text: null,
@ -60,6 +68,7 @@ describe("derivedImageExecutor", () => {
it("recognizes crop and splitGrid derived image nodes", () => {
expect(isDerivedImageExecutionNode({ derivedImage: { operation: "crop" } })).toBe(true);
expect(isDerivedImageExecutionNode({ derivedImage: { operation: "splitGrid" } })).toBe(true);
expect(isDerivedImageExecutionNode({ derivedMedia: { operation: "videoFrame" } })).toBe(true);
expect(isDerivedImageExecutionNode({ derivedImage: { operation: "multiAngle" } })).toBe(false);
expect(isDerivedImageExecutionNode({ derivedImage: { operation: "highDefinition" } })).toBe(false);
});
@ -113,7 +122,7 @@ describe("derivedImageExecutor", () => {
},
},
});
const ctx = makeCtx(node, []);
const ctx = makeCtx(node, { images: [] });
await expect(executeDerivedImage(ctx)).rejects.toThrow("Missing source image.");
expect(ctx.updateNodeData).toHaveBeenCalledWith("derived-1", {
@ -162,4 +171,60 @@ describe("derivedImageExecutor", () => {
status: "complete",
}), { width: 20, height: 20 });
});
it("executes captured video frame tasks from a connected video", async () => {
const blob = new Blob(["frame"], { type: "image/png" });
vi.mocked(captureFrameFromUrlAtTime).mockResolvedValue({
blob,
dimensions: { width: 1920, height: 1080 },
});
vi.mocked(uploadBlobToGatewayMedia).mockResolvedValue({
url: "https://media.example/video-frame.png",
filename: "video-frame.png",
contentType: "image/png",
});
const node = makeNode({
filename: "video-frame.png",
derivedMedia: {
sourceNodeId: "video-source-1",
operation: "videoFrame",
task: {
time: 1.25,
mode: "current",
},
},
});
const ctx = makeCtx(node, { videos: ["https://example.com/source.mp4"] });
await executeDerivedImage(ctx);
expect(captureFrameFromUrlAtTime).toHaveBeenCalledWith("https://example.com/source.mp4", 1.25);
expect(uploadBlobToGatewayMedia).toHaveBeenCalledWith(blob, "video-frame.png", "image/png", "image");
expect(ctx.updateMediaNodeData).toHaveBeenCalledWith("derived-1", expect.objectContaining({
image: "https://media.example/video-frame.png",
dimensions: { width: 1920, height: 1080 },
status: "complete",
error: null,
}), { width: 1920, height: 1080 });
});
it("sets an error when a video frame task has no connected video", async () => {
const node = makeNode({
derivedMedia: {
sourceNodeId: "video-source-1",
operation: "videoFrame",
task: {
time: 1.25,
mode: "current",
},
},
});
const ctx = makeCtx(node, { videos: [] });
await expect(executeDerivedImage(ctx)).rejects.toThrow("Missing source video.");
expect(ctx.updateNodeData).toHaveBeenCalledWith("derived-1", {
status: "error",
error: "Missing source video.",
});
});
});

9
src/store/execution/batchExecution.ts

@ -13,6 +13,7 @@ import { executeNanoBanana } from "./nanoBananaExecutor";
import { executeGenerateVideo } from "./generateVideoExecutor";
import { executeGenerateAudio } from "./generateAudioExecutor";
import { executeLlmGenerate } from "./llmGenerateExecutor";
import { isSmartVideoAutoTaskNode, isVideoFrameDerivedMediaNode } from "@/utils/smartMediaMode";
const BATCH_NODE_TYPES = new Set(["smartImage", "nanoBanana", "generateVideo", "smartAudio", "generateAudio", "llmGenerate", "smartText"]);
@ -47,6 +48,14 @@ export async function runBatchIfApplicable(
return false;
}
if (node.type === "smartImage" && isVideoFrameDerivedMediaNode(node.data)) {
return false;
}
if (node.type === "smartVideo" && isSmartVideoAutoTaskNode(node.data)) {
return false;
}
const connectedInputs = executionCtx.getConnectedInputs(node.id);
const batchTextItems = connectedInputs.batchTextItems ?? [];
if (batchTextItems.length === 0) {

49
src/store/execution/derivedImageExecutor.ts

@ -7,9 +7,15 @@ import { createCroppedImageBlob } from "@/components/media/imageCropUtils";
import { uploadBlobToGatewayMedia } from "@/utils/gatewayMediaUpload";
import { createGridForDimensions } from "@/utils/gridSplitter";
import { loadCropImage } from "@/components/media/imageCropUtils";
import { isSmartImageLocalDerivedTaskNode } from "@/utils/smartMediaMode";
import { isSmartImageLocalDerivedTaskNode, isVideoFrameDerivedMediaNode } from "@/utils/smartMediaMode";
import { captureFrameFromUrlAtTime } from "@/utils/videoFrameCapture";
import type { NodeExecutionContext } from "./types";
type VideoFrameDerivedTask = {
time: number;
mode: "first" | "last" | "current";
};
function isCropTask(value: unknown): value is DerivedImageCropTask {
if (!value || typeof value !== "object") return false;
const task = value as DerivedImageCropTask;
@ -31,6 +37,14 @@ function isSplitGridTask(value: unknown): value is DerivedImageSplitGridTask {
);
}
function isVideoFrameTask(value: unknown): value is VideoFrameDerivedTask {
if (!value || typeof value !== "object") return false;
const task = value as VideoFrameDerivedTask;
return typeof task.time === "number" &&
Number.isFinite(task.time) &&
(task.mode === "first" || task.mode === "last" || task.mode === "current");
}
async function createSplitGridCellBlob(
sourceImage: string,
task: DerivedImageSplitGridTask
@ -79,17 +93,26 @@ async function createSplitGridCellBlob(
}
export function isDerivedImageExecutionNode(data: unknown): boolean {
return isSmartImageLocalDerivedTaskNode(data);
return isSmartImageLocalDerivedTaskNode(data) || isVideoFrameDerivedMediaNode(data);
}
export async function executeDerivedImage(ctx: NodeExecutionContext): Promise<void> {
const nodeData = ctx.node.data as SmartImageNodeData;
const derivedImage = nodeData.derivedImage;
if (!derivedImage) return;
const derivedMedia = nodeData.derivedMedia;
const isVideoFrame = derivedMedia?.operation === "videoFrame";
if (!derivedImage && !isVideoFrame) return;
const { images } = ctx.getConnectedInputs(ctx.node.id);
const sourceImage = images[0];
if (!sourceImage) {
const inputs = ctx.getConnectedInputs(ctx.node.id);
const sourceImage = inputs.images[0];
const sourceVideo = inputs.videos[0];
if (isVideoFrame) {
if (!sourceVideo) {
const message = "Missing source video.";
ctx.updateNodeData(ctx.node.id, { status: "error", error: message });
throw new Error(message);
}
} else if (!sourceImage) {
const message = "Missing source image.";
ctx.updateNodeData(ctx.node.id, { status: "error", error: message });
throw new Error(message);
@ -104,7 +127,14 @@ export async function executeDerivedImage(ctx: NodeExecutionContext): Promise<vo
let blob: Blob;
let dimensions: { width: number; height: number };
if (derivedImage.operation === "crop") {
if (isVideoFrame) {
if (!isVideoFrameTask(derivedMedia?.task)) {
throw new Error("Missing video frame task parameters.");
}
const frame = await captureFrameFromUrlAtTime(sourceVideo, derivedMedia.task.time);
blob = frame.blob;
dimensions = frame.dimensions;
} else if (derivedImage?.operation === "crop") {
if (!isCropTask(derivedImage.task)) {
throw new Error("Missing crop task parameters.");
}
@ -117,7 +147,7 @@ export async function executeDerivedImage(ctx: NodeExecutionContext): Promise<vo
width: derivedImage.task.outputWidth,
height: derivedImage.task.outputHeight,
};
} else if (derivedImage.operation === "splitGrid") {
} else if (derivedImage?.operation === "splitGrid") {
if (!isSplitGridTask(derivedImage.task)) {
throw new Error("Missing split grid task parameters.");
}
@ -128,7 +158,8 @@ export async function executeDerivedImage(ctx: NodeExecutionContext): Promise<vo
return;
}
const filename = nodeData.filename || `${derivedImage.operation}-${Date.now()}.png`;
const operation = isVideoFrame ? "video-frame" : derivedImage?.operation ?? "derived-image";
const filename = nodeData.filename || `${operation}-${Date.now()}.png`;
const uploaded = await uploadBlobToGatewayMedia(blob, filename, "image/png", "image");
const completePatch = {
image: uploaded.url,

21
src/store/workflowStore.ts

@ -67,7 +67,9 @@ import {
isSmartImageAiStaticDerivedTaskNode,
isSmartImageAutoTaskNode,
isSmartImageGenerateMode,
isSmartVideoAutoTaskNode,
isSmartVideoGenerateMode,
isVideoFrameDerivedMediaNode,
} from "@/utils/smartMediaMode";
import { isSmartTextLlmMode } from "@/utils/smartTextMode";
import { normalizeSelectedModel } from "@/utils/selectedModel";
@ -574,6 +576,19 @@ function canConnectToDerivedMediaTarget(connection: Connection, nodes: WorkflowN
return isDerivedMediaParentConnection(connection, targetNode);
}
function shouldSkipGroupAutoTaskNode(node: WorkflowNode | undefined): boolean {
if (!node) return false;
if (node.type === "smartImage") {
return (isSmartImageAutoTaskNode(node.data) || isVideoFrameDerivedMediaNode(node.data)) &&
!isHighDefinitionDerivedImageNode(node.data);
}
if (node.type === "smartVideo") {
return isSmartVideoAutoTaskNode(node.data) &&
!isHighDefinitionDerivedVideoNode(node.data);
}
return false;
}
function migrateWorkflowNodeData(data: WorkflowNodeData): WorkflowNodeData {
const record = { ...(data as Record<string, unknown>) };
delete record.config;
@ -2819,7 +2834,7 @@ const workflowStoreImpl: StateCreator<WorkflowStore> = (set, get) => ({
const currentNodes = get().nodes;
const memberNodeIds = getGroupMemberNodeIds(currentNodes, groupId).filter((nodeId) => {
const node = currentNodes.find((candidate) => candidate.id === nodeId);
return !(node?.type === "smartImage" && isSmartImageAutoTaskNode(node.data) && !isHighDefinitionDerivedImageNode(node.data));
return !shouldSkipGroupAutoTaskNode(node);
});
if (memberNodeIds.length === 0) {
logger.warn("node.execution", "No nodes found in group for execution", { groupId });
@ -3112,8 +3127,8 @@ const workflowStoreImpl: StateCreator<WorkflowStore> = (set, get) => ({
throw new DOMException('Aborted', 'AbortError');
}
if (options?.groupId && node.type === "smartImage" && isSmartImageAutoTaskNode(node.data) && !isHighDefinitionDerivedImageNode(node.data)) {
logger.info("node.execution", "Skipping smart image auto task node during group execution", {
if (options?.groupId && shouldSkipGroupAutoTaskNode(node)) {
logger.info("node.execution", "Skipping media auto task node during group execution", {
nodeId: node.id,
groupId: options.groupId,
});

11
src/types/nodes.ts

@ -89,6 +89,15 @@ export type DerivedVideoCropTask = DerivedImageCropTask;
export type DerivedVideoTask = DerivedVideoCropTask;
export interface DerivedMediaInfo {
sourceNodeId: string;
operation: "videoFrame";
task?: {
time: number;
mode: "first" | "last" | "current";
};
}
/**
* Image input node - loads/uploads images into the workflow
*/
@ -106,6 +115,7 @@ export interface ImageInputNodeData extends BaseNodeData {
operation: "crop" | "multiAngle" | "splitGrid" | "highDefinition" | "outpainting" | "inpainting" | "annotate";
task?: DerivedImageTask;
};
derivedMedia?: DerivedMediaInfo;
}
/**
@ -154,6 +164,7 @@ export interface VideoInputNodeData extends BaseNodeData {
operation: "crop" | "highDefinition" | "smartErase" | "boxErase";
task?: DerivedVideoTask;
};
derivedMedia?: DerivedMediaInfo;
}
/**

29
src/utils/__tests__/nodeConnectionSpec.test.ts

@ -126,6 +126,35 @@ describe("node connection spec", () => {
expect(canConnectByNodeConnectionSpec(other, derived, [])).toBe(false);
});
it("lets captured video frame smart image nodes accept only their parent video", () => {
const parent = { id: "parent-video", type: "smartVideo", data: { video: "parent.mp4" } };
const other = { id: "other-video", type: "smartVideo", data: { video: "other.mp4" } };
const imageParent = { id: "parent-image", type: "smartImage", data: { image: "parent.png" } };
const derived = {
id: "video-frame-image",
type: "smartImage",
data: {
derivedMedia: {
sourceNodeId: "parent-video",
operation: "videoFrame",
task: {
time: 1.25,
mode: "current",
},
},
},
};
expect(getNodeConnectionSpec(derived, [])).toMatchObject({
mode: "derived",
input: { visible: true, accepts: ["video"], sourceNodeId: "parent-video" },
output: { visible: true, produces: "image" },
});
expect(canConnectByNodeConnectionSpec(parent, derived, [])).toBe(true);
expect(canConnectByNodeConnectionSpec(other, derived, [])).toBe(false);
expect(canConnectByNodeConnectionSpec(imageParent, derived, [])).toBe(false);
});
it("lets high definition nodes use regular smart image input rules", () => {
const parent = { id: "parent-image", type: "smartImage", data: { image: "parent.png" } };
const other = { id: "other-image", type: "smartImage", data: { image: "other.png" } };

18
src/utils/__tests__/smartNodeModes.test.ts

@ -3,6 +3,7 @@ import {
deriveSmartAudioMode,
deriveSmartImageMode,
deriveSmartVideoMode,
isSmartVideoAutoTaskNode,
} from "@/utils/smartMediaMode";
import type { WorkflowEdge } from "@/types";
@ -67,6 +68,15 @@ describe("smart node mode derivation", () => {
}, [inputEdge("smart-image-1")])).toBe("generate");
});
it("treats captured video frame smart image nodes as image mode when connected", () => {
expect(deriveSmartImageMode("smart-image-1", {
image: "https://example.com/video-frame.png",
outputImage: null,
imageHistory: [],
derivedMedia: { sourceNodeId: "source-1", operation: "videoFrame" },
}, [inputEdge("smart-image-1")])).toBe("image");
});
it("keeps an unconnected empty smart video node in upload-capable neutral mode", () => {
expect(deriveSmartVideoMode("smart-video-1", { video: null }, [])).toBe("neutral");
});
@ -101,6 +111,14 @@ describe("smart node mode derivation", () => {
}, [])).toBe("neutral");
});
it("treats video crop and erase nodes as video auto task nodes", () => {
for (const operation of ["crop", "smartErase", "boxErase"]) {
const data = { derivedVideo: { sourceNodeId: "source-1", operation } };
expect(isSmartVideoAutoTaskNode(data)).toBe(true);
expect(deriveSmartVideoMode("smart-video-1", data, [])).toBe("generate");
}
});
it("keeps an unconnected empty smart audio node in upload-capable empty mode", () => {
expect(deriveSmartAudioMode("smart-audio-1", { audioFile: null }, [])).toBe("empty");
});

53
src/utils/__tests__/videoFrameNode.test.ts

@ -1,28 +1,17 @@
import { describe, expect, it, vi, beforeEach } from "vitest";
import { createVideoFrameSmartImageNode } from "@/utils/videoFrameNode";
import { uploadBlobToGatewayMedia } from "@/utils/gatewayMediaUpload";
vi.mock("@/utils/gatewayMediaUpload", () => ({
uploadBlobToGatewayMedia: vi.fn(),
}));
describe("createVideoFrameSmartImageNode", () => {
beforeEach(() => {
vi.clearAllMocks();
});
it("uploads the captured frame and connects the image node to its source video node", async () => {
it("creates a smart image task node, connects it to the source video node, and starts execution", async () => {
const addNode = vi.fn().mockReturnValue("frame-node-1");
const onConnect = vi.fn();
const selectSingleNode = vi.fn();
const blob = new Blob(["frame"], { type: "image/png" });
vi.mocked(uploadBlobToGatewayMedia).mockResolvedValue({
url: "https://media.example/video-frame.png",
previewUrl: "https://media.example/video-frame-thumb.png",
filename: "video-frame.png",
contentType: "image/png",
});
const regenerateNode = vi.fn();
const nodeId = await createVideoFrameSmartImageNode({
sourceNode: {
@ -32,31 +21,38 @@ describe("createVideoFrameSmartImageNode", () => {
data: {},
style: { width: 300 },
} as any,
frame: {
blob,
dimensions: { width: 1920, height: 1080 },
},
time: 1.25,
mode: "current",
addNode,
onConnect,
selectSingleNode,
regenerateNode,
});
expect(nodeId).toBe("frame-node-1");
expect(uploadBlobToGatewayMedia).toHaveBeenCalledWith(
blob,
expect.stringContaining("video-frame-"),
"image/png",
"image"
);
expect(addNode).toHaveBeenCalledWith("smartImage", { x: 530, y: 20 }, expect.objectContaining({
image: "https://media.example/video-frame.png",
dimensions: { width: 1920, height: 1080 },
image: null,
dimensions: null,
status: "loading",
derivedMedia: {
sourceNodeId: "video-node-1",
operation: "videoFrame",
task: {
time: 1.25,
mode: "current",
},
},
}));
// The uploaded network URL is stored — never a base64 data URL, and no
// deferred videoFrame derived-image task remains.
const addNodeData = addNode.mock.calls[0][2];
expect(addNodeData.image).not.toContain("data:");
expect(addNodeData.derivedImage).toBeUndefined();
expect(addNodeData.derivedMedia).toEqual({
sourceNodeId: "video-node-1",
operation: "videoFrame",
task: {
time: 1.25,
mode: "current",
},
});
expect(onConnect).toHaveBeenCalledWith({
source: "video-node-1",
sourceHandle: "output",
@ -64,5 +60,6 @@ describe("createVideoFrameSmartImageNode", () => {
targetHandle: "input",
});
expect(selectSingleNode).toHaveBeenCalledWith("frame-node-1");
expect(regenerateNode).toHaveBeenCalledWith("frame-node-1");
});
});

6
src/utils/nodeConnectionSpec.ts

@ -5,6 +5,7 @@ import {
deriveSmartAudioMode,
deriveSmartImageMode,
deriveSmartVideoMode,
isVideoFrameDerivedMediaNode,
} from "@/utils/smartMediaMode";
import { deriveSmartTextMode } from "@/utils/smartTextMode";
import {
@ -127,10 +128,13 @@ export function getNodeConnectionSpec(node: SpecNode, edges: WorkflowEdge[]): No
const acceptedInputs = getAcceptedInputs(nodeType, node.data);
const derived = getDerivedMediaInfo(node.data);
if (derived && !isHighDefinitionDerivedImageNode(node.data)) {
const derivedAccepts: ConnectionInputGroup[] = isVideoFrameDerivedMediaNode(node.data)
? ["video"]
: acceptedInputs;
return makeSpec({
mode: "derived",
inputVisible: true,
accepts: acceptedInputs,
accepts: derivedAccepts,
outputVisible: Boolean(outputType),
produces: outputType,
sourceNodeId: derived.sourceNodeId,

26
src/utils/smartMediaMode.ts

@ -1,6 +1,5 @@
import type { WorkflowEdge } from "@/types";
import { isHighDefinitionDerivedImageNode } from "@/utils/highDefinitionNodes";
import { isVideoCropDerivedNode } from "@/utils/videoCropNodes";
import { SINGLE_INPUT_HANDLE_ID } from "@/utils/nodeHandles";
export type MediaSmartMode = "empty" | "source" | "generate";
@ -14,6 +13,8 @@ export const SMART_IMAGE_LOCAL_DERIVED_TASK_OPERATIONS: SmartImageAutoTaskOperat
export const SMART_IMAGE_AI_DERIVED_TASK_OPERATIONS: SmartImageAutoTaskOperation[] = ["multiAngle", "highDefinition", "outpainting", "inpainting"];
export const SMART_IMAGE_AI_STATIC_DERIVED_TASK_OPERATIONS: SmartImageAutoTaskOperation[] = ["multiAngle", "outpainting", "inpainting"];
export const SMART_IMAGE_ONE_SHOT_GENERATION_OPERATIONS: SmartImageAutoTaskOperation[] = ["inpainting"];
export type SmartVideoAutoTaskOperation = "crop" | "smartErase" | "boxErase";
export const SMART_VIDEO_AUTO_TASK_OPERATIONS: SmartVideoAutoTaskOperation[] = ["crop", "smartErase", "boxErase"];
export interface SmartMediaSpec<T> {
hasContent: (data: T) => boolean;
@ -102,6 +103,25 @@ export function isSmartImageOneShotGenerationNode(data: unknown): boolean {
return operation !== null && SMART_IMAGE_ONE_SHOT_GENERATION_OPERATIONS.includes(operation);
}
export function getSmartVideoAutoTaskOperation(data: unknown): SmartVideoAutoTaskOperation | null {
const record = asRecord(data);
const derivedVideo = asRecord(record.derivedVideo);
const operation = derivedVideo.operation;
return SMART_VIDEO_AUTO_TASK_OPERATIONS.includes(operation as SmartVideoAutoTaskOperation)
? operation as SmartVideoAutoTaskOperation
: null;
}
export function isSmartVideoAutoTaskNode(data: unknown): boolean {
return getSmartVideoAutoTaskOperation(data) !== null;
}
export function isVideoFrameDerivedMediaNode(data: unknown): boolean {
const record = asRecord(data);
const derivedMedia = asRecord(record.derivedMedia);
return derivedMedia.operation === "videoFrame";
}
export function hasSmartVideoGenerationArtifact(data: unknown): boolean {
const record = asRecord(data);
return record.status === "loading" ||
@ -117,14 +137,14 @@ export function hasSmartAudioGenerationArtifact(data: unknown): boolean {
}
const SMART_IMAGE_MODE_SPEC: SmartMediaSpec<unknown> = {
forceSource: (data) => isSmartImageAutoTaskNode(data) && !isHighDefinitionDerivedImageNode(data),
forceSource: (data) => (isSmartImageAutoTaskNode(data) || isVideoFrameDerivedMediaNode(data)) && !isHighDefinitionDerivedImageNode(data),
hasContent: (data) => Boolean(asRecord(data).image),
hasArtifact: hasSmartImageGenerationArtifact,
labels: { empty: "neutral", source: "image", generate: "generate" },
};
const SMART_VIDEO_MODE_SPEC: SmartMediaSpec<unknown> = {
forceGenerate: isVideoCropDerivedNode,
forceGenerate: isSmartVideoAutoTaskNode,
hasContent: (data) => Boolean(asRecord(data).video),
hasArtifact: hasSmartVideoGenerationArtifact,
labels: { empty: "neutral", source: "video", generate: "generate" },

30
src/utils/videoFrameNode.ts

@ -1,8 +1,8 @@
import type { NodeType, SmartImageNodeData, WorkflowNode } from "@/types";
import type { CapturedVideoFrame } from "@/utils/videoFrameCapture";
import type { Connection } from "@xyflow/react";
import { SINGLE_INPUT_HANDLE_ID, SINGLE_OUTPUT_HANDLE_ID } from "@/utils/nodeHandles";
import { uploadBlobToGatewayMedia } from "@/utils/gatewayMediaUpload";
export type VideoFrameCaptureMode = "first" | "last" | "current";
type AddNode = (
type: NodeType,
@ -17,32 +17,43 @@ function getNodeWidth(node: WorkflowNode): number {
export async function createVideoFrameSmartImageNode({
sourceNode,
frame,
time,
mode,
addNode,
onConnect,
selectSingleNode,
regenerateNode,
}: {
sourceNode: WorkflowNode;
frame: Pick<CapturedVideoFrame, "blob" | "dimensions">;
time: number;
mode: VideoFrameCaptureMode;
addNode: AddNode;
onConnect: (connection: Connection) => void;
selectSingleNode: (nodeId: string) => void;
regenerateNode?: (nodeId: string) => void | Promise<void>;
}): Promise<string> {
const filename = `video-frame-${Date.now()}.png`;
const uploaded = await uploadBlobToGatewayMedia(frame.blob, filename, "image/png", "image");
const sourcePosition = sourceNode.position ?? { x: 0, y: 0 };
const nodeId = addNode("smartImage", {
x: sourcePosition.x + getNodeWidth(sourceNode) + 220,
y: sourcePosition.y,
}, {
image: uploaded.url,
image: null,
imageRef: undefined,
assetId: undefined,
filename: uploaded.filename,
dimensions: frame.dimensions,
filename,
dimensions: null,
customTitle: "视频截帧",
status: "idle",
derivedMedia: {
sourceNodeId: sourceNode.id,
operation: "videoFrame",
task: {
time,
mode,
},
},
status: "loading",
error: null,
});
@ -54,5 +65,6 @@ export async function createVideoFrameSmartImageNode({
});
selectSingleNode(nodeId);
await regenerateNode?.(nodeId);
return nodeId;
}

Loading…
Cancel
Save