Browse Source

feat(quick-009): wire VideoFrameGrab into canvas, connection menu, and execution

- Register VideoFrameGrabNode in node exports, nodeTypes map, and minimap colors
- Add to ConnectionDropMenu as video target (Frame Grab) and image source
- Add executeVideoFrameGrab executor with canvas-based frame extraction
- Wire into all three execution paths: executeWorkflow, regenerateNode, executeSelectedNodes
- Add video connection validation for videoFrameGrab target type
- Add handle mapping for video->videoFrameGrab and videoFrameGrab->image connections
- Add videoFrameGrab case to quickstart validation defaults
handoff-20260429-1057
shrimbly 5 months ago
parent
commit
6e35b741c2
  1. 18
      src/components/ConnectionDropMenu.tsx
  2. 12
      src/components/WorkflowCanvas.tsx
  3. 1
      src/components/nodes/index.ts
  4. 7
      src/lib/quickstart/validation.ts
  5. 1
      src/store/execution/index.ts
  6. 96
      src/store/execution/videoProcessingExecutors.ts
  7. 12
      src/store/workflowStore.ts

18
src/components/ConnectionDropMenu.tsx

@ -167,6 +167,15 @@ const IMAGE_SOURCE_OPTIONS: MenuOption[] = [
</svg>
),
},
{
type: "videoFrameGrab",
label: "Frame Grab",
icon: (
<svg className="w-4 h-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={1.5}>
<path strokeLinecap="round" strokeLinejoin="round" d="M2.25 15.75l5.159-5.159a2.25 2.25 0 013.182 0l5.159 5.159m-1.5-1.5l1.409-1.409a2.25 2.25 0 013.182 0l2.909 2.909M10.5 6h9.75m-9.75 0a1.5 1.5 0 01-3 0m3 0a1.5 1.5 0 00-3 0M3.75 6H7.5m3 12h9.75m-9.75 0a1.5 1.5 0 01-3 0m3 0a1.5 1.5 0 00-3 0m-3.75 0H7.5" />
</svg>
),
},
{
type: "annotation",
label: "Annotate",
@ -246,6 +255,15 @@ const VIDEO_TARGET_OPTIONS: MenuOption[] = [
</svg>
),
},
{
type: "videoFrameGrab",
label: "Frame Grab",
icon: (
<svg className="w-4 h-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={1.5}>
<path strokeLinecap="round" strokeLinejoin="round" d="M2.25 15.75l5.159-5.159a2.25 2.25 0 013.182 0l5.159 5.159m-1.5-1.5l1.409-1.409a2.25 2.25 0 013.182 0l2.909 2.909M10.5 6h9.75m-9.75 0a1.5 1.5 0 01-3 0m3 0a1.5 1.5 0 00-3 0M3.75 6H7.5m3 12h9.75m-9.75 0a1.5 1.5 0 01-3 0m3 0a1.5 1.5 0 00-3 0m-3.75 0H7.5" />
</svg>
),
},
{
type: "generateVideo",
label: "Generate Video",

12
src/components/WorkflowCanvas.tsx

@ -38,6 +38,7 @@ import {
VideoStitchNode,
EaseCurveNode,
VideoTrimNode,
VideoFrameGrabNode,
} from "./nodes";
// Lazy-load GLBViewerNode to avoid bundling three.js for users who don't use 3D nodes
@ -76,6 +77,7 @@ const nodeTypes: NodeTypes = {
videoStitch: VideoStitchNode,
easeCurve: EaseCurveNode,
videoTrim: VideoTrimNode,
videoFrameGrab: VideoFrameGrabNode,
glbViewer: GLBViewerNode,
};
@ -144,6 +146,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[]
return { inputs: ["video", "easeCurve"], outputs: ["video", "easeCurve"] };
case "videoTrim":
return { inputs: ["video"], outputs: ["video"] };
case "videoFrameGrab":
return { inputs: ["video"], outputs: ["image"] };
case "glbViewer":
return { inputs: ["3d"], outputs: ["image"] };
default:
@ -336,7 +340,7 @@ export function WorkflowCanvas() {
if (!targetNode) return false;
const targetNodeType = targetNode.type;
if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "output") {
if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "videoFrameGrab" || targetNodeType === "output") {
// For output node, we allow video even though its handle is typed as "image"
// because output node can display both images and videos
return true;
@ -849,6 +853,10 @@ export function WorkflowCanvas() {
// VideoTrim accepts video input and outputs video
targetHandleId = "video";
sourceHandleIdForNewNode = "video";
} else if (nodeType === "videoFrameGrab") {
// VideoFrameGrab accepts video input and outputs image
targetHandleId = "video";
sourceHandleIdForNewNode = "image";
} else if (nodeType === "generateVideo") {
// GenerateVideo outputs video
sourceHandleIdForNewNode = "video";
@ -1725,6 +1733,8 @@ export function WorkflowCanvas() {
return "#bef264"; // lime-300 (easy-peasy-ease)
case "videoTrim":
return "#60a5fa"; // blue-400 (trim/cut)
case "videoFrameGrab":
return "#38bdf8"; // sky-400 (image from video)
case "glbViewer":
return "#38bdf8"; // sky-400 (3D viewport)
default:

1
src/components/nodes/index.ts

@ -15,4 +15,5 @@ export { ImageCompareNode } from "./ImageCompareNode";
export { VideoStitchNode } from "./VideoStitchNode";
export { EaseCurveNode } from "./EaseCurveNode";
export { VideoTrimNode } from "./VideoTrimNode";
export { VideoFrameGrabNode } from "./VideoFrameGrabNode";
export { GroupNode } from "./GroupNode";

7
src/lib/quickstart/validation.ts

@ -356,6 +356,13 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData {
progress: 0,
encoderSupported: null,
};
case "videoFrameGrab":
return {
framePosition: "first",
outputImage: null,
status: "idle",
error: null,
};
case "glbViewer":
return {
glbUrl: null,

1
src/store/execution/index.ts

@ -39,4 +39,5 @@ export {
executeVideoStitch,
executeEaseCurve,
executeVideoTrim,
executeVideoFrameGrab,
} from "./videoProcessingExecutors";

96
src/store/execution/videoProcessingExecutors.ts

@ -5,7 +5,7 @@
* Used by both executeWorkflow and regenerateNode.
*/
import type { VideoStitchNodeData, EaseCurveNodeData, VideoTrimNodeData } from "@/types";
import type { VideoStitchNodeData, EaseCurveNodeData, VideoTrimNodeData, VideoFrameGrabNodeData } from "@/types";
import { revokeBlobUrl } from "@/store/utils/executionUtils";
import type { NodeExecutionContext } from "./types";
@ -342,3 +342,97 @@ export async function executeEaseCurve(ctx: NodeExecutionContext): Promise<void>
throw err instanceof Error ? err : new Error(errorMessage);
}
}
/**
* VideoFrameGrab: extracts the first or last frame from a video as a full-resolution PNG image.
*/
export async function executeVideoFrameGrab(ctx: NodeExecutionContext): Promise<void> {
const { node, getConnectedInputs, updateNodeData } = ctx;
const nodeData = node.data as VideoFrameGrabNodeData;
updateNodeData(node.id, { status: "loading", error: null });
try {
const inputs = getConnectedInputs(node.id);
if (inputs.videos.length === 0) {
updateNodeData(node.id, {
status: "error",
error: "Connect a video input to extract a frame",
});
throw new Error("Connect a video input to extract a frame");
}
const videoUrl = inputs.videos[0];
// Create video element and seek to target frame
const video = document.createElement("video");
video.crossOrigin = "anonymous";
video.preload = "auto";
const outputImage = await new Promise<string>((resolve, reject) => {
video.onloadedmetadata = () => {
// For "first" frame, seek to 0.001 (not exactly 0 to ensure a decoded frame)
// For "last" frame, seek to duration - small epsilon
const seekTime = nodeData.framePosition === "first"
? 0.001
: Math.max(0, video.duration - 0.1);
video.currentTime = seekTime;
};
video.onseeked = () => {
try {
const canvas = document.createElement("canvas");
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const ctx2d = canvas.getContext("2d");
if (!ctx2d) {
reject(new Error("Could not get canvas 2d context"));
return;
}
ctx2d.drawImage(video, 0, 0, canvas.width, canvas.height);
const frameDataUrl = canvas.toDataURL("image/png");
resolve(frameDataUrl);
} catch (err) {
reject(err instanceof Error ? err : new Error("Frame extraction failed"));
}
};
video.onerror = () => {
reject(new Error("Failed to load video for frame extraction"));
};
// If data URL, convert to blob URL for better performance
if (videoUrl.startsWith("data:")) {
fetch(videoUrl)
.then((r) => r.blob())
.then((blob) => {
video.src = URL.createObjectURL(blob);
})
.catch(() => {
video.src = videoUrl;
});
} else {
video.src = videoUrl;
}
});
// Cleanup blob URL if we created one
if (video.src.startsWith("blob:")) {
URL.revokeObjectURL(video.src);
}
updateNodeData(node.id, {
outputImage,
status: "complete",
error: null,
});
} catch (err) {
const errorMessage = err instanceof Error ? err.message : "Frame extraction failed";
updateNodeData(node.id, {
status: "error",
error: errorMessage,
});
throw err instanceof Error ? err : new Error(errorMessage);
}
}

12
src/store/workflowStore.ts

@ -74,6 +74,7 @@ import {
executeVideoStitch,
executeEaseCurve,
executeVideoTrim,
executeVideoFrameGrab,
executeGlbViewer,
} from "./execution";
import type { NodeExecutionContext } from "./execution";
@ -940,6 +941,9 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
case "easeCurve":
await executeEaseCurve(executionCtx);
break;
case "videoFrameGrab":
await executeVideoFrameGrab(executionCtx);
break;
}
}; // End of executeSingleNode helper
@ -1093,6 +1097,11 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
set({ isRunning: false, currentNodeIds: [] });
await logger.endSession();
return;
} else if (node.type === "videoFrameGrab") {
await executeVideoFrameGrab(executionCtx);
set({ isRunning: false, currentNodeIds: [] });
await logger.endSession();
return;
}
// After regeneration, execute directly connected downstream consumer nodes
@ -1237,6 +1246,9 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
case "videoTrim":
await executeVideoTrim(executionCtx);
break;
case "videoFrameGrab":
await executeVideoFrameGrab(executionCtx);
break;
}
};

Loading…
Cancel
Save