diff --git a/src/components/ConnectionDropMenu.tsx b/src/components/ConnectionDropMenu.tsx index c464751d..d4bfbffe 100644 --- a/src/components/ConnectionDropMenu.tsx +++ b/src/components/ConnectionDropMenu.tsx @@ -167,6 +167,15 @@ const IMAGE_SOURCE_OPTIONS: MenuOption[] = [ ), }, + { + type: "videoFrameGrab", + label: "Frame Grab", + icon: ( + + + + ), + }, { type: "annotation", label: "Annotate", @@ -246,6 +255,15 @@ const VIDEO_TARGET_OPTIONS: MenuOption[] = [ ), }, + { + type: "videoFrameGrab", + label: "Frame Grab", + icon: ( + + + + ), + }, { type: "generateVideo", label: "Generate Video", diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 6b247e67..207ca493 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -38,6 +38,7 @@ import { VideoStitchNode, EaseCurveNode, VideoTrimNode, + VideoFrameGrabNode, } from "./nodes"; // Lazy-load GLBViewerNode to avoid bundling three.js for users who don't use 3D nodes @@ -76,6 +77,7 @@ const nodeTypes: NodeTypes = { videoStitch: VideoStitchNode, easeCurve: EaseCurveNode, videoTrim: VideoTrimNode, + videoFrameGrab: VideoFrameGrabNode, glbViewer: GLBViewerNode, }; @@ -144,6 +146,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] return { inputs: ["video", "easeCurve"], outputs: ["video", "easeCurve"] }; case "videoTrim": return { inputs: ["video"], outputs: ["video"] }; + case "videoFrameGrab": + return { inputs: ["video"], outputs: ["image"] }; case "glbViewer": return { inputs: ["3d"], outputs: ["image"] }; default: @@ -336,7 +340,7 @@ export function WorkflowCanvas() { if (!targetNode) return false; const targetNodeType = targetNode.type; - if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "output") { + if (targetNodeType === "generateVideo" || targetNodeType === "videoStitch" || targetNodeType === "easeCurve" || targetNodeType === "videoTrim" || targetNodeType === "videoFrameGrab" || targetNodeType === "output") { // For output node, we allow video even though its handle is typed as "image" // because output node can display both images and videos return true; @@ -849,6 +853,10 @@ export function WorkflowCanvas() { // VideoTrim accepts video input and outputs video targetHandleId = "video"; sourceHandleIdForNewNode = "video"; + } else if (nodeType === "videoFrameGrab") { + // VideoFrameGrab accepts video input and outputs image + targetHandleId = "video"; + sourceHandleIdForNewNode = "image"; } else if (nodeType === "generateVideo") { // GenerateVideo outputs video sourceHandleIdForNewNode = "video"; @@ -1725,6 +1733,8 @@ export function WorkflowCanvas() { return "#bef264"; // lime-300 (easy-peasy-ease) case "videoTrim": return "#60a5fa"; // blue-400 (trim/cut) + case "videoFrameGrab": + return "#38bdf8"; // sky-400 (image from video) case "glbViewer": return "#38bdf8"; // sky-400 (3D viewport) default: diff --git a/src/components/nodes/index.ts b/src/components/nodes/index.ts index 8d580db5..dca9b415 100644 --- a/src/components/nodes/index.ts +++ b/src/components/nodes/index.ts @@ -15,4 +15,5 @@ export { ImageCompareNode } from "./ImageCompareNode"; export { VideoStitchNode } from "./VideoStitchNode"; export { EaseCurveNode } from "./EaseCurveNode"; export { VideoTrimNode } from "./VideoTrimNode"; +export { VideoFrameGrabNode } from "./VideoFrameGrabNode"; export { GroupNode } from "./GroupNode"; diff --git a/src/lib/quickstart/validation.ts b/src/lib/quickstart/validation.ts index 14eb9630..44591073 100644 --- a/src/lib/quickstart/validation.ts +++ b/src/lib/quickstart/validation.ts @@ -356,6 +356,13 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData { progress: 0, encoderSupported: null, }; + case "videoFrameGrab": + return { + framePosition: "first", + outputImage: null, + status: "idle", + error: null, + }; case "glbViewer": return { glbUrl: null, diff --git a/src/store/execution/index.ts b/src/store/execution/index.ts index 92667219..80fee3b7 100644 --- a/src/store/execution/index.ts +++ b/src/store/execution/index.ts @@ -39,4 +39,5 @@ export { executeVideoStitch, executeEaseCurve, executeVideoTrim, + executeVideoFrameGrab, } from "./videoProcessingExecutors"; diff --git a/src/store/execution/videoProcessingExecutors.ts b/src/store/execution/videoProcessingExecutors.ts index 78b66143..da5d21f1 100644 --- a/src/store/execution/videoProcessingExecutors.ts +++ b/src/store/execution/videoProcessingExecutors.ts @@ -5,7 +5,7 @@ * Used by both executeWorkflow and regenerateNode. */ -import type { VideoStitchNodeData, EaseCurveNodeData, VideoTrimNodeData } from "@/types"; +import type { VideoStitchNodeData, EaseCurveNodeData, VideoTrimNodeData, VideoFrameGrabNodeData } from "@/types"; import { revokeBlobUrl } from "@/store/utils/executionUtils"; import type { NodeExecutionContext } from "./types"; @@ -342,3 +342,97 @@ export async function executeEaseCurve(ctx: NodeExecutionContext): Promise throw err instanceof Error ? err : new Error(errorMessage); } } + +/** + * VideoFrameGrab: extracts the first or last frame from a video as a full-resolution PNG image. + */ +export async function executeVideoFrameGrab(ctx: NodeExecutionContext): Promise { + const { node, getConnectedInputs, updateNodeData } = ctx; + const nodeData = node.data as VideoFrameGrabNodeData; + + updateNodeData(node.id, { status: "loading", error: null }); + + try { + const inputs = getConnectedInputs(node.id); + + if (inputs.videos.length === 0) { + updateNodeData(node.id, { + status: "error", + error: "Connect a video input to extract a frame", + }); + throw new Error("Connect a video input to extract a frame"); + } + + const videoUrl = inputs.videos[0]; + + // Create video element and seek to target frame + const video = document.createElement("video"); + video.crossOrigin = "anonymous"; + video.preload = "auto"; + + const outputImage = await new Promise((resolve, reject) => { + video.onloadedmetadata = () => { + // For "first" frame, seek to 0.001 (not exactly 0 to ensure a decoded frame) + // For "last" frame, seek to duration - small epsilon + const seekTime = nodeData.framePosition === "first" + ? 0.001 + : Math.max(0, video.duration - 0.1); + video.currentTime = seekTime; + }; + + video.onseeked = () => { + try { + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + const ctx2d = canvas.getContext("2d"); + if (!ctx2d) { + reject(new Error("Could not get canvas 2d context")); + return; + } + ctx2d.drawImage(video, 0, 0, canvas.width, canvas.height); + const frameDataUrl = canvas.toDataURL("image/png"); + resolve(frameDataUrl); + } catch (err) { + reject(err instanceof Error ? err : new Error("Frame extraction failed")); + } + }; + + video.onerror = () => { + reject(new Error("Failed to load video for frame extraction")); + }; + + // If data URL, convert to blob URL for better performance + if (videoUrl.startsWith("data:")) { + fetch(videoUrl) + .then((r) => r.blob()) + .then((blob) => { + video.src = URL.createObjectURL(blob); + }) + .catch(() => { + video.src = videoUrl; + }); + } else { + video.src = videoUrl; + } + }); + + // Cleanup blob URL if we created one + if (video.src.startsWith("blob:")) { + URL.revokeObjectURL(video.src); + } + + updateNodeData(node.id, { + outputImage, + status: "complete", + error: null, + }); + } catch (err) { + const errorMessage = err instanceof Error ? err.message : "Frame extraction failed"; + updateNodeData(node.id, { + status: "error", + error: errorMessage, + }); + throw err instanceof Error ? err : new Error(errorMessage); + } +} diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index 79343ac7..204eee26 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -74,6 +74,7 @@ import { executeVideoStitch, executeEaseCurve, executeVideoTrim, + executeVideoFrameGrab, executeGlbViewer, } from "./execution"; import type { NodeExecutionContext } from "./execution"; @@ -940,6 +941,9 @@ export const useWorkflowStore = create((set, get) => ({ case "easeCurve": await executeEaseCurve(executionCtx); break; + case "videoFrameGrab": + await executeVideoFrameGrab(executionCtx); + break; } }; // End of executeSingleNode helper @@ -1093,6 +1097,11 @@ export const useWorkflowStore = create((set, get) => ({ set({ isRunning: false, currentNodeIds: [] }); await logger.endSession(); return; + } else if (node.type === "videoFrameGrab") { + await executeVideoFrameGrab(executionCtx); + set({ isRunning: false, currentNodeIds: [] }); + await logger.endSession(); + return; } // After regeneration, execute directly connected downstream consumer nodes @@ -1237,6 +1246,9 @@ export const useWorkflowStore = create((set, get) => ({ case "videoTrim": await executeVideoTrim(executionCtx); break; + case "videoFrameGrab": + await executeVideoFrameGrab(executionCtx); + break; } };