From 62244c52c246b4aea6f6b1bee047a093de23c4b1 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Tue, 3 Feb 2026 21:07:40 +1300 Subject: [PATCH] feat(41-01): install MediaBunny and define AudioInput/VideoStitch node types - Install mediabunny@^1.28.0 for video processing - Add audioInput and videoStitch to NodeType union - Add AudioInputNodeData interface (audioFile, filename, duration, format) - Add VideoStitchClip interface for filmstrip items - Add VideoStitchNodeData interface (clips, clipOrder, outputVideo, progress, encoderSupported) - Update WorkflowNodeData union to include new node types - Extend HandleType to include audio and video handles - Create src/lib/video-encoding.ts with AVC encoding config helpers --- package-lock.json | 33 +++++++++++++++++++++++++++++ package.json | 1 + src/lib/video-encoding.ts | 39 ++++++++++++++++++++++++++++++++++ src/types/nodes.ts | 44 ++++++++++++++++++++++++++++++++++++--- 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 src/lib/video-encoding.ts diff --git a/package-lock.json b/package-lock.json index f29cb08c..45b8bd83 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "autoprefixer": "^10.4.22", "jszip": "^3.10.1", "konva": "^10.0.12", + "mediabunny": "^1.31.0", "next": "^16.0.7", "postcss": "^8.5.6", "react": "^19.2.0", @@ -2729,6 +2730,21 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/dom-mediacapture-transform": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.11.tgz", + "integrity": "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==", + "license": "MIT", + "dependencies": { + "@types/dom-webcodecs": "*" + } + }, + "node_modules/@types/dom-webcodecs": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz", + "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -5010,6 +5026,23 @@ "dev": true, "license": "CC0-1.0" }, + "node_modules/mediabunny": { + "version": "1.31.0", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.31.0.tgz", + "integrity": "sha512-nqM+6cOpNC/aDxCAZKnZe7oXnGaCn4rlgprzAmiH6C8GRdOHFnB6bZC0+WXGTT6mtAxXQd+BXuZ2q2zkma7dWg==", + "license": "MPL-2.0", + "workspaces": [ + "packages/*" + ], + "dependencies": { + "@types/dom-mediacapture-transform": "^0.1.11", + "@types/dom-webcodecs": "0.1.13" + }, + "funding": { + "type": "individual", + "url": "https://github.com/sponsors/Vanilagy" + } + }, "node_modules/micromark": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", diff --git a/package.json b/package.json index 1aeefda8..007438c0 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "autoprefixer": "^10.4.22", "jszip": "^3.10.1", "konva": "^10.0.12", + "mediabunny": "^1.31.0", "next": "^16.0.7", "postcss": "^8.5.6", "react": "^19.2.0", diff --git a/src/lib/video-encoding.ts b/src/lib/video-encoding.ts new file mode 100644 index 00000000..8356929d --- /dev/null +++ b/src/lib/video-encoding.ts @@ -0,0 +1,39 @@ +import type { VideoEncodingConfig } from 'mediabunny'; + +// Inlined constants from easy-peasy-ease +const DEFAULT_KEYFRAME_INTERVAL = 1.0; +const MAX_OUTPUT_FPS = 60; + +export const AVC_LEVEL_4_0 = 'avc1.420028'; +export const AVC_LEVEL_5_1 = 'avc1.640033'; + +export const createAvcEncodingConfig = ( + bitrate: number, + width?: number, + height?: number, + codecString: string = AVC_LEVEL_4_0, + framerate?: number, + useHardwareAcceleration: boolean = true +): VideoEncodingConfig => ({ + codec: 'avc', + bitrate, + keyFrameInterval: DEFAULT_KEYFRAME_INTERVAL, + bitrateMode: 'variable', + latencyMode: 'quality', + fullCodecString: codecString, + hardwareAcceleration: useHardwareAcceleration ? 'prefer-hardware' : 'prefer-software', + onEncoderConfig: (config) => { + config.avc = { ...(config.avc ?? {}), format: 'avc' }; + if (!config.latencyMode) { + config.latencyMode = 'quality'; + } + if (framerate && framerate > 0) { + config.framerate = framerate; + } else if (!config.framerate) { + config.framerate = MAX_OUTPUT_FPS; + } + config.bitrate = bitrate; + if (width) config.width = width; + if (height) config.height = height; + }, +}); diff --git a/src/types/nodes.ts b/src/types/nodes.ts index 9ae4c58c..aa72440b 100644 --- a/src/types/nodes.ts +++ b/src/types/nodes.ts @@ -24,6 +24,7 @@ import type { LLMProvider, LLMModelType, SelectedModel, ProviderType } from "./p */ export type NodeType = | "imageInput" + | "audioInput" | "annotation" | "prompt" | "promptConstructor" @@ -33,7 +34,8 @@ export type NodeType = | "splitGrid" | "output" | "outputGallery" - | "imageCompare"; + | "imageCompare" + | "videoStitch"; /** * Node execution status @@ -50,6 +52,16 @@ export interface ImageInputNodeData extends BaseNodeData { dimensions: { width: number; height: number } | null; } +/** + * Audio input node - loads/uploads audio files into the workflow + */ +export interface AudioInputNodeData extends BaseNodeData { + audioFile: string | null; // Base64 data URL of the audio file + filename: string | null; // Original filename for display + duration: number | null; // Duration in seconds + format: string | null; // MIME type (audio/mp3, audio/wav, etc.) +} + /** * Prompt node - text input for AI generation */ @@ -202,6 +214,30 @@ export interface ImageCompareNodeData extends BaseNodeData { imageB: string | null; } +/** + * Video stitch clip - represents a single video clip in the filmstrip + */ +export interface VideoStitchClip { + edgeId: string; // Edge ID for disconnect capability + sourceNodeId: string; // Source node producing this video + thumbnail: string | null; // Base64 JPEG thumbnail + duration: number | null; // Clip duration in seconds + handleId: string; // Which input handle (video-0, video-1, etc.) +} + +/** + * Video Stitch node - concatenates multiple videos into a single output + */ +export interface VideoStitchNodeData extends BaseNodeData { + clips: VideoStitchClip[]; // Ordered clip sequence for filmstrip + clipOrder: string[]; // Edge IDs in user-defined order (drag reorder) + outputVideo: string | null; // Stitched video blob URL or data URL + status: NodeStatus; + error: string | null; + progress: number; // 0-100 processing progress + encoderSupported: boolean | null; // null = not checked yet, true/false after check +} + /** * Split Grid node - splits image into grid cells for parallel processing */ @@ -233,6 +269,7 @@ export interface SplitGridNodeData extends BaseNodeData { */ export type WorkflowNodeData = | ImageInputNodeData + | AudioInputNodeData | AnnotationNodeData | PromptNodeData | PromptConstructorNodeData @@ -242,7 +279,8 @@ export type WorkflowNodeData = | SplitGridNodeData | OutputNodeData | OutputGalleryNodeData - | ImageCompareNodeData; + | ImageCompareNodeData + | VideoStitchNodeData; /** * Workflow node with typed data (extended with optional groupId) @@ -254,7 +292,7 @@ export type WorkflowNode = Node & { /** * Handle types for node connections */ -export type HandleType = "image" | "text"; +export type HandleType = "image" | "text" | "audio" | "video"; /** * Default settings for node types - stored in localStorage