Browse Source

feat(41-01): install MediaBunny and define AudioInput/VideoStitch node types

- Install mediabunny@^1.28.0 for video processing
- Add audioInput and videoStitch to NodeType union
- Add AudioInputNodeData interface (audioFile, filename, duration, format)
- Add VideoStitchClip interface for filmstrip items
- Add VideoStitchNodeData interface (clips, clipOrder, outputVideo, progress, encoderSupported)
- Update WorkflowNodeData union to include new node types
- Extend HandleType to include audio and video handles
- Create src/lib/video-encoding.ts with AVC encoding config helpers
handoff-20260429-1057
shrimbly 5 months ago
parent
commit
62244c52c2
  1. 33
      package-lock.json
  2. 1
      package.json
  3. 39
      src/lib/video-encoding.ts
  4. 44
      src/types/nodes.ts

33
package-lock.json

@ -17,6 +17,7 @@
"autoprefixer": "^10.4.22", "autoprefixer": "^10.4.22",
"jszip": "^3.10.1", "jszip": "^3.10.1",
"konva": "^10.0.12", "konva": "^10.0.12",
"mediabunny": "^1.31.0",
"next": "^16.0.7", "next": "^16.0.7",
"postcss": "^8.5.6", "postcss": "^8.5.6",
"react": "^19.2.0", "react": "^19.2.0",
@ -2729,6 +2730,21 @@
"dev": true, "dev": true,
"license": "MIT" "license": "MIT"
}, },
"node_modules/@types/dom-mediacapture-transform": {
"version": "0.1.11",
"resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.11.tgz",
"integrity": "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==",
"license": "MIT",
"dependencies": {
"@types/dom-webcodecs": "*"
}
},
"node_modules/@types/dom-webcodecs": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz",
"integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==",
"license": "MIT"
},
"node_modules/@types/estree": { "node_modules/@types/estree": {
"version": "1.0.8", "version": "1.0.8",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
@ -5010,6 +5026,23 @@
"dev": true, "dev": true,
"license": "CC0-1.0" "license": "CC0-1.0"
}, },
"node_modules/mediabunny": {
"version": "1.31.0",
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.31.0.tgz",
"integrity": "sha512-nqM+6cOpNC/aDxCAZKnZe7oXnGaCn4rlgprzAmiH6C8GRdOHFnB6bZC0+WXGTT6mtAxXQd+BXuZ2q2zkma7dWg==",
"license": "MPL-2.0",
"workspaces": [
"packages/*"
],
"dependencies": {
"@types/dom-mediacapture-transform": "^0.1.11",
"@types/dom-webcodecs": "0.1.13"
},
"funding": {
"type": "individual",
"url": "https://github.com/sponsors/Vanilagy"
}
},
"node_modules/micromark": { "node_modules/micromark": {
"version": "4.0.2", "version": "4.0.2",
"resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz",

1
package.json

@ -21,6 +21,7 @@
"autoprefixer": "^10.4.22", "autoprefixer": "^10.4.22",
"jszip": "^3.10.1", "jszip": "^3.10.1",
"konva": "^10.0.12", "konva": "^10.0.12",
"mediabunny": "^1.31.0",
"next": "^16.0.7", "next": "^16.0.7",
"postcss": "^8.5.6", "postcss": "^8.5.6",
"react": "^19.2.0", "react": "^19.2.0",

39
src/lib/video-encoding.ts

@ -0,0 +1,39 @@
import type { VideoEncodingConfig } from 'mediabunny';
// Inlined constants from easy-peasy-ease
const DEFAULT_KEYFRAME_INTERVAL = 1.0;
const MAX_OUTPUT_FPS = 60;
export const AVC_LEVEL_4_0 = 'avc1.420028';
export const AVC_LEVEL_5_1 = 'avc1.640033';
export const createAvcEncodingConfig = (
bitrate: number,
width?: number,
height?: number,
codecString: string = AVC_LEVEL_4_0,
framerate?: number,
useHardwareAcceleration: boolean = true
): VideoEncodingConfig => ({
codec: 'avc',
bitrate,
keyFrameInterval: DEFAULT_KEYFRAME_INTERVAL,
bitrateMode: 'variable',
latencyMode: 'quality',
fullCodecString: codecString,
hardwareAcceleration: useHardwareAcceleration ? 'prefer-hardware' : 'prefer-software',
onEncoderConfig: (config) => {
config.avc = { ...(config.avc ?? {}), format: 'avc' };
if (!config.latencyMode) {
config.latencyMode = 'quality';
}
if (framerate && framerate > 0) {
config.framerate = framerate;
} else if (!config.framerate) {
config.framerate = MAX_OUTPUT_FPS;
}
config.bitrate = bitrate;
if (width) config.width = width;
if (height) config.height = height;
},
});

44
src/types/nodes.ts

@ -24,6 +24,7 @@ import type { LLMProvider, LLMModelType, SelectedModel, ProviderType } from "./p
*/ */
export type NodeType = export type NodeType =
| "imageInput" | "imageInput"
| "audioInput"
| "annotation" | "annotation"
| "prompt" | "prompt"
| "promptConstructor" | "promptConstructor"
@ -33,7 +34,8 @@ export type NodeType =
| "splitGrid" | "splitGrid"
| "output" | "output"
| "outputGallery" | "outputGallery"
| "imageCompare"; | "imageCompare"
| "videoStitch";
/** /**
* Node execution status * Node execution status
@ -50,6 +52,16 @@ export interface ImageInputNodeData extends BaseNodeData {
dimensions: { width: number; height: number } | null; dimensions: { width: number; height: number } | null;
} }
/**
* Audio input node - loads/uploads audio files into the workflow
*/
export interface AudioInputNodeData extends BaseNodeData {
audioFile: string | null; // Base64 data URL of the audio file
filename: string | null; // Original filename for display
duration: number | null; // Duration in seconds
format: string | null; // MIME type (audio/mp3, audio/wav, etc.)
}
/** /**
* Prompt node - text input for AI generation * Prompt node - text input for AI generation
*/ */
@ -202,6 +214,30 @@ export interface ImageCompareNodeData extends BaseNodeData {
imageB: string | null; imageB: string | null;
} }
/**
* Video stitch clip - represents a single video clip in the filmstrip
*/
export interface VideoStitchClip {
edgeId: string; // Edge ID for disconnect capability
sourceNodeId: string; // Source node producing this video
thumbnail: string | null; // Base64 JPEG thumbnail
duration: number | null; // Clip duration in seconds
handleId: string; // Which input handle (video-0, video-1, etc.)
}
/**
* Video Stitch node - concatenates multiple videos into a single output
*/
export interface VideoStitchNodeData extends BaseNodeData {
clips: VideoStitchClip[]; // Ordered clip sequence for filmstrip
clipOrder: string[]; // Edge IDs in user-defined order (drag reorder)
outputVideo: string | null; // Stitched video blob URL or data URL
status: NodeStatus;
error: string | null;
progress: number; // 0-100 processing progress
encoderSupported: boolean | null; // null = not checked yet, true/false after check
}
/** /**
* Split Grid node - splits image into grid cells for parallel processing * Split Grid node - splits image into grid cells for parallel processing
*/ */
@ -233,6 +269,7 @@ export interface SplitGridNodeData extends BaseNodeData {
*/ */
export type WorkflowNodeData = export type WorkflowNodeData =
| ImageInputNodeData | ImageInputNodeData
| AudioInputNodeData
| AnnotationNodeData | AnnotationNodeData
| PromptNodeData | PromptNodeData
| PromptConstructorNodeData | PromptConstructorNodeData
@ -242,7 +279,8 @@ export type WorkflowNodeData =
| SplitGridNodeData | SplitGridNodeData
| OutputNodeData | OutputNodeData
| OutputGalleryNodeData | OutputGalleryNodeData
| ImageCompareNodeData; | ImageCompareNodeData
| VideoStitchNodeData;
/** /**
* Workflow node with typed data (extended with optional groupId) * Workflow node with typed data (extended with optional groupId)
@ -254,7 +292,7 @@ export type WorkflowNode = Node<WorkflowNodeData, NodeType> & {
/** /**
* Handle types for node connections * Handle types for node connections
*/ */
export type HandleType = "image" | "text"; export type HandleType = "image" | "text" | "audio" | "video";
/** /**
* Default settings for node types - stored in localStorage * Default settings for node types - stored in localStorage

Loading…
Cancel
Save