Browse Source

feat(41-01): install MediaBunny and define AudioInput/VideoStitch node types

- Install mediabunny@^1.28.0 for video processing
- Add audioInput and videoStitch to NodeType union
- Add AudioInputNodeData interface (audioFile, filename, duration, format)
- Add VideoStitchClip interface for filmstrip items
- Add VideoStitchNodeData interface (clips, clipOrder, outputVideo, progress, encoderSupported)
- Update WorkflowNodeData union to include new node types
- Extend HandleType to include audio and video handles
- Create src/lib/video-encoding.ts with AVC encoding config helpers
handoff-20260429-1057
shrimbly 5 months ago
parent
commit
62244c52c2
  1. 33
      package-lock.json
  2. 1
      package.json
  3. 39
      src/lib/video-encoding.ts
  4. 44
      src/types/nodes.ts

33
package-lock.json

@ -17,6 +17,7 @@
"autoprefixer": "^10.4.22",
"jszip": "^3.10.1",
"konva": "^10.0.12",
"mediabunny": "^1.31.0",
"next": "^16.0.7",
"postcss": "^8.5.6",
"react": "^19.2.0",
@ -2729,6 +2730,21 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/dom-mediacapture-transform": {
"version": "0.1.11",
"resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.11.tgz",
"integrity": "sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==",
"license": "MIT",
"dependencies": {
"@types/dom-webcodecs": "*"
}
},
"node_modules/@types/dom-webcodecs": {
"version": "0.1.13",
"resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz",
"integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==",
"license": "MIT"
},
"node_modules/@types/estree": {
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
@ -5010,6 +5026,23 @@
"dev": true,
"license": "CC0-1.0"
},
"node_modules/mediabunny": {
"version": "1.31.0",
"resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.31.0.tgz",
"integrity": "sha512-nqM+6cOpNC/aDxCAZKnZe7oXnGaCn4rlgprzAmiH6C8GRdOHFnB6bZC0+WXGTT6mtAxXQd+BXuZ2q2zkma7dWg==",
"license": "MPL-2.0",
"workspaces": [
"packages/*"
],
"dependencies": {
"@types/dom-mediacapture-transform": "^0.1.11",
"@types/dom-webcodecs": "0.1.13"
},
"funding": {
"type": "individual",
"url": "https://github.com/sponsors/Vanilagy"
}
},
"node_modules/micromark": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz",

1
package.json

@ -21,6 +21,7 @@
"autoprefixer": "^10.4.22",
"jszip": "^3.10.1",
"konva": "^10.0.12",
"mediabunny": "^1.31.0",
"next": "^16.0.7",
"postcss": "^8.5.6",
"react": "^19.2.0",

39
src/lib/video-encoding.ts

@ -0,0 +1,39 @@
import type { VideoEncodingConfig } from 'mediabunny';
// Inlined constants from easy-peasy-ease
const DEFAULT_KEYFRAME_INTERVAL = 1.0;
const MAX_OUTPUT_FPS = 60;
export const AVC_LEVEL_4_0 = 'avc1.420028';
export const AVC_LEVEL_5_1 = 'avc1.640033';
export const createAvcEncodingConfig = (
bitrate: number,
width?: number,
height?: number,
codecString: string = AVC_LEVEL_4_0,
framerate?: number,
useHardwareAcceleration: boolean = true
): VideoEncodingConfig => ({
codec: 'avc',
bitrate,
keyFrameInterval: DEFAULT_KEYFRAME_INTERVAL,
bitrateMode: 'variable',
latencyMode: 'quality',
fullCodecString: codecString,
hardwareAcceleration: useHardwareAcceleration ? 'prefer-hardware' : 'prefer-software',
onEncoderConfig: (config) => {
config.avc = { ...(config.avc ?? {}), format: 'avc' };
if (!config.latencyMode) {
config.latencyMode = 'quality';
}
if (framerate && framerate > 0) {
config.framerate = framerate;
} else if (!config.framerate) {
config.framerate = MAX_OUTPUT_FPS;
}
config.bitrate = bitrate;
if (width) config.width = width;
if (height) config.height = height;
},
});

44
src/types/nodes.ts

@ -24,6 +24,7 @@ import type { LLMProvider, LLMModelType, SelectedModel, ProviderType } from "./p
*/
export type NodeType =
| "imageInput"
| "audioInput"
| "annotation"
| "prompt"
| "promptConstructor"
@ -33,7 +34,8 @@ export type NodeType =
| "splitGrid"
| "output"
| "outputGallery"
| "imageCompare";
| "imageCompare"
| "videoStitch";
/**
* Node execution status
@ -50,6 +52,16 @@ export interface ImageInputNodeData extends BaseNodeData {
dimensions: { width: number; height: number } | null;
}
/**
* Audio input node - loads/uploads audio files into the workflow
*/
export interface AudioInputNodeData extends BaseNodeData {
audioFile: string | null; // Base64 data URL of the audio file
filename: string | null; // Original filename for display
duration: number | null; // Duration in seconds
format: string | null; // MIME type (audio/mp3, audio/wav, etc.)
}
/**
* Prompt node - text input for AI generation
*/
@ -202,6 +214,30 @@ export interface ImageCompareNodeData extends BaseNodeData {
imageB: string | null;
}
/**
* Video stitch clip - represents a single video clip in the filmstrip
*/
export interface VideoStitchClip {
edgeId: string; // Edge ID for disconnect capability
sourceNodeId: string; // Source node producing this video
thumbnail: string | null; // Base64 JPEG thumbnail
duration: number | null; // Clip duration in seconds
handleId: string; // Which input handle (video-0, video-1, etc.)
}
/**
* Video Stitch node - concatenates multiple videos into a single output
*/
export interface VideoStitchNodeData extends BaseNodeData {
clips: VideoStitchClip[]; // Ordered clip sequence for filmstrip
clipOrder: string[]; // Edge IDs in user-defined order (drag reorder)
outputVideo: string | null; // Stitched video blob URL or data URL
status: NodeStatus;
error: string | null;
progress: number; // 0-100 processing progress
encoderSupported: boolean | null; // null = not checked yet, true/false after check
}
/**
* Split Grid node - splits image into grid cells for parallel processing
*/
@ -233,6 +269,7 @@ export interface SplitGridNodeData extends BaseNodeData {
*/
export type WorkflowNodeData =
| ImageInputNodeData
| AudioInputNodeData
| AnnotationNodeData
| PromptNodeData
| PromptConstructorNodeData
@ -242,7 +279,8 @@ export type WorkflowNodeData =
| SplitGridNodeData
| OutputNodeData
| OutputGalleryNodeData
| ImageCompareNodeData;
| ImageCompareNodeData
| VideoStitchNodeData;
/**
* Workflow node with typed data (extended with optional groupId)
@ -254,7 +292,7 @@ export type WorkflowNode = Node<WorkflowNodeData, NodeType> & {
/**
* Handle types for node connections
*/
export type HandleType = "image" | "text";
export type HandleType = "image" | "text" | "audio" | "video";
/**
* Default settings for node types - stored in localStorage

Loading…
Cancel
Save