diff --git a/src/components/ConnectionDropMenu.tsx b/src/components/ConnectionDropMenu.tsx index cb3b5c4a..b4656aaf 100644 --- a/src/components/ConnectionDropMenu.tsx +++ b/src/components/ConnectionDropMenu.tsx @@ -234,9 +234,27 @@ const VIDEO_SOURCE_OPTIONS: MenuOption[] = [ }, ]; +// Audio target options (nodes that accept audio input) +const AUDIO_TARGET_OPTIONS: MenuOption[] = [ + // VideoStitch will be added in Plan 03 +]; + +// Audio source options (nodes that produce audio output) +const AUDIO_SOURCE_OPTIONS: MenuOption[] = [ + { + type: "audioInput", + label: "Audio Input", + icon: ( + + + + ), + }, +]; + interface ConnectionDropMenuProps { position: { x: number; y: number }; - handleType: "image" | "text" | "video" | null; + handleType: "image" | "text" | "video" | "audio" | null; connectionType: "source" | "target"; // source = dragging from output, target = dragging from input onSelect: (selection: { type: NodeType | MenuAction; isAction: boolean }) => void; onClose: () => void; @@ -259,10 +277,12 @@ export function ConnectionDropMenu({ if (connectionType === "source") { // Dragging from a source handle (output), need nodes with target handles (inputs) if (handleType === "video") return VIDEO_TARGET_OPTIONS; + if (handleType === "audio") return AUDIO_TARGET_OPTIONS; return handleType === "image" ? IMAGE_TARGET_OPTIONS : TEXT_TARGET_OPTIONS; } else { // Dragging from a target handle (input), need nodes with source handles (outputs) if (handleType === "video") return VIDEO_SOURCE_OPTIONS; + if (handleType === "audio") return AUDIO_SOURCE_OPTIONS; return handleType === "image" ? IMAGE_SOURCE_OPTIONS : TEXT_SOURCE_OPTIONS; } }, [handleType, connectionType]); diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 0c325b6b..693ee34c 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -21,6 +21,7 @@ import { useWorkflowStore, WorkflowFile } from "@/store/workflowStore"; import { useToast } from "@/components/Toast"; import { ImageInputNode, + AudioInputNode, AnnotationNode, PromptNode, PromptConstructorNode, @@ -49,6 +50,7 @@ import { stripBinaryData } from "@/lib/chat/contextBuilder"; const nodeTypes: NodeTypes = { imageInput: ImageInputNode, + audioInput: AudioInputNode, annotation: AnnotationNode, prompt: PromptNode, promptConstructor: PromptConstructorNode, @@ -72,10 +74,11 @@ const edgeTypes: EdgeTypes = { // - Video handles can only connect to generateVideo or output nodes // Helper to determine handle type from handle ID // For dynamic handles, we use naming convention: image inputs contain "image", text inputs are "prompt" or "negative_prompt" -const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | null => { +const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | "audio" | null => { if (!handleId) return null; // Standard handles if (handleId === "video") return "video"; + if (handleId === "audio" || handleId.startsWith("audio")) return "audio"; if (handleId === "image" || handleId === "text") return handleId; // Dynamic handles - check naming patterns (including indexed: text-0, image-0) if (handleId.includes("video")) return "video"; @@ -89,6 +92,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] switch (nodeType) { case "imageInput": return { inputs: ["reference"], outputs: ["image"] }; + case "audioInput": + return { inputs: [], outputs: ["audio"] }; case "annotation": return { inputs: ["image"], outputs: ["image"] }; case "prompt": @@ -117,7 +122,7 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] interface ConnectionDropState { position: { x: number; y: number }; flowPosition: { x: number; y: number }; - handleType: "image" | "text" | "video" | null; + handleType: "image" | "text" | "video" | "audio" | null; connectionType: "source" | "target"; sourceNodeId: string | null; sourceHandleId: string | null; @@ -299,6 +304,11 @@ export function WorkflowCanvas() { return false; } + // Audio connections: audio handles can only connect to audio handles + if (sourceType === "audio" || targetType === "audio") { + return sourceType === "audio" && targetType === "audio"; + } + // Standard type matching for image and text // Image handles connect to image handles, text handles connect to text handles return sourceType === targetType; @@ -387,7 +397,7 @@ export function WorkflowCanvas() { // Helper to find a compatible handle on a node by type const findCompatibleHandle = ( node: Node, - handleType: "image" | "text" | "video", + handleType: "image" | "text" | "video" | "audio", needInput: boolean ): string | null => { // Check for dynamic inputSchema first @@ -923,6 +933,7 @@ export function WorkflowCanvas() { // Offset by half the default node dimensions to center it const defaultDimensions: Record = { imageInput: { width: 300, height: 280 }, + audioInput: { width: 300, height: 200 }, annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, promptConstructor: { width: 340, height: 280 }, @@ -933,6 +944,7 @@ export function WorkflowCanvas() { output: { width: 320, height: 320 }, outputGallery: { width: 320, height: 360 }, imageCompare: { width: 400, height: 360 }, + videoStitch: { width: 400, height: 280 }, }; const dims = defaultDimensions[nodeType]; addNode(nodeType, { x: centerX - dims.width / 2, y: centerY - dims.height / 2 }); @@ -1426,6 +1438,8 @@ export function WorkflowCanvas() { switch (node.type) { case "imageInput": return "#3b82f6"; + case "audioInput": + return "#a78bfa"; case "annotation": return "#8b5cf6"; case "prompt": diff --git a/src/components/nodes/AudioInputNode.tsx b/src/components/nodes/AudioInputNode.tsx index b9453ea7..951fdfb9 100644 --- a/src/components/nodes/AudioInputNode.tsx +++ b/src/components/nodes/AudioInputNode.tsx @@ -18,7 +18,7 @@ export function AudioInputNode({ id, data, selected }: NodeProps(null); const canvasRef = useRef(null); const waveformContainerRef = useRef(null); - const animationFrameRef = useRef(); + const animationFrameRef = useRef(undefined); const [isPlaying, setIsPlaying] = useState(false); const [currentTime, setCurrentTime] = useState(0); diff --git a/src/components/nodes/index.ts b/src/components/nodes/index.ts index 0962be21..fa135738 100644 --- a/src/components/nodes/index.ts +++ b/src/components/nodes/index.ts @@ -1,4 +1,5 @@ export { ImageInputNode } from "./ImageInputNode"; +export { AudioInputNode } from "./AudioInputNode"; export { AnnotationNode } from "./AnnotationNode"; export { PromptNode } from "./PromptNode"; export { PromptConstructorNode } from "./PromptConstructorNode"; diff --git a/src/lib/quickstart/validation.ts b/src/lib/quickstart/validation.ts index f3bb97c5..f506f0eb 100644 --- a/src/lib/quickstart/validation.ts +++ b/src/lib/quickstart/validation.ts @@ -27,6 +27,7 @@ const VALID_HANDLE_TYPES = ["image", "text", "reference"]; // Default node dimensions const DEFAULT_DIMENSIONS: Record = { imageInput: { width: 300, height: 280 }, + audioInput: { width: 300, height: 200 }, annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, promptConstructor: { width: 340, height: 280 }, @@ -37,6 +38,7 @@ const DEFAULT_DIMENSIONS: Record = output: { width: 320, height: 320 }, outputGallery: { width: 320, height: 360 }, imageCompare: { width: 400, height: 360 }, + videoStitch: { width: 400, height: 280 }, }; /** @@ -198,6 +200,13 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData { filename: null, dimensions: null, }; + case "audioInput": + return { + audioFile: null, + filename: null, + duration: null, + format: null, + }; case "annotation": return { sourceImage: null, @@ -282,6 +291,16 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData { imageA: null, imageB: null, }; + case "videoStitch": + return { + clips: [], + clipOrder: [], + outputVideo: null, + status: "idle", + error: null, + progress: 0, + encoderSupported: null, + }; } } diff --git a/src/store/utils/nodeDefaults.ts b/src/store/utils/nodeDefaults.ts index de962053..65d8c56e 100644 --- a/src/store/utils/nodeDefaults.ts +++ b/src/store/utils/nodeDefaults.ts @@ -1,6 +1,7 @@ import { NodeType, ImageInputNodeData, + AudioInputNodeData, AnnotationNodeData, PromptNodeData, PromptConstructorNodeData, @@ -23,6 +24,7 @@ import { loadGenerateImageDefaults, loadNodeDefaults } from "./localStorage"; */ export const defaultNodeDimensions: Record = { imageInput: { width: 300, height: 280 }, + audioInput: { width: 300, height: 200 }, annotation: { width: 300, height: 280 }, prompt: { width: 320, height: 220 }, promptConstructor: { width: 340, height: 280 }, @@ -33,6 +35,7 @@ export const defaultNodeDimensions: Record { filename: null, dimensions: null, } as ImageInputNodeData; + case "audioInput": + return { + audioFile: null, + filename: null, + duration: null, + format: null, + } as AudioInputNodeData; case "annotation": return { sourceImage: null, @@ -178,5 +188,15 @@ export const createDefaultNodeData = (type: NodeType): WorkflowNodeData => { imageA: null, imageB: null, } as ImageCompareNodeData; + case "videoStitch": + return { + clips: [], + clipOrder: [], + outputVideo: null, + status: "idle", + error: null, + progress: 0, + encoderSupported: null, + }; } }; diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index 70a4c795..1fd17d03 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -13,6 +13,7 @@ import { WorkflowEdge, NodeType, ImageInputNodeData, + AudioInputNodeData, AnnotationNodeData, PromptNodeData, PromptConstructorNodeData, @@ -135,7 +136,7 @@ interface WorkflowStore { // Helpers getNodeById: (id: string) => WorkflowNode | undefined; - getConnectedInputs: (nodeId: string) => { images: string[]; videos: string[]; text: string | null; dynamicInputs: Record }; + getConnectedInputs: (nodeId: string) => { images: string[]; videos: string[]; audio: string[]; text: string | null; dynamicInputs: Record }; validateWorkflow: () => { valid: boolean; errors: string[] }; // Global Image History @@ -750,6 +751,7 @@ export const useWorkflowStore = create((set, get) => ({ const { edges, nodes } = get(); const images: string[] = []; const videos: string[] = []; + const audio: string[] = []; let text: string | null = null; const dynamicInputs: Record = {}; @@ -799,9 +801,11 @@ export const useWorkflowStore = create((set, get) => ({ }; // Helper to extract output from source node - const getSourceOutput = (sourceNode: WorkflowNode): { type: "image" | "text" | "video"; value: string | null } => { + const getSourceOutput = (sourceNode: WorkflowNode): { type: "image" | "text" | "video" | "audio"; value: string | null } => { if (sourceNode.type === "imageInput") { return { type: "image", value: (sourceNode.data as ImageInputNodeData).image }; + } else if (sourceNode.type === "audioInput") { + return { type: "audio", value: (sourceNode.data as AudioInputNodeData).audioFile }; } else if (sourceNode.type === "annotation") { return { type: "image", value: (sourceNode.data as AnnotationNodeData).outputImage }; } else if (sourceNode.type === "nanoBanana") { @@ -849,6 +853,8 @@ export const useWorkflowStore = create((set, get) => ({ // This preserves type information from the source node if (type === "video") { videos.push(value); + } else if (type === "audio") { + audio.push(value); } else if (type === "text" || isTextHandle(handleId)) { text = value; } else if (isImageHandle(handleId) || !handleId) { @@ -856,7 +862,7 @@ export const useWorkflowStore = create((set, get) => ({ } }); - return { images, videos, text, dynamicInputs }; + return { images, videos, audio, text, dynamicInputs }; }, validateWorkflow: () => { @@ -1040,6 +1046,10 @@ export const useWorkflowStore = create((set, get) => ({ // Nothing to execute, data is already set break; + case "audioInput": + // Audio input is a data source - no execution needed + break; + case "annotation": { // Get connected image and set as source (use first image) const { images } = getConnectedInputs(node.id);