feat(41-02): register AudioInput in all integration points

- Added AudioInputNode export to nodes/index.ts - Added audioInput to defaultNodeDimensions (300x200) and createDefaultNodeData - Registered audioInput in WorkflowCanvas nodeTypes map - Updated getHandleType to recognize audio handles - Added audioInput to getNodeHandles (no inputs, audio output) - Updated isValidConnection for audio-to-audio validation - Added audio to ConnectionDropState handleType union - Updated findCompatibleHandle to accept audio type - Added AUDIO_SOURCE_OPTIONS and AUDIO_TARGET_OPTIONS to ConnectionDropMenu - Added audio array to getConnectedInputs return type - Updated getSourceOutput to handle audioInput nodes - Added audioInput execution case (no-op, data source) - Added AudioInputNodeData import to workflowStore - Updated DEFAULT_DIMENSIONS in WorkflowCanvas and quickstart/validation.ts - Updated createDefaultNodeData in quickstart/validation.ts - Added violet minimap color (#a78bfa) for audioInput nodes
5 months ago · d8a7e83126
7 changed files with 92 additions and 8 deletions
--- a/src/components/ConnectionDropMenu.tsx
+++ b/src/components/ConnectionDropMenu.tsx
@ -234,9 +234,27 @@ const VIDEO_SOURCE_OPTIONS: MenuOption[] = [
  },
 ];

+// Audio target options (nodes that accept audio input)
+const AUDIO_TARGET_OPTIONS: MenuOption[] = [
+  // VideoStitch will be added in Plan 03
+];
+
+// Audio source options (nodes that produce audio output)
+const AUDIO_SOURCE_OPTIONS: MenuOption[] = [
+  {
+    type: "audioInput",
+    label: "Audio Input",
+    icon: (
+      <svg className="w-4 h-4" fill="none" viewBox="0 0 24 24" stroke="currentColor" strokeWidth={1.5}>
+        <path strokeLinecap="round" strokeLinejoin="round" d="M9 9l10.5-3m0 6.553v3.75a2.25 2.25 0 01-1.632 2.163l-1.32.377a1.803 1.803 0 11-.99-3.467l2.31-.66a2.25 2.25 0 001.632-2.163zm0 0V2.25L9 5.25v10.303m0 0v3.75a2.25 2.25 0 01-1.632 2.163l-1.32.377a1.803 1.803 0 01-.99-3.467l2.31-.66A2.25 2.25 0 009 15.553z" />
+      </svg>
+    ),
+  },
+];
+
 interface ConnectionDropMenuProps {
  position: { x: number; y: number };
-  handleType: "image" | "text" | "video" | null;
+  handleType: "image" | "text" | "video" | "audio" | null;
  connectionType: "source" | "target"; // source = dragging from output, target = dragging from input
  onSelect: (selection: { type: NodeType | MenuAction; isAction: boolean }) => void;
  onClose: () => void;
@ -259,10 +277,12 @@ export function ConnectionDropMenu({
    if (connectionType === "source") {
      // Dragging from a source handle (output), need nodes with target handles (inputs)
      if (handleType === "video") return VIDEO_TARGET_OPTIONS;
+      if (handleType === "audio") return AUDIO_TARGET_OPTIONS;
      return handleType === "image" ? IMAGE_TARGET_OPTIONS : TEXT_TARGET_OPTIONS;
    } else {
      // Dragging from a target handle (input), need nodes with source handles (outputs)
      if (handleType === "video") return VIDEO_SOURCE_OPTIONS;
+      if (handleType === "audio") return AUDIO_SOURCE_OPTIONS;
      return handleType === "image" ? IMAGE_SOURCE_OPTIONS : TEXT_SOURCE_OPTIONS;
    }
  }, [handleType, connectionType]);
--- a/src/components/WorkflowCanvas.tsx
+++ b/src/components/WorkflowCanvas.tsx
@ -21,6 +21,7 @@ import { useWorkflowStore, WorkflowFile } from "@/store/workflowStore";
 import { useToast } from "@/components/Toast";
 import {
  ImageInputNode,
+  AudioInputNode,
  AnnotationNode,
  PromptNode,
  PromptConstructorNode,
@ -49,6 +50,7 @@ import { stripBinaryData } from "@/lib/chat/contextBuilder";

 const nodeTypes: NodeTypes = {
  imageInput: ImageInputNode,
+  audioInput: AudioInputNode,
  annotation: AnnotationNode,
  prompt: PromptNode,
  promptConstructor: PromptConstructorNode,
@ -72,10 +74,11 @@ const edgeTypes: EdgeTypes = {
 // - Video handles can only connect to generateVideo or output nodes
 // Helper to determine handle type from handle ID
 // For dynamic handles, we use naming convention: image inputs contain "image", text inputs are "prompt" or "negative_prompt"
-const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | null => {
+const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | "audio" | null => {
  if (!handleId) return null;
  // Standard handles
  if (handleId === "video") return "video";
+  if (handleId === "audio" || handleId.startsWith("audio")) return "audio";
  if (handleId === "image" || handleId === "text") return handleId;
  // Dynamic handles - check naming patterns (including indexed: text-0, image-0)
  if (handleId.includes("video")) return "video";
@ -89,6 +92,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[]
  switch (nodeType) {
    case "imageInput":
      return { inputs: ["reference"], outputs: ["image"] };
+    case "audioInput":
+      return { inputs: [], outputs: ["audio"] };
    case "annotation":
      return { inputs: ["image"], outputs: ["image"] };
    case "prompt":
@ -117,7 +122,7 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[]
 interface ConnectionDropState {
  position: { x: number; y: number };
  flowPosition: { x: number; y: number };
-  handleType: "image" | "text" | "video" | null;
+  handleType: "image" | "text" | "video" | "audio" | null;
  connectionType: "source" | "target";
  sourceNodeId: string | null;
  sourceHandleId: string | null;
@ -299,6 +304,11 @@ export function WorkflowCanvas() {
        return false;
      }

+      // Audio connections: audio handles can only connect to audio handles
+      if (sourceType === "audio" || targetType === "audio") {
+        return sourceType === "audio" && targetType === "audio";
+      }
+
      // Standard type matching for image and text
      // Image handles connect to image handles, text handles connect to text handles
      return sourceType === targetType;
@ -387,7 +397,7 @@ export function WorkflowCanvas() {
      // Helper to find a compatible handle on a node by type
      const findCompatibleHandle = (
        node: Node,
-        handleType: "image" | "text" | "video",
+        handleType: "image" | "text" | "video" | "audio",
        needInput: boolean
      ): string | null => {
        // Check for dynamic inputSchema first
@ -923,6 +933,7 @@ export function WorkflowCanvas() {
          // Offset by half the default node dimensions to center it
          const defaultDimensions: Record<NodeType, { width: number; height: number }> = {
            imageInput: { width: 300, height: 280 },
+            audioInput: { width: 300, height: 200 },
            annotation: { width: 300, height: 280 },
            prompt: { width: 320, height: 220 },
            promptConstructor: { width: 340, height: 280 },
@ -933,6 +944,7 @@ export function WorkflowCanvas() {
            output: { width: 320, height: 320 },
            outputGallery: { width: 320, height: 360 },
            imageCompare: { width: 400, height: 360 },
+            videoStitch: { width: 400, height: 280 },
          };
          const dims = defaultDimensions[nodeType];
          addNode(nodeType, { x: centerX - dims.width / 2, y: centerY - dims.height / 2 });
@ -1426,6 +1438,8 @@ export function WorkflowCanvas() {
            switch (node.type) {
              case "imageInput":
                return "#3b82f6";
+              case "audioInput":
+                return "#a78bfa";
              case "annotation":
                return "#8b5cf6";
              case "prompt":
--- a/src/components/nodes/AudioInputNode.tsx
+++ b/src/components/nodes/AudioInputNode.tsx
@ -18,7 +18,7 @@ export function AudioInputNode({ id, data, selected }: NodeProps<AudioInputNodeT
  const audioRef = useRef<HTMLAudioElement | null>(null);
  const canvasRef = useRef<HTMLCanvasElement>(null);
  const waveformContainerRef = useRef<HTMLDivElement>(null);
-  const animationFrameRef = useRef<number>();
+  const animationFrameRef = useRef<number | undefined>(undefined);

  const [isPlaying, setIsPlaying] = useState(false);
  const [currentTime, setCurrentTime] = useState(0);
--- a/src/components/nodes/index.ts
+++ b/src/components/nodes/index.ts
@ -1,4 +1,5 @@
 export { ImageInputNode } from "./ImageInputNode";
+export { AudioInputNode } from "./AudioInputNode";
 export { AnnotationNode } from "./AnnotationNode";
 export { PromptNode } from "./PromptNode";
 export { PromptConstructorNode } from "./PromptConstructorNode";
--- a/src/lib/quickstart/validation.ts
+++ b/src/lib/quickstart/validation.ts
@ -27,6 +27,7 @@ const VALID_HANDLE_TYPES = ["image", "text", "reference"];
 // Default node dimensions
 const DEFAULT_DIMENSIONS: Record<NodeType, { width: number; height: number }> = {
  imageInput: { width: 300, height: 280 },
+  audioInput: { width: 300, height: 200 },
  annotation: { width: 300, height: 280 },
  prompt: { width: 320, height: 220 },
  promptConstructor: { width: 340, height: 280 },
@ -37,6 +38,7 @@ const DEFAULT_DIMENSIONS: Record<NodeType, { width: number; height: number }> =
  output: { width: 320, height: 320 },
  outputGallery: { width: 320, height: 360 },
  imageCompare: { width: 400, height: 360 },
+  videoStitch: { width: 400, height: 280 },
 };

 /**
@ -198,6 +200,13 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData {
        filename: null,
        dimensions: null,
      };
+    case "audioInput":
+      return {
+        audioFile: null,
+        filename: null,
+        duration: null,
+        format: null,
+      };
    case "annotation":
      return {
        sourceImage: null,
@ -282,6 +291,16 @@ function createDefaultNodeData(type: NodeType): WorkflowNodeData {
        imageA: null,
        imageB: null,
      };
+    case "videoStitch":
+      return {
+        clips: [],
+        clipOrder: [],
+        outputVideo: null,
+        status: "idle",
+        error: null,
+        progress: 0,
+        encoderSupported: null,
+      };
  }
 }

--- a/src/store/utils/nodeDefaults.ts
+++ b/src/store/utils/nodeDefaults.ts
@ -1,6 +1,7 @@
 import {
  NodeType,
  ImageInputNodeData,
+  AudioInputNodeData,
  AnnotationNodeData,
  PromptNodeData,
  PromptConstructorNodeData,
@ -23,6 +24,7 @@ import { loadGenerateImageDefaults, loadNodeDefaults } from "./localStorage";
 */
 export const defaultNodeDimensions: Record<NodeType, { width: number; height: number }> = {
  imageInput: { width: 300, height: 280 },
+  audioInput: { width: 300, height: 200 },
  annotation: { width: 300, height: 280 },
  prompt: { width: 320, height: 220 },
  promptConstructor: { width: 340, height: 280 },
@ -33,6 +35,7 @@ export const defaultNodeDimensions: Record<NodeType, { width: number; height: nu
  output: { width: 320, height: 320 },
  outputGallery: { width: 320, height: 360 },
  imageCompare: { width: 400, height: 360 },
+  videoStitch: { width: 400, height: 280 },
 };

 /**
@ -65,6 +68,13 @@ export const createDefaultNodeData = (type: NodeType): WorkflowNodeData => {
        filename: null,
        dimensions: null,
      } as ImageInputNodeData;
+    case "audioInput":
+      return {
+        audioFile: null,
+        filename: null,
+        duration: null,
+        format: null,
+      } as AudioInputNodeData;
    case "annotation":
      return {
        sourceImage: null,
@ -178,5 +188,15 @@ export const createDefaultNodeData = (type: NodeType): WorkflowNodeData => {
        imageA: null,
        imageB: null,
      } as ImageCompareNodeData;
+    case "videoStitch":
+      return {
+        clips: [],
+        clipOrder: [],
+        outputVideo: null,
+        status: "idle",
+        error: null,
+        progress: 0,
+        encoderSupported: null,
+      };
  }
 };
--- a/src/store/workflowStore.ts
+++ b/src/store/workflowStore.ts
@ -13,6 +13,7 @@ import {
  WorkflowEdge,
  NodeType,
  ImageInputNodeData,
+  AudioInputNodeData,
  AnnotationNodeData,
  PromptNodeData,
  PromptConstructorNodeData,
@ -135,7 +136,7 @@ interface WorkflowStore {

  // Helpers
  getNodeById: (id: string) => WorkflowNode | undefined;
-  getConnectedInputs: (nodeId: string) => { images: string[]; videos: string[]; text: string | null; dynamicInputs: Record<string, string | string[]> };
+  getConnectedInputs: (nodeId: string) => { images: string[]; videos: string[]; audio: string[]; text: string | null; dynamicInputs: Record<string, string | string[]> };
  validateWorkflow: () => { valid: boolean; errors: string[] };

  // Global Image History
@ -750,6 +751,7 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
    const { edges, nodes } = get();
    const images: string[] = [];
    const videos: string[] = [];
+    const audio: string[] = [];
    let text: string | null = null;
    const dynamicInputs: Record<string, string | string[]> = {};

@ -799,9 +801,11 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
    };

    // Helper to extract output from source node
-    const getSourceOutput = (sourceNode: WorkflowNode): { type: "image" | "text" | "video"; value: string | null } => {
+    const getSourceOutput = (sourceNode: WorkflowNode): { type: "image" | "text" | "video" | "audio"; value: string | null } => {
      if (sourceNode.type === "imageInput") {
        return { type: "image", value: (sourceNode.data as ImageInputNodeData).image };
+      } else if (sourceNode.type === "audioInput") {
+        return { type: "audio", value: (sourceNode.data as AudioInputNodeData).audioFile };
      } else if (sourceNode.type === "annotation") {
        return { type: "image", value: (sourceNode.data as AnnotationNodeData).outputImage };
      } else if (sourceNode.type === "nanoBanana") {
@ -849,6 +853,8 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
        // This preserves type information from the source node
        if (type === "video") {
          videos.push(value);
+        } else if (type === "audio") {
+          audio.push(value);
        } else if (type === "text" || isTextHandle(handleId)) {
          text = value;
        } else if (isImageHandle(handleId) || !handleId) {
@ -856,7 +862,7 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
        }
      });

-    return { images, videos, text, dynamicInputs };
+    return { images, videos, audio, text, dynamicInputs };
  },

  validateWorkflow: () => {
@ -1040,6 +1046,10 @@ export const useWorkflowStore = create<WorkflowStore>((set, get) => ({
            // Nothing to execute, data is already set
            break;

+          case "audioInput":
+            // Audio input is a data source - no execution needed
+            break;
+
          case "annotation": {
            // Get connected image and set as source (use first image)
            const { images } = getConnectedInputs(node.id);