From 57d0cd860e8038844c0dc2f20c5b291a758ac652 Mon Sep 17 00:00:00 2001 From: Willie Date: Sun, 15 Feb 2026 23:02:30 +1300 Subject: [PATCH 1/2] Release: 3D viewer, execute selected nodes, canvas navigation settings (#66) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor: extract schema utilities from generate route Extract INPUT_PATTERNS, InputMapping, ParameterTypeInfo, getParameterTypesFromSchema, coerceParameterTypes, and getInputMappingFromSchema into schemaUtils.ts with 23 new tests. Co-Authored-By: Claude Opus 4.6 * refactor: extract Gemini provider from generate route Move MODEL_MAP and generateWithGemini() into providers/gemini.ts. Remove unused imports (GoogleGenAI, uploadImageForUrl, shouldUseImageUrl, deleteImages, ProviderModel) from route.ts. Co-Authored-By: Claude Opus 4.6 * refactor: extract Replicate provider from generate route Move generateWithReplicate() into providers/replicate.ts. It imports schema utilities from schemaUtils.ts. Co-Authored-By: Claude Opus 4.6 * refactor: extract fal.ai, Kie.ai, WaveSpeed providers from generate route Move generateWithFalQueue, generateWithKie, generateWithWaveSpeed and all their helpers into providers/fal.ts, providers/kie.ts, providers/wavespeed.ts. Add barrel export at providers/index.ts. route.ts reduced from 2,838 to 518 lines (POST handler + routing only). Re-export clearFalInputMappingCache from route.ts for test backward compat. Co-Authored-By: Claude Opus 4.6 * refactor: extract API header builder from workflow store Replace 6 duplicated header-building if/else chains (across executeWorkflow and regenerateNode) with buildGenerateHeaders() and buildLlmHeaders() utility functions. Add 13 new tests. Co-Authored-By: Claude Opus 4.6 * refactor: extract execution utilities from workflow store Move groupNodesByLevel, chunk, revokeBlobUrl, clearNodeImageRefs, and concurrency settings into executionUtils.ts with 21 new tests. Co-Authored-By: Claude Opus 4.6 * refactor: extract getConnectedInputs and validateWorkflow as pure functions Move getConnectedInputs (136 lines) and validateWorkflow (59 lines) into connectedInputs.ts as pure functions taking (nodeId, nodes, edges). Store methods become thin wrappers. Add 22 new tests. Co-Authored-By: Claude Opus 4.6 * refactor: define node executor interface and context types Co-Authored-By: Claude Opus 4.6 * refactor: extract simple node executors Co-Authored-By: Claude Opus 4.6 * refactor: unify nanoBanana execution into shared executor Co-Authored-By: Claude Opus 4.6 * refactor: unify generateVideo execution into shared executor Co-Authored-By: Claude Opus 4.6 * refactor: unify llmGenerate execution into shared executor Co-Authored-By: Claude Opus 4.6 * refactor: unify splitGrid execution into shared executor Co-Authored-By: Claude Opus 4.6 * refactor: unify videoStitch and easeCurve executors Co-Authored-By: Claude Opus 4.6 * refactor: replace switch/if-else with node executor registry Wire extracted executors into workflowStore, replacing the ~1000-line switch statement in executeWorkflow and ~750-line if-else chain in regenerateNode with executor registry calls. Store reduced from 3,402 to 1,782 lines. Co-Authored-By: Claude Opus 4.6 * refactor: final cleanup of workflowStore module structure Remove dead trackSaveGeneration function (no longer called after executor extraction) and unused type imports. Store: 3,838 -> 1,721 lines (55% reduction). Co-Authored-By: Claude Opus 4.6 * fix: video regeneration spinner stuck + cancel error message Fix video element not reloading on regeneration by adding key prop tied to video history ID. Fix cancel showing timeout message by passing "user-cancelled" reason through AbortController and checking it in all three executor catch blocks. Co-Authored-By: Claude Opus 4.6 * fix: address PR #59 review — 18 bug, security, and robustness fixes Providers: - fal: don't cache failed schema fetches, remove duplicate param spread, SSRF-validate mediaUrl - gemini: remove redundant header overrides, fix nano-banana model ID - kie: module-level MAX_MEDIA_SIZE, post-download size check, upload size guard - replicate: remove duplicate param spread, validate modelId format - wavespeed: post-download size check, NaN-safe parseInt, prevent prompt overwrite Executors: - generateVideo: validate stored fallback inputs same as regular path - nanoBanana: cap imageHistory at 50 entries - splitGrid: handle img.onerror with warning + null data - videoProcessing: add FileReader onerror/onabort handlers Utilities: - connectedInputs: use ?? instead of || for outputText/template - executionUtils: guard orphan children in topological sort, validate chunk() size Co-Authored-By: Claude Opus 4.6 * fix: additional PR review fixes — SSRF, data semantics, sync tracking, error propagation Providers: - replicate: add SSRF validation on mediaUrl before fetch - fal/kie/replicate/wavespeed: large-video returns data="" with url set instead of putting raw URL in data field (violates GenerationOutput.data semantic) - kie: remove stale WaveSpeed JSDoc block from kie provider file - route.ts: update all output handling to support data="" + url for large videos Executors: - generateVideoExecutor/nanoBananaExecutor: track save-generation fetch in pendingImageSyncs via new trackSaveGeneration context method so auto-save waits for server ID resolution - videoProcessingExecutors: throw errors instead of returning in validation branches (encoderSupported, insufficient videos, no video connected) so Promise.allSettled sees rejections - types.ts: add trackSaveGeneration to NodeExecutionContext interface - workflowStore: wire trackSaveGeneration into both execution context sites Co-Authored-By: Claude Opus 4.6 * fix: content-type fallback, replicate polling/validation, atomic gallery append - fal.ts: Use model capabilities to determine fallback content-type (video/mp4 vs image/png) instead of defaulting to video for all models - replicate.ts: Increase polling timeout to 10 min for video models; filter output array to valid strings before URL validation - nanoBananaExecutor.ts: Replace read-modify-write on outputGallery with atomic appendOutputGalleryImage using Zustand set() callback Co-Authored-By: Claude Opus 4.6 * refactor: extract _buildExecutionContext to deduplicate NodeExecutionContext construction Replace identical inline context creation in executeWorkflow and regenerateNode with a single shared helper, ensuring consistent behavior across both code paths. Co-Authored-By: Claude Opus 4.6 * feat: add keyboard shortcuts help dialog - New KeyboardShortcutsDialog component showing all available shortcuts - Accessible via ? button in header and ? key on canvas - Groups shortcuts by category: General, Add Nodes, Layout, Canvas - Auto-detects Mac vs Windows/Linux for modifier key display - Follows existing modal patterns (Escape to close, backdrop click) * feat: add keyboard shortcuts dialog and 3D GLB viewer node - Keyboard shortcuts: press ? or click keyboard icon to view all shortcuts - 3D GLB viewer: upload .GLB files, orbit/rotate, capture as image output - Fix: restore trailing newline in gemini.ts provider * fix: make Prompt node editable when connected to LLM node When a Prompt node receives text from an LLM Generate node, the text should be editable by the user. Previously, the textarea was marked as readOnly when there was an incoming text connection, preventing edits in both the main view and the expanded modal. Changes: - Track last received LLM output to detect when it changes - Only update prompt when LLM node runs again (output changes) - Allow user edits in between LLM runs - Remove readOnly restriction on textarea - Update placeholder text to indicate editability This allows users to edit the LLM-generated text, which will be replaced only when the LLM node runs again, not on every render. Fixes #1470061054246518927 Co-Authored-By: Claude Sonnet 4.5 * fix: PR review fixes — lazy-load three.js, blob URL cleanup, missing integrations - Remove unused `useLoader` import from GLBViewerNode - Add useEffect cleanup to revoke blob URL on node unmount (memory leak) - Replace alert() with useToast for file validation errors - Lazy-load GLBViewerNode via next/dynamic to avoid bundling three.js for all users - Add glbViewer to ConnectionDropMenu IMAGE_SOURCE_OPTIONS - Add glbViewer case to quickstart validation createDefaultNodeData (build fix) - Restore trailing newline in gemini.ts Co-Authored-By: Claude Opus 4.6 * fix: address PR #64 review — blob leak, error handling, tests, docs Move @types/three to devDependencies, fix blob URL cleanup to track changes (not just unmount), add try/catch to capture and scene disposal, remove GLBViewerNode from barrel export to preserve lazy-loading, fix JSDoc, update CLAUDE.md with glbViewer node and ? shortcut, add connectedInputs and nodeDefaults tests for glbViewer. Co-Authored-By: Claude Opus 4.6 * feat: improve GLB viewer — fix viewport sizing, add spot light, handle labels, scroll isolation - Fix 3D viewport not filling node by adding resize={{ offsetSize: true }} for correct measurement in CSS-transformed React Flow nodes - Replace grid/axis indicators with spot light for cleaner scene - Add labeled input (3D) and output (Image) handles matching generate node style - Add native wheel event listener to prevent graph zoom/pan while scrolling to zoom 3D model - Remove redundant CSS workarounds (absolute positioning style, extra relative wrapper) - Add edge-to-edge viewport via contentClassName (removes default padding when GLB loaded) - Overlay controls bar on viewport with gradient background Co-Authored-By: Claude Opus 4.6 * feat: add customizable canvas navigation settings (#65) * feat: add customizable canvas navigation settings Add a new settings modal that allows users to configure canvas navigation preferences: - Pan Mode: Space + Drag (default), Middle Mouse, or Always On (ComfyUI-style) - Zoom Mode: Alt + Scroll (default), Ctrl + Scroll, or Scroll (no modifier) - Selection Mode: Click (default) or Alt + Drag Features: - Settings persist in localStorage - New canvas settings button in header (monitor icon) - Modal UI with radio button groups for each setting - React Flow props dynamically configured based on user preferences - Updated wheel event handler to respect zoom mode settings This addresses user requests for ComfyUI/TouchDesigner-style navigation where panning and zooming work without holding modifier keys. Co-Authored-By: Claude Sonnet 4.5 * fix: PR #65 review — altDrag selection bug, icon, dedup, tests - Fix altDrag selection mode: use selectionKeyCode="Alt" instead of selectionOnDrag=true which made drag-select work without any key - Replace misleading video camera icon with arrows-pointing-out for canvas navigation settings button - Extract duplicated settings buttons JSX into settingsButtons variable shared by both configured/unconfigured project branches - Add defensive useEffect to sync CanvasSettingsModal state on open - Add missing semicolon in types/index.ts canvas re-export - Add localStorage tests for getCanvasNavigationSettings and saveCanvasNavigationSettings - Add integration tests for updateCanvasNavigationSettings - Fix WorkflowCanvas test mocks to include canvasNavigationSettings Co-Authored-By: Claude Opus 4.6 * refactor: move canvas navigation settings into Project Settings modal Consolidate canvas settings from standalone CanvasSettingsModal into a new "Canvas" tab in ProjectSetupModal. Remove "like ComfyUI" text, add Shift+Drag selection mode, and fix save button to bottom of dialog. Co-Authored-By: Claude Opus 4.6 * fix: remove double semicolon in types/index.ts Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Sonnet 4.5 * feat: add executeSelectedNodes with topo sort, abort, and downstream propagation Implements the "Run selected nodes" feature (PR #61) with fixes for all 8 review issues: topological ordering via groupNodesByLevel, AbortController cancellation, glbViewer case coverage, AbortError filtering, downstream consumer propagation with dedup guard, and dynamic button label. Also extracts saveLogSession helper to replace 4 inline duplicates across executeWorkflow and regenerateNode. Co-Authored-By: Claude Opus 4.6 * fix: 3D model saving, error handling, and node improvements - Save 3D models (GLB) to generations folder via save-generation API - Add try-catch to WaveSpeed polling loop to handle network errors - Skip downloading 3D model bodies in WaveSpeed (return URL directly) - Fix schemaUtils false-positive matching for short property names - Stabilize GenerateImageNode output handle ID (always "image") - Fix GLBViewerNode: block-body forEach lint, store-based resize - Use useRef instead of useState for PromptNode lastReceivedText - Add executeGlbViewer tests and abort signal support - Fix annotation stale pass-through when upstream image changes - Skip error status on AbortError in all executor catch blocks - Revoke glbUrl blob URLs in clearNodeImageRefs Co-Authored-By: Claude Opus 4.6 * docs: require per-task commits in multi-task plans Co-Authored-By: Claude Opus 4.6 * feat: add Generate3DNodeData type definitions Add "generate3d" to NodeType union, Generate3DNodeData interface, Generate3DNodeDefaults, and update NodeDefaultsConfig. Co-Authored-By: Claude Opus 4.6 * feat: add generate3d node defaults and dimensions Add default dimensions (300x300) and createDefaultNodeData case for the generate3d node type. Co-Authored-By: Claude Opus 4.6 * feat: create generate3d executor Extracted from nanoBananaExecutor's 3D handling code. Handles 3D model generation via /api/generate with mediaType: "3d", cost tracking, and auto-save to generations folder. Co-Authored-By: Claude Opus 4.6 * feat: create Generate3DNode component Dedicated node component for 3D model generation, modeled after GenerateVideoNode. Features: dynamic input handles from schema, 3D output handle, model browsing, parameter controls, status overlays. Co-Authored-By: Claude Opus 4.6 * feat: register generate3d node type across codebase Register in execution index, workflowStore (executeWorkflow, regenerateNode, executeSelectedNodes), nodes index, WorkflowCanvas (nodeTypes, getNodeHandles, minimap color, findCompatibleHandle). Co-Authored-By: Claude Opus 4.6 * feat: add generate3d source output in connectedInputs Add generate3d case in getSourceOutput() to return 3D URL data, enabling downstream nodes to receive 3D model output. Co-Authored-By: Claude Opus 4.6 * feat: route 3D models to generate3d in ModelSearchDialog Add "3d" capability filter, update handleSelectModel to detect 3D models and create generate3d nodes with capabilities passed through, update recent models filter for 3D. Co-Authored-By: Claude Opus 4.6 * feat: update ConnectionDropMenu 3D source to use generate3d Change THREE_D_SOURCE_OPTIONS from nanoBanana to generate3d with cube icon and "Generate 3D" label. Co-Authored-By: Claude Opus 4.6 * refactor: remove 3D code from GenerateImageNode and nanoBananaExecutor - Remove text-to-3d/image-to-3d from IMAGE_CAPABILITIES - Remove is3D memo and conditional handle/label styling - Remove 3D output preview block - Remove output3dUrl from model change handlers - Remove is3DModel detection and mediaType from nanoBanana executor - Remove 3D response handling from nanoBanana executor - Remove output3dUrl from NanoBananaNodeData and its defaults - Remove 3D fallback from nanoBanana in connectedInputs Co-Authored-By: Claude Opus 4.6 * feat: update tests and remaining registrations for generate3d - Add Generate3DNode.test.tsx with render, handle, and output tests - Add 3D handle tests to ConnectionDropMenu.test.tsx - Add 3D model routing test to ModelSearchDialog.test.tsx - Update ModelSearchDialog test expectations for capabilities passthrough - Register generate3d in executeNode.ts dispatcher - Add generate3d to WorkflowCanvas keyboard shortcut dimensions - Add generate3d to quickstart validation (VALID_NODE_TYPES, dimensions, defaults) - Add generate3d to chat tools (VALID_NODE_TYPES, description) Co-Authored-By: Claude Opus 4.6 * fix: unwrap array inputs for non-array API parameters When multiple images are connected to a generation node, dynamicInputs aggregates them into arrays. Providers wrapped single values into arrays for array-typed params, but never unwrapped arrays for string-typed params, causing API failures (e.g. fal's image_url receiving an array instead of a string). Add symmetric unwrap logic across all four providers. Co-Authored-By: Claude Opus 4.6 * feat: move 3D from floating action bar to Generate submenu Replace top-level glbViewer button with generate3d option in the Generate combo dropdown, between Video and Text (LLM). Co-Authored-By: Claude Opus 4.6 * feat: hide CostIndicator when non-Gemini providers are in workflow The pricing calculator only has reliable data for Gemini models. When non-Gemini providers (fal, replicate, kie, wavespeed) are used, the displayed cost is incomplete/misleading. Add hasNonGeminiProviders() utility and use it to hide the CostIndicator entirely in that case. Co-Authored-By: Claude Opus 4.6 * feat: add 3D capability badges to ModelSearchDialog Add text-to-3d and image-to-3d capability badge styling. Co-Authored-By: Claude Opus 4.6 * feat: add play/run icon to LLM Generate node title bar Wire onRun and isExecuting props to BaseNode, matching the pattern used in GenerateImageNode and GenerateVideoNode. Co-Authored-By: Claude Opus 4.6 * fix: clear imageRef when user provides new image to prevent stale saves When a user pastes/uploads a new image into a node that already has a saved imageRef, the externalization logic incorrectly assumes the base64 is just hydrated data and discards it. Clear *Ref fields whenever new image data is set so the save logic correctly persists the new image. Co-Authored-By: Claude Opus 4.6 * fix: guard against undefined from empty array unwrap in fal provider Co-Authored-By: Claude Opus 4.6 * feat: add 3x2 grid layout option to split grid node Change layout selector from count-based to layout-based, allowing both 2x3 and 3x2 (portrait) layouts that produce 6 images. Now shows 7 layout options (2x2, 1x5, 2x3, 3x2, 2x4, 3x3, 2x5) with RxC labels. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: charlieshoelace --- .gitignore | 5 +- CLAUDE.md | 13 +- package-lock.json | 518 +++- package.json | 4 + src/app/api/generate/__tests__/route.test.ts | 151 +- .../generate/__tests__/schemaUtils.test.ts | 262 ++ src/app/api/generate/providers/fal.ts | 577 ++++ src/app/api/generate/providers/gemini.ts | 183 ++ src/app/api/generate/providers/index.ts | 9 + src/app/api/generate/providers/kie.ts | 823 ++++++ src/app/api/generate/providers/replicate.ts | 318 ++ src/app/api/generate/providers/wavespeed.ts | 412 +++ src/app/api/generate/route.ts | 2431 +-------------- src/app/api/generate/schemaUtils.ts | 193 ++ src/app/api/models/route.ts | 52 +- src/app/api/save-generation/route.ts | 16 +- src/app/globals.css | 6 + src/components/AnnotationModal.tsx | 2 +- src/components/ConnectionDropMenu.tsx | 39 +- src/components/CostIndicator.tsx | 5 +- src/components/FloatingActionBar.tsx | 47 +- src/components/Header.tsx | 99 +- src/components/KeyboardShortcutsDialog.tsx | 149 + src/components/ProjectSetupModal.tsx | 155 +- src/components/SplitGridSettingsModal.tsx | 59 +- src/components/WorkflowCanvas.tsx | 141 +- .../__tests__/AnnotationNode.test.tsx | 4 + .../__tests__/ConnectionDropMenu.test.tsx | 19 + .../__tests__/CostIndicator.test.tsx | 166 ++ .../__tests__/Generate3DNode.test.tsx | 234 ++ .../__tests__/ImageInputNode.test.tsx | 2 + .../__tests__/ModelSearchDialog.test.tsx | 43 +- .../__tests__/SplitGridSettingsModal.test.tsx | 66 +- .../__tests__/WorkflowCanvas.test.tsx | 4 + src/components/modals/ModelSearchDialog.tsx | 25 +- src/components/nodes/AnnotationNode.tsx | 4 + src/components/nodes/BaseNode.tsx | 8 +- src/components/nodes/GLBViewerNode.tsx | 550 ++++ src/components/nodes/Generate3DNode.tsx | 443 +++ src/components/nodes/GenerateImageNode.tsx | 4 +- src/components/nodes/GenerateVideoNode.tsx | 1 + src/components/nodes/ImageInputNode.tsx | 2 + src/components/nodes/LLMGenerateNode.tsx | 2 + src/components/nodes/PromptNode.tsx | 19 +- src/components/nodes/index.ts | 1 + src/lib/chat/tools.ts | 3 +- src/lib/providers/types.ts | 8 +- src/lib/quickstart/validation.ts | 18 + .../workflowStore.integration.test.ts | 33 + .../__tests__/generateVideoExecutor.test.ts | 276 ++ .../__tests__/llmGenerateExecutor.test.ts | 241 ++ .../__tests__/nanoBananaExecutor.test.ts | 347 +++ .../__tests__/simpleNodeExecutors.test.ts | 576 ++++ .../__tests__/splitGridExecutor.test.ts | 172 ++ .../videoProcessingExecutors.test.ts | 238 ++ src/store/execution/executeNode.ts | 90 + src/store/execution/generate3dExecutor.ts | 168 ++ src/store/execution/generateVideoExecutor.ts | 219 ++ src/store/execution/index.ts | 38 + src/store/execution/llmGenerateExecutor.ts | 129 + src/store/execution/nanoBananaExecutor.ts | 234 ++ src/store/execution/simpleNodeExecutors.ts | 256 ++ src/store/execution/splitGridExecutor.ts | 90 + src/store/execution/types.ts | 58 + .../execution/videoProcessingExecutors.ts | 248 ++ .../utils/__tests__/buildApiHeaders.test.ts | 96 + .../utils/__tests__/connectedInputs.test.ts | 263 ++ .../utils/__tests__/executionUtils.test.ts | 219 ++ .../utils/__tests__/localStorage.test.ts | 45 + .../utils/__tests__/nodeDefaults.test.ts | 9 + src/store/utils/buildApiHeaders.ts | 71 + src/store/utils/connectedInputs.ts | 250 ++ src/store/utils/executionUtils.ts | 140 + src/store/utils/localStorage.ts | 22 + src/store/utils/nodeDefaults.ts | 21 + src/store/workflowStore.ts | 2621 +++-------------- src/types/api.ts | 5 +- src/types/canvas.ts | 21 + src/types/index.ts | 1 + src/types/nodes.ts | 43 +- src/types/providers.ts | 1 + src/utils/__tests__/costCalculator.test.ts | 235 ++ src/utils/costCalculator.ts | 24 +- 83 files changed, 11009 insertions(+), 4786 deletions(-) create mode 100644 src/app/api/generate/__tests__/schemaUtils.test.ts create mode 100644 src/app/api/generate/providers/fal.ts create mode 100644 src/app/api/generate/providers/gemini.ts create mode 100644 src/app/api/generate/providers/index.ts create mode 100644 src/app/api/generate/providers/kie.ts create mode 100644 src/app/api/generate/providers/replicate.ts create mode 100644 src/app/api/generate/providers/wavespeed.ts create mode 100644 src/app/api/generate/schemaUtils.ts create mode 100644 src/components/KeyboardShortcutsDialog.tsx create mode 100644 src/components/__tests__/Generate3DNode.test.tsx create mode 100644 src/components/nodes/GLBViewerNode.tsx create mode 100644 src/components/nodes/Generate3DNode.tsx create mode 100644 src/store/execution/__tests__/generateVideoExecutor.test.ts create mode 100644 src/store/execution/__tests__/llmGenerateExecutor.test.ts create mode 100644 src/store/execution/__tests__/nanoBananaExecutor.test.ts create mode 100644 src/store/execution/__tests__/simpleNodeExecutors.test.ts create mode 100644 src/store/execution/__tests__/splitGridExecutor.test.ts create mode 100644 src/store/execution/__tests__/videoProcessingExecutors.test.ts create mode 100644 src/store/execution/executeNode.ts create mode 100644 src/store/execution/generate3dExecutor.ts create mode 100644 src/store/execution/generateVideoExecutor.ts create mode 100644 src/store/execution/index.ts create mode 100644 src/store/execution/llmGenerateExecutor.ts create mode 100644 src/store/execution/nanoBananaExecutor.ts create mode 100644 src/store/execution/simpleNodeExecutors.ts create mode 100644 src/store/execution/splitGridExecutor.ts create mode 100644 src/store/execution/types.ts create mode 100644 src/store/execution/videoProcessingExecutors.ts create mode 100644 src/store/utils/__tests__/buildApiHeaders.test.ts create mode 100644 src/store/utils/__tests__/connectedInputs.test.ts create mode 100644 src/store/utils/__tests__/executionUtils.test.ts create mode 100644 src/store/utils/buildApiHeaders.ts create mode 100644 src/store/utils/connectedInputs.ts create mode 100644 src/store/utils/executionUtils.ts create mode 100644 src/types/canvas.ts create mode 100644 src/utils/__tests__/costCalculator.test.ts diff --git a/.gitignore b/.gitignore index e8cce894..486f1a8c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,7 @@ logs/*.json .claude/ # planning docs -.planning/ \ No newline at end of file +.planning/ + +# agents +.agents/ \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index b8805ad5..933023f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -82,6 +82,7 @@ LLM models: | `nanoBanana` | AI image generation | image, text | image | | `llmGenerate` | AI text generation | text, image | text | | `splitGrid` | Split image into grid cells | image | reference | +| `glbViewer` | Load/display 3D GLB models | none | image | | `output` | Display final result | image | none | ## Node Connection System @@ -125,6 +126,7 @@ Returns `{ images: string[], text: string | null }`. - `H` - Stack selected nodes horizontally - `V` - Stack selected nodes vertically - `G` - Arrange selected nodes in grid +- `?` - Show keyboard shortcuts ## Adding New Node Types @@ -208,7 +210,16 @@ All routes in `src/app/api/`: - `node-banana-workflow-costs` - Cost tracking per workflow - `node-banana-nanoBanana-defaults` - Sticky generation settings -## Commits +## Git Workflow + +- The primary development branch is `develop`, NOT `main` or `master` +- Always checkout `develop` before creating feature branches: `git checkout develop` +- Create feature branches from `develop` using: `feature/` or `fix/` +- All PRs MUST target `develop`: use `gh pr create --base develop` +- Never push directly to `main`, `master`, or `develop` +## Commits +- Commit after each logical task or unit of work is complete. When implementing a multi-task plan, commit after finishing each task — do NOT batch all tasks into a single commit at the end. +- Each commit should be atomic and self-contained: one task = one commit. - The .planning directory is untracked, do not attempt to commit any changes to the files in this directory. diff --git a/package-lock.json b/package-lock.json index 45b8bd83..8818b7bd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,10 @@ "@ai-sdk/google": "^3.0.13", "@ai-sdk/react": "^3.0.51", "@google/genai": "^1.30.0", + "@react-three/drei": "^10.7.7", + "@react-three/fiber": "^9.5.0", "@tailwindcss/postcss": "^4.1.17", + "@types/three": "^0.182.0", "@xyflow/react": "^12.9.3", "ai": "^6.0.49", "autoprefixer": "^10.4.22", @@ -26,6 +29,7 @@ "react-konva": "^19.2.1", "react-markdown": "^10.1.0", "tailwindcss": "^4.1.17", + "three": "^0.182.0", "zustand": "^5.0.9" }, "devDependencies": { @@ -472,7 +476,6 @@ "version": "7.28.6", "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.28.6.tgz", "integrity": "sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==", - "dev": true, "license": "MIT", "engines": { "node": ">=6.9.0" @@ -671,6 +674,12 @@ "node": ">=18" } }, + "node_modules/@dimforge/rapier3d-compat": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@dimforge/rapier3d-compat/-/rapier3d-compat-0.12.0.tgz", + "integrity": "sha512-uekIGetywIgopfD97oDL5PfeezkFpNhwlzlaEYNOA0N6ghdsOvh/HYjSMek5Q2O1PYvRSDFcqFVJl4r4ZBwOow==", + "license": "Apache-2.0" + }, "node_modules/@emnapi/runtime": { "version": "1.7.1", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.7.1.tgz", @@ -1690,6 +1699,24 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@mediapipe/tasks-vision": { + "version": "0.10.17", + "resolved": "https://registry.npmjs.org/@mediapipe/tasks-vision/-/tasks-vision-0.10.17.tgz", + "integrity": "sha512-CZWV/q6TTe8ta61cZXjfnnHsfWIdFhms03M9T7Cnd5y2mdpylJM0rF1qRq+wsQVRMLz1OYPVEBU9ph2Bx8cxrg==", + "license": "Apache-2.0" + }, + "node_modules/@monogrid/gainmap-js": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/@monogrid/gainmap-js/-/gainmap-js-3.4.0.tgz", + "integrity": "sha512-2Z0FATFHaoYJ8b+Y4y4Hgfn3FRFwuU5zRrk+9dFWp4uGAdHGqVEdP7HP+gLA3X469KXHmfupJaUbKo1b/aDKIg==", + "license": "MIT", + "dependencies": { + "promise-worker-transferable": "^1.0.4" + }, + "peerDependencies": { + "three": ">= 0.159.0" + } + }, "node_modules/@next/env": { "version": "16.1.6", "resolved": "https://registry.npmjs.org/@next/env/-/env-16.1.6.tgz", @@ -1843,6 +1870,94 @@ "node": ">=14" } }, + "node_modules/@react-three/drei": { + "version": "10.7.7", + "resolved": "https://registry.npmjs.org/@react-three/drei/-/drei-10.7.7.tgz", + "integrity": "sha512-ff+J5iloR0k4tC++QtD/j9u3w5fzfgFAWDtAGQah9pF2B1YgOq/5JxqY0/aVoQG5r3xSZz0cv5tk2YuBob4xEQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mediapipe/tasks-vision": "0.10.17", + "@monogrid/gainmap-js": "^3.0.6", + "@use-gesture/react": "^10.3.1", + "camera-controls": "^3.1.0", + "cross-env": "^7.0.3", + "detect-gpu": "^5.0.56", + "glsl-noise": "^0.0.0", + "hls.js": "^1.5.17", + "maath": "^0.10.8", + "meshline": "^3.3.1", + "stats-gl": "^2.2.8", + "stats.js": "^0.17.0", + "suspend-react": "^0.1.3", + "three-mesh-bvh": "^0.8.3", + "three-stdlib": "^2.35.6", + "troika-three-text": "^0.52.4", + "tunnel-rat": "^0.1.2", + "use-sync-external-store": "^1.4.0", + "utility-types": "^3.11.0", + "zustand": "^5.0.1" + }, + "peerDependencies": { + "@react-three/fiber": "^9.0.0", + "react": "^19", + "react-dom": "^19", + "three": ">=0.159" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, + "node_modules/@react-three/fiber": { + "version": "9.5.0", + "resolved": "https://registry.npmjs.org/@react-three/fiber/-/fiber-9.5.0.tgz", + "integrity": "sha512-FiUzfYW4wB1+PpmsE47UM+mCads7j2+giRBltfwH7SNhah95rqJs3ltEs9V3pP8rYdS0QlNne+9Aj8dS/SiaIA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.17.8", + "@types/webxr": "*", + "base64-js": "^1.5.1", + "buffer": "^6.0.3", + "its-fine": "^2.0.0", + "react-use-measure": "^2.1.7", + "scheduler": "^0.27.0", + "suspend-react": "^0.1.3", + "use-sync-external-store": "^1.4.0", + "zustand": "^5.0.3" + }, + "peerDependencies": { + "expo": ">=43.0", + "expo-asset": ">=8.4", + "expo-file-system": ">=11.0", + "expo-gl": ">=11.0", + "react": ">=19 <19.3", + "react-dom": ">=19 <19.3", + "react-native": ">=0.78", + "three": ">=0.156" + }, + "peerDependenciesMeta": { + "expo": { + "optional": true + }, + "expo-asset": { + "optional": true + }, + "expo-file-system": { + "optional": true + }, + "expo-gl": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "react-native": { + "optional": true + } + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-beta.27", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.27.tgz", @@ -2601,6 +2716,12 @@ } } }, + "node_modules/@tweenjs/tween.js": { + "version": "23.1.3", + "resolved": "https://registry.npmjs.org/@tweenjs/tween.js/-/tween.js-23.1.3.tgz", + "integrity": "sha512-vJmvvwFxYuGnF2axRtPYocag6Clbb5YS7kLL+SO/TeVFzHqDIWrNKYtcsPMibjDx9O+bu+psAy9NKfWklassUA==", + "license": "MIT" + }, "node_modules/@types/aria-query": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/@types/aria-query/-/aria-query-5.0.4.tgz", @@ -2745,6 +2866,12 @@ "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==", "license": "MIT" }, + "node_modules/@types/draco3d": { + "version": "1.4.10", + "resolved": "https://registry.npmjs.org/@types/draco3d/-/draco3d-1.4.10.tgz", + "integrity": "sha512-AX22jp8Y7wwaBgAixaSvkoG4M/+PlAcm3Qs4OW8yT9DM4xUpWKeFhLueTAyZF39pviAdcDdeJoACapiAceqNcw==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -2804,6 +2931,12 @@ "undici-types": "~7.16.0" } }, + "node_modules/@types/offscreencanvas": { + "version": "2019.7.3", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.7.3.tgz", + "integrity": "sha512-ieXiYmgSRXUDeOntE1InxjWyvEelZGP63M+cGuquuRLuIKKT1osnkXjxev9B7d1nXSug5vpunx+gNlbVxMlC9A==", + "license": "MIT" + }, "node_modules/@types/react": { "version": "19.2.7", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.7.tgz", @@ -2832,18 +2965,63 @@ "@types/react": "*" } }, + "node_modules/@types/stats.js": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/stats.js/-/stats.js-0.17.4.tgz", + "integrity": "sha512-jIBvWWShCvlBqBNIZt0KAshWpvSjhkwkEu4ZUcASoAvhmrgAUI2t1dXrjSL4xXVLB4FznPrIsX3nKXFl/Dt4vA==", + "license": "MIT" + }, + "node_modules/@types/three": { + "version": "0.182.0", + "resolved": "https://registry.npmjs.org/@types/three/-/three-0.182.0.tgz", + "integrity": "sha512-WByN9V3Sbwbe2OkWuSGyoqQO8Du6yhYaXtXLoA5FkKTUJorZ+yOHBZ35zUUPQXlAKABZmbYp5oAqpA4RBjtJ/Q==", + "license": "MIT", + "dependencies": { + "@dimforge/rapier3d-compat": "~0.12.0", + "@tweenjs/tween.js": "~23.1.3", + "@types/stats.js": "*", + "@types/webxr": ">=0.5.17", + "@webgpu/types": "*", + "fflate": "~0.8.2", + "meshoptimizer": "~0.22.0" + } + }, "node_modules/@types/unist": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", "license": "MIT" }, + "node_modules/@types/webxr": { + "version": "0.5.24", + "resolved": "https://registry.npmjs.org/@types/webxr/-/webxr-0.5.24.tgz", + "integrity": "sha512-h8fgEd/DpoS9CBrjEQXR+dIDraopAEfu4wYVNY2tEPwk60stPWhvZMf4Foo5FakuQ7HFZoa8WceaWFervK2Ovg==", + "license": "MIT" + }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", "license": "ISC" }, + "node_modules/@use-gesture/core": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/@use-gesture/core/-/core-10.3.1.tgz", + "integrity": "sha512-WcINiDt8WjqBdUXye25anHiNxPc0VOrlT8F6LLkU6cycrOGUDyY/yyFmsg3k8i5OLvv25llc0QC45GhR/C8llw==", + "license": "MIT" + }, + "node_modules/@use-gesture/react": { + "version": "10.3.1", + "resolved": "https://registry.npmjs.org/@use-gesture/react/-/react-10.3.1.tgz", + "integrity": "sha512-Yy19y6O2GJq8f7CHf7L0nxL8bf4PZCPaVOCgJrusOeFHY1LvHgYXnmnXg6N5iwAnbgbZCDjo60SiM6IPJi9C5g==", + "license": "MIT", + "dependencies": { + "@use-gesture/core": "10.3.1" + }, + "peerDependencies": { + "react": ">= 16.8.0" + } + }, "node_modules/@vercel/oidc": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@vercel/oidc/-/oidc-3.1.0.tgz", @@ -3017,6 +3195,12 @@ "url": "https://opencollective.com/vitest" } }, + "node_modules/@webgpu/types": { + "version": "0.1.69", + "resolved": "https://registry.npmjs.org/@webgpu/types/-/types-0.1.69.tgz", + "integrity": "sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ==", + "license": "BSD-3-Clause" + }, "node_modules/@xyflow/react": { "version": "12.9.3", "resolved": "https://registry.npmjs.org/@xyflow/react/-/react-12.9.3.tgz", @@ -3253,7 +3437,6 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/bidi-js/-/bidi-js-1.0.3.tgz", "integrity": "sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==", - "dev": true, "license": "MIT", "dependencies": { "require-from-string": "^2.0.2" @@ -3310,12 +3493,49 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/camera-controls": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/camera-controls/-/camera-controls-3.1.2.tgz", + "integrity": "sha512-xkxfpG2ECZ6Ww5/9+kf4mfg1VEYAoe9aDSY+IwF0UEs7qEzwy0aVRfs2grImIECs/PoBtWFrh7RXsQkwG922JA==", + "license": "MIT", + "engines": { + "node": ">=22.0.0", + "npm": ">=10.5.1" + }, + "peerDependencies": { + "three": ">=0.126.1" + } + }, "node_modules/caniuse-lite": { "version": "1.0.30001757", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001757.tgz", @@ -3449,6 +3669,24 @@ "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==", "license": "MIT" }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -3690,6 +3928,15 @@ "node": ">=6" } }, + "node_modules/detect-gpu": { + "version": "5.0.70", + "resolved": "https://registry.npmjs.org/detect-gpu/-/detect-gpu-5.0.70.tgz", + "integrity": "sha512-bqerEP1Ese6nt3rFkwPnGbsUF9a4q+gMmpTVVOEzoCyeCc+y7/RvJnQZJx1JwhgQI5Ntg0Kgat8Uu7XpBqnz1w==", + "license": "MIT", + "dependencies": { + "webgl-constants": "^1.1.1" + } + }, "node_modules/detect-libc": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", @@ -3720,6 +3967,12 @@ "license": "MIT", "peer": true }, + "node_modules/draco3d": { + "version": "1.5.7", + "resolved": "https://registry.npmjs.org/draco3d/-/draco3d-1.5.7.tgz", + "integrity": "sha512-m6WCKt/erDXcw+70IJXnG7M3awwQPAsZvJGX5zY7beBqpELw6RDGkYVU0W43AFxye4pDZ5i2Lbyc/NNGqwjUVQ==", + "license": "Apache-2.0" + }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -3917,6 +4170,12 @@ "node": "^12.20 || >= 14.13" } }, + "node_modules/fflate": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", + "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "license": "MIT" + }, "node_modules/foreground-child": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.1.tgz", @@ -4039,6 +4298,12 @@ "dev": true, "license": "MIT" }, + "node_modules/glsl-noise": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/glsl-noise/-/glsl-noise-0.0.0.tgz", + "integrity": "sha512-b/ZCF6amfAUb7dJM/MxRs7AetQEahYzJ8PtgfrmEdtw6uyGOr+ZSGtgjFm6mfsBkxJ4d2W7kg+Nlqzqvn3Bc0w==", + "license": "MIT" + }, "node_modules/google-auth-library": { "version": "10.5.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.5.0.tgz", @@ -4135,6 +4400,12 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hls.js": { + "version": "1.6.15", + "resolved": "https://registry.npmjs.org/hls.js/-/hls.js-1.6.15.tgz", + "integrity": "sha512-E3a5VwgXimGHwpRGV+WxRTKeSp2DW5DI5MWv34ulL3t5UNmyJWCQ1KmLEHbYzcfThfXG8amBL+fCYPneGHC4VA==", + "license": "Apache-2.0" + }, "node_modules/html-encoding-sniffer": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-6.0.0.tgz", @@ -4192,6 +4463,26 @@ "node": ">= 14" } }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, "node_modules/immediate": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", @@ -4292,6 +4583,12 @@ "dev": true, "license": "MIT" }, + "node_modules/is-promise": { + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-2.2.2.tgz", + "integrity": "sha512-+lP4/6lKUBfQjZ2pdxThZvLUAafmZb8OAxFb8XXtiQmS35INgr85hdOGoEs124ez1FCnZJt6jau/T+alh58QFQ==", + "license": "MIT" + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -4829,6 +5126,16 @@ "lz-string": "bin/bin.js" } }, + "node_modules/maath": { + "version": "0.10.8", + "resolved": "https://registry.npmjs.org/maath/-/maath-0.10.8.tgz", + "integrity": "sha512-tRvbDF0Pgqz+9XUa4jjfgAQ8/aPKmQdWXilFu2tMy4GWj4NOsx99HlULO4IeREfbO3a0sA145DZYyvXPkybm0g==", + "license": "MIT", + "peerDependencies": { + "@types/three": ">=0.134.0", + "three": ">=0.134.0" + } + }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -5043,6 +5350,21 @@ "url": "https://github.com/sponsors/Vanilagy" } }, + "node_modules/meshline": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/meshline/-/meshline-3.3.1.tgz", + "integrity": "sha512-/TQj+JdZkeSUOl5Mk2J7eLcYTLiQm2IDzmlSvYm7ov15anEcDJ92GHqqazxTSreeNgfnYu24kiEvvv0WlbCdFQ==", + "license": "MIT", + "peerDependencies": { + "three": ">=0.137" + } + }, + "node_modules/meshoptimizer": { + "version": "0.22.0", + "resolved": "https://registry.npmjs.org/meshoptimizer/-/meshoptimizer-0.22.0.tgz", + "integrity": "sha512-IebiK79sqIy+E4EgOr+CAw+Ke8hAspXKzBd0JdgEmPHiAwmvEj2S4h1rfvo+o/BnfEYd/jAOg5IeeIjzlzSnDg==", + "license": "MIT" + }, "node_modules/micromark": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.2.tgz", @@ -5823,6 +6145,12 @@ "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", "license": "MIT" }, + "node_modules/potpack": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/potpack/-/potpack-1.0.2.tgz", + "integrity": "sha512-choctRBIV9EMT9WGAZHn3V7t0Z2pMQyl0EZE6pFc/6ml3ssw7Dlf/oAOvFwjm1HVsqfQN8GfeFyJ+d8tRzqueQ==", + "license": "ISC" + }, "node_modules/pretty-format": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", @@ -5870,6 +6198,16 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/promise-worker-transferable": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/promise-worker-transferable/-/promise-worker-transferable-1.0.4.tgz", + "integrity": "sha512-bN+0ehEnrXfxV2ZQvU2PetO0n4gqBD4ulq3MI1WOPLgr7/Mg9yRQkX5+0v1vagr74ZTsl7XtzlaYDo2EuCeYJw==", + "license": "Apache-2.0", + "dependencies": { + "is-promise": "^2.1.0", + "lie": "^3.0.2" + } + }, "node_modules/property-information": { "version": "7.1.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.1.0.tgz", @@ -6015,6 +6353,21 @@ "node": ">=0.10.0" } }, + "node_modules/react-use-measure": { + "version": "2.1.7", + "resolved": "https://registry.npmjs.org/react-use-measure/-/react-use-measure-2.1.7.tgz", + "integrity": "sha512-KrvcAo13I/60HpwGO5jpW7E9DfusKyLPLvuHlUyP5zqnmAPhNc6qTRjUQrdTADl0lpPpDVU2/Gg51UlOGHXbdg==", + "license": "MIT", + "peerDependencies": { + "react": ">=16.13", + "react-dom": ">=16.13" + }, + "peerDependenciesMeta": { + "react-dom": { + "optional": true + } + } + }, "node_modules/readable-stream": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.8.tgz", @@ -6087,7 +6440,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -6322,6 +6674,32 @@ "dev": true, "license": "MIT" }, + "node_modules/stats-gl": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/stats-gl/-/stats-gl-2.4.2.tgz", + "integrity": "sha512-g5O9B0hm9CvnM36+v7SFl39T7hmAlv541tU81ME8YeSb3i1CIP5/QdDeSB3A0la0bKNHpxpwxOVRo2wFTYEosQ==", + "license": "MIT", + "dependencies": { + "@types/three": "*", + "three": "^0.170.0" + }, + "peerDependencies": { + "@types/three": "*", + "three": "*" + } + }, + "node_modules/stats-gl/node_modules/three": { + "version": "0.170.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.170.0.tgz", + "integrity": "sha512-FQK+LEpYc0fBD+J8g6oSEyyNzjp+Q7Ks1C568WWaoMRLW+TkNNWmenWeGgJjV105Gd+p/2ql1ZcjYvNiPZBhuQ==", + "license": "MIT" + }, + "node_modules/stats.js": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/stats.js/-/stats.js-0.17.0.tgz", + "integrity": "sha512-hNKz8phvYLPEcRkeG1rsGmV5ChMjKDAWU7/OJJdDErPBNChQXxCo3WZurGpnWc6gZhAzEPFad1aVgyOANH1sMw==", + "license": "MIT" + }, "node_modules/std-env": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", @@ -6521,6 +6899,15 @@ "node": ">=8" } }, + "node_modules/suspend-react": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/suspend-react/-/suspend-react-0.1.3.tgz", + "integrity": "sha512-aqldKgX9aZqpoDp3e8/BZ8Dm7x1pJl+qI3ZKxDN0i/IQTWUwBx/ManmlVJ3wowqbno6c2bmiIfs+Um6LbsjJyQ==", + "license": "MIT", + "peerDependencies": { + "react": ">=17.0" + } + }, "node_modules/swr": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/swr/-/swr-2.3.8.tgz", @@ -6560,6 +6947,44 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/three": { + "version": "0.182.0", + "resolved": "https://registry.npmjs.org/three/-/three-0.182.0.tgz", + "integrity": "sha512-GbHabT+Irv+ihI1/f5kIIsZ+Ef9Sl5A1Y7imvS5RQjWgtTPfPnZ43JmlYI7NtCRDK9zir20lQpfg8/9Yd02OvQ==", + "license": "MIT" + }, + "node_modules/three-mesh-bvh": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/three-mesh-bvh/-/three-mesh-bvh-0.8.3.tgz", + "integrity": "sha512-4G5lBaF+g2auKX3P0yqx+MJC6oVt6sB5k+CchS6Ob0qvH0YIhuUk1eYr7ktsIpY+albCqE80/FVQGV190PmiAg==", + "license": "MIT", + "peerDependencies": { + "three": ">= 0.159.0" + } + }, + "node_modules/three-stdlib": { + "version": "2.36.1", + "resolved": "https://registry.npmjs.org/three-stdlib/-/three-stdlib-2.36.1.tgz", + "integrity": "sha512-XyGQrFmNQ5O/IoKm556ftwKsBg11TIb301MB5dWNicziQBEs2g3gtOYIf7pFiLa0zI2gUwhtCjv9fmjnxKZ1Cg==", + "license": "MIT", + "dependencies": { + "@types/draco3d": "^1.4.0", + "@types/offscreencanvas": "^2019.6.4", + "@types/webxr": "^0.5.2", + "draco3d": "^1.4.1", + "fflate": "^0.6.9", + "potpack": "^1.0.1" + }, + "peerDependencies": { + "three": ">=0.128.0" + } + }, + "node_modules/three-stdlib/node_modules/fflate": { + "version": "0.6.10", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.6.10.tgz", + "integrity": "sha512-IQrh3lEPM93wVCEczc9SaAOvkmcoQn/G8Bo1e8ZPlY3X3bnAxWaBdvTdvM1hP62iZp0BXWDy4vTAy4fF0+Dlpg==", + "license": "MIT" + }, "node_modules/throttleit": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", @@ -6672,6 +7097,36 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/troika-three-text": { + "version": "0.52.4", + "resolved": "https://registry.npmjs.org/troika-three-text/-/troika-three-text-0.52.4.tgz", + "integrity": "sha512-V50EwcYGruV5rUZ9F4aNsrytGdKcXKALjEtQXIOBfhVoZU9VAqZNIoGQ3TMiooVqFAbR1w15T+f+8gkzoFzawg==", + "license": "MIT", + "dependencies": { + "bidi-js": "^1.0.2", + "troika-three-utils": "^0.52.4", + "troika-worker-utils": "^0.52.0", + "webgl-sdf-generator": "1.1.1" + }, + "peerDependencies": { + "three": ">=0.125.0" + } + }, + "node_modules/troika-three-utils": { + "version": "0.52.4", + "resolved": "https://registry.npmjs.org/troika-three-utils/-/troika-three-utils-0.52.4.tgz", + "integrity": "sha512-NORAStSVa/BDiG52Mfudk4j1FG4jC4ILutB3foPnfGbOeIs9+G5vZLa0pnmnaftZUGm4UwSoqEpWdqvC7zms3A==", + "license": "MIT", + "peerDependencies": { + "three": ">=0.125.0" + } + }, + "node_modules/troika-worker-utils": { + "version": "0.52.0", + "resolved": "https://registry.npmjs.org/troika-worker-utils/-/troika-worker-utils-0.52.0.tgz", + "integrity": "sha512-W1CpvTHykaPH5brv5VHLfQo9D1OYuo0cSBEUQFFT/nBUzM8iD6Lq2/tgG/f1OelbAS1WtaTPQzE5uM49egnngw==", + "license": "MIT" + }, "node_modules/trough": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", @@ -6709,6 +7164,43 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/tunnel-rat": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tunnel-rat/-/tunnel-rat-0.1.2.tgz", + "integrity": "sha512-lR5VHmkPhzdhrM092lI2nACsLO4QubF0/yoOhzX7c+wIpbN1GjHNzCc91QlpxBi+cnx8vVJ+Ur6vL5cEoQPFpQ==", + "license": "MIT", + "dependencies": { + "zustand": "^4.3.2" + } + }, + "node_modules/tunnel-rat/node_modules/zustand": { + "version": "4.5.7", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.5.7.tgz", + "integrity": "sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw==", + "license": "MIT", + "dependencies": { + "use-sync-external-store": "^1.2.2" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0.6", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } + }, "node_modules/typescript": { "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", @@ -6862,6 +7354,15 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "license": "MIT" }, + "node_modules/utility-types": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/utility-types/-/utility-types-3.11.0.tgz", + "integrity": "sha512-6Z7Ma2aVEWisaL6TvBCy7P8rm2LQoPv6dJ7ecIaIixHcwfbJ0x7mWdbcwlIM5IGQxPZSFYeqRCqlOOeKoJYMkw==", + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", @@ -7086,6 +7587,17 @@ "node": ">= 8" } }, + "node_modules/webgl-constants": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/webgl-constants/-/webgl-constants-1.1.1.tgz", + "integrity": "sha512-LkBXKjU5r9vAW7Gcu3T5u+5cvSvh5WwINdr0C+9jpzVB41cjQAP5ePArDtk/WHYdVj0GefCgM73BA7FlIiNtdg==" + }, + "node_modules/webgl-sdf-generator": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/webgl-sdf-generator/-/webgl-sdf-generator-1.1.1.tgz", + "integrity": "sha512-9Z0JcMTFxeE+b2x1LJTdnaT8rT8aEp7MVxkNwoycNmJWwPdzoXzMh0BjJSh/AEFP+KPYZUli814h8bJZFIZ2jA==", + "license": "MIT" + }, "node_modules/webidl-conversions": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-8.0.1.tgz", diff --git a/package.json b/package.json index 007438c0..9e21827e 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "@ai-sdk/google": "^3.0.13", "@ai-sdk/react": "^3.0.51", "@google/genai": "^1.30.0", + "@react-three/drei": "^10.7.7", + "@react-three/fiber": "^9.5.0", "@tailwindcss/postcss": "^4.1.17", "@xyflow/react": "^12.9.3", "ai": "^6.0.49", @@ -30,6 +32,7 @@ "react-konva": "^19.2.1", "react-markdown": "^10.1.0", "tailwindcss": "^4.1.17", + "three": "^0.182.0", "zustand": "^5.0.9" }, "devDependencies": { @@ -37,6 +40,7 @@ "@testing-library/react": "^16.3.1", "@types/jszip": "^3.4.0", "@types/node": "^24.10.1", + "@types/three": "^0.182.0", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^4.7.0", diff --git a/src/app/api/generate/__tests__/route.test.ts b/src/app/api/generate/__tests__/route.test.ts index 079f7f6e..318388b2 100644 --- a/src/app/api/generate/__tests__/route.test.ts +++ b/src/app/api/generate/__tests__/route.test.ts @@ -525,7 +525,7 @@ describe("/api/generate route", () => { expect(data.success).toBe(true); expect(mockGenerateContent).toHaveBeenCalledWith( expect.objectContaining({ - model: "gemini-2.5-flash-image", + model: "gemini-2.5-flash-preview-image-generation", }) ); }); @@ -1431,6 +1431,73 @@ describe("/api/generate route", () => { expect(requestBody.input.prompt).toBe("Test prompt"); }); + it("should unwrap Replicate dynamicInputs array to single value when schema type is NOT 'array'", async () => { + // Model info fetch - with schema showing image_url has type: "string" (NOT array) + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + latest_version: { + id: "version123", + openapi_schema: { + components: { + schemas: { + Input: { + properties: { + image_url: { type: "string" }, // Single string, not array + prompt: { type: "string" }, + }, + }, + }, + }, + }, + }, + }), + }); + + // Create prediction + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + id: "prediction123", + status: "succeeded", + output: ["https://replicate.delivery/output.png"], + }), + }); + + // Fetch output media + mockFetch.mockResolvedValueOnce({ + ok: true, + headers: new Headers({ "content-type": "image/png" }), + arrayBuffer: () => Promise.resolve(new ArrayBuffer(1024)), + }); + + const request = createMockPostRequest( + { + prompt: "", + selectedModel: { + provider: "replicate", + modelId: "some-model/with-string-input", + displayName: "String Input Model", + }, + dynamicInputs: { + prompt: "Test prompt", + image_url: ["data:image/png;base64,image1", "data:image/png;base64,image2"], // Array sent for string param + }, + }, + { "X-Replicate-API-Key": "test-replicate-key" } + ); + + const response = await POST(request); + expect(response.status).toBe(200); + + // Verify image_url was unwrapped to first element because schema says type: "string" + const createPredictionCall = mockFetch.mock.calls[1]; + const requestBody = JSON.parse(createPredictionCall[1].body); + expect(requestBody.input.image_url).toBe("data:image/png;base64,image1"); + expect(Array.isArray(requestBody.input.image_url)).toBe(false); + expect(requestBody.input.prompt).toBe("Test prompt"); + }); + it("should use env var API key when header not provided", async () => { process.env.REPLICATE_API_KEY = "env-replicate-key"; @@ -2370,6 +2437,88 @@ describe("/api/generate route", () => { expect(requestBody.prompt).toBe("Test prompt"); }); + it("should unwrap array dynamicInputs to single value when schema type is NOT 'array'", async () => { + // Schema fetch - return schema showing image_url has type: "string" (NOT array) + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ + models: [{ + openapi: { + paths: { + "/": { + post: { + requestBody: { + content: { + "application/json": { + schema: { + properties: { + image_url: { type: "string" }, // Single string, not array + prompt: { type: "string" }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }], + }), + }); + + // CDN uploads for both images in the array + // Promise.all fires both initiates concurrently before either PUT, + // so mock order is: initiate1, initiate2, PUT1, PUT2 + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ upload_url: "https://fal.ai/cdn/put-target-1", file_url: "https://fal.ai/cdn/first.png" }), + }); + mockFetch.mockResolvedValueOnce({ + ok: true, + json: () => Promise.resolve({ upload_url: "https://fal.ai/cdn/put-target-2", file_url: "https://fal.ai/cdn/second.png" }), + }); + mockFetch.mockResolvedValueOnce({ ok: true }); // PUT 1 + mockFetch.mockResolvedValueOnce({ ok: true }); // PUT 2 + + // Queue flow: submit → poll → result → media + mockFalQueueSuccess( + { images: [{ url: "https://fal.media/output.png" }] }, + "image/png", + 1024 + ); + + const request = createMockPostRequest( + { + prompt: "Test prompt", + selectedModel: { + provider: "fal", + modelId: "fal-ai/flux/schnell", + displayName: "Flux Schnell", + }, + dynamicInputs: { + prompt: "Test prompt", + image_url: ["data:image/png;base64,image1", "data:image/png;base64,image2"], // Array of images + }, + }, + { "X-Fal-API-Key": "test-fal-key" } + ); + + const response = await POST(request); + expect(response.status).toBe(200); + + // Find the queue submit call + const queueSubmitCall = mockFetch.mock.calls.find( + (call: [string, ...unknown[]]) => typeof call[0] === "string" && call[0].includes("queue.fal.run") && !call[0].includes("/requests/") + ); + expect(queueSubmitCall).toBeDefined(); + const requestBody = JSON.parse((queueSubmitCall![1] as { body: string }).body); + // image_url should be unwrapped to a single CDN URL string (first element), not an array + expect(requestBody.image_url).toBe("https://fal.ai/cdn/first.png"); + expect(Array.isArray(requestBody.image_url)).toBe(false); + expect(requestBody.prompt).toBe("Test prompt"); + }); + it("should handle error response with error.message format", async () => { // Schema fetch (for input mapping when no dynamicInputs) mockFetch.mockResolvedValueOnce({ diff --git a/src/app/api/generate/__tests__/schemaUtils.test.ts b/src/app/api/generate/__tests__/schemaUtils.test.ts new file mode 100644 index 00000000..3cec95a6 --- /dev/null +++ b/src/app/api/generate/__tests__/schemaUtils.test.ts @@ -0,0 +1,262 @@ +import { describe, it, expect } from "vitest"; +import { + INPUT_PATTERNS, + getParameterTypesFromSchema, + coerceParameterTypes, + getInputMappingFromSchema, +} from "../schemaUtils"; +import type { ParameterTypeInfo } from "../schemaUtils"; + +describe("schemaUtils", () => { + describe("INPUT_PATTERNS", () => { + it("should contain all expected input categories", () => { + expect(INPUT_PATTERNS).toHaveProperty("prompt"); + expect(INPUT_PATTERNS).toHaveProperty("negativePrompt"); + expect(INPUT_PATTERNS).toHaveProperty("image"); + expect(INPUT_PATTERNS).toHaveProperty("aspectRatio"); + expect(INPUT_PATTERNS).toHaveProperty("duration"); + expect(INPUT_PATTERNS).toHaveProperty("fps"); + expect(INPUT_PATTERNS).toHaveProperty("audio"); + expect(INPUT_PATTERNS).toHaveProperty("seed"); + expect(INPUT_PATTERNS).toHaveProperty("steps"); + expect(INPUT_PATTERNS).toHaveProperty("guidance"); + expect(INPUT_PATTERNS).toHaveProperty("scheduler"); + expect(INPUT_PATTERNS).toHaveProperty("strength"); + }); + + it("should have 'prompt' as first pattern for prompt category", () => { + expect(INPUT_PATTERNS.prompt[0]).toBe("prompt"); + }); + + it("should include common image parameter names", () => { + expect(INPUT_PATTERNS.image).toContain("image_url"); + expect(INPUT_PATTERNS.image).toContain("image_urls"); + expect(INPUT_PATTERNS.image).toContain("first_frame"); + }); + }); + + describe("getParameterTypesFromSchema", () => { + it("should return empty object for undefined schema", () => { + expect(getParameterTypesFromSchema(undefined)).toEqual({}); + }); + + it("should return empty object for schema without components", () => { + expect(getParameterTypesFromSchema({})).toEqual({}); + }); + + it("should extract types from valid schema", () => { + const schema = { + components: { + schemas: { + Input: { + properties: { + prompt: { type: "string" }, + steps: { type: "integer" }, + guidance_scale: { type: "number" }, + enable_audio: { type: "boolean" }, + images: { type: "array" }, + }, + }, + }, + }, + }; + + const result = getParameterTypesFromSchema(schema); + expect(result).toEqual({ + prompt: "string", + steps: "integer", + guidance_scale: "number", + enable_audio: "boolean", + images: "array", + }); + }); + + it("should ignore unknown types", () => { + const schema = { + components: { + schemas: { + Input: { + properties: { + prompt: { type: "string" }, + custom: { type: "custom_type" }, + }, + }, + }, + }, + }; + + const result = getParameterTypesFromSchema(schema); + expect(result).toEqual({ prompt: "string" }); + }); + + it("should handle schema without properties", () => { + const schema = { + components: { + schemas: { + Input: {}, + }, + }, + }; + + expect(getParameterTypesFromSchema(schema)).toEqual({}); + }); + }); + + describe("coerceParameterTypes", () => { + it("should return empty object for undefined parameters", () => { + expect(coerceParameterTypes(undefined, {})).toEqual({}); + }); + + it("should coerce string to integer", () => { + const params = { steps: "20" }; + const types: ParameterTypeInfo = { steps: "integer" }; + expect(coerceParameterTypes(params, types)).toEqual({ steps: 20 }); + }); + + it("should coerce string to number", () => { + const params = { guidance_scale: "7.5" }; + const types: ParameterTypeInfo = { guidance_scale: "number" }; + expect(coerceParameterTypes(params, types)).toEqual({ guidance_scale: 7.5 }); + }); + + it("should coerce string to boolean", () => { + const params = { enable_audio: "true" }; + const types: ParameterTypeInfo = { enable_audio: "boolean" }; + expect(coerceParameterTypes(params, types)).toEqual({ enable_audio: true }); + }); + + it("should coerce 'false' string to false boolean", () => { + const params = { enable_audio: "false" }; + const types: ParameterTypeInfo = { enable_audio: "boolean" }; + expect(coerceParameterTypes(params, types)).toEqual({ enable_audio: false }); + }); + + it("should not coerce non-string values", () => { + const params = { steps: 20, guidance: 7.5 }; + const types: ParameterTypeInfo = { steps: "integer", guidance: "number" }; + expect(coerceParameterTypes(params, types)).toEqual({ steps: 20, guidance: 7.5 }); + }); + + it("should skip null and undefined values", () => { + const params = { steps: null, guidance: undefined, prompt: "hello" }; + const types: ParameterTypeInfo = { steps: "integer", guidance: "number", prompt: "string" }; + expect(coerceParameterTypes(params as Record, types)).toEqual({ + steps: null, + guidance: undefined, + prompt: "hello", + }); + }); + + it("should skip params with no type info", () => { + const params = { unknown_param: "42" }; + const types: ParameterTypeInfo = {}; + expect(coerceParameterTypes(params, types)).toEqual({ unknown_param: "42" }); + }); + + it("should not coerce invalid number strings", () => { + const params = { steps: "abc" }; + const types: ParameterTypeInfo = { steps: "integer" }; + const result = coerceParameterTypes(params, types); + expect(result.steps).toBe("abc"); // NaN check prevents coercion + }); + }); + + describe("getInputMappingFromSchema", () => { + it("should return empty mapping for undefined schema", () => { + const result = getInputMappingFromSchema(undefined); + expect(result.paramMap).toEqual({}); + expect(result.arrayParams.size).toBe(0); + expect(result.schemaArrayParams.size).toBe(0); + }); + + it("should map exact matches", () => { + const schema = { + components: { + schemas: { + Input: { + properties: { + prompt: { type: "string" }, + image_url: { type: "string" }, + aspect_ratio: { type: "string" }, + }, + }, + }, + }, + }; + + const result = getInputMappingFromSchema(schema); + expect(result.paramMap.prompt).toBe("prompt"); + expect(result.paramMap.image).toBe("image_url"); + expect(result.paramMap.aspectRatio).toBe("aspect_ratio"); + }); + + it("should detect array types", () => { + const schema = { + components: { + schemas: { + Input: { + properties: { + image_urls: { type: "array" }, + prompt: { type: "string" }, + }, + }, + }, + }, + }; + + const result = getInputMappingFromSchema(schema); + expect(result.paramMap.image).toBe("image_urls"); + expect(result.arrayParams.has("image")).toBe(true); + expect(result.schemaArrayParams.has("image_urls")).toBe(true); + }); + + it("should match case-insensitive partial matches", () => { + const schema = { + components: { + schemas: { + Input: { + properties: { + my_prompt_text: { type: "string" }, + }, + }, + }, + }, + }; + + const result = getInputMappingFromSchema(schema); + // "prompt" pattern should match "my_prompt_text" via partial match + expect(result.paramMap.prompt).toBe("my_prompt_text"); + }); + + it("should prefer exact matches over partial matches", () => { + const schema = { + components: { + schemas: { + Input: { + properties: { + prompt: { type: "string" }, + my_prompt_text: { type: "string" }, + }, + }, + }, + }, + }; + + const result = getInputMappingFromSchema(schema); + expect(result.paramMap.prompt).toBe("prompt"); + }); + + it("should handle schema with no Input schema", () => { + const schema = { + components: { + schemas: { + Output: { properties: {} }, + }, + }, + }; + + const result = getInputMappingFromSchema(schema); + expect(result.paramMap).toEqual({}); + }); + }); +}); diff --git a/src/app/api/generate/providers/fal.ts b/src/app/api/generate/providers/fal.ts new file mode 100644 index 00000000..112e6c9e --- /dev/null +++ b/src/app/api/generate/providers/fal.ts @@ -0,0 +1,577 @@ +/** + * fal.ai Provider for Generate API Route + * + * Handles image/video generation using fal.ai's Queue API. + * Images are uploaded to fal CDN before submission to avoid payload size issues. + */ + +import { GenerationInput, GenerationOutput } from "@/lib/providers/types"; +import { validateMediaUrl } from "@/utils/urlValidation"; +import { + INPUT_PATTERNS, + InputMapping, + ParameterTypeInfo, + coerceParameterTypes, +} from "../schemaUtils"; + +/** + * Extended input mapping with parameter types for fal.ai + */ +interface FalInputMapping extends InputMapping { + parameterTypes: ParameterTypeInfo; +} + +/** + * In-memory cache for fal.ai schema mappings to avoid extra API call per generation + */ +const falInputMappingCache = new Map(); +const FAL_MAPPING_CACHE_TTL = 30 * 60 * 1000; // 30 minutes + +/** Clear the fal schema mapping cache (exported for testing) */ +export function clearFalInputMappingCache() { + falInputMappingCache.clear(); +} + +/** + * Fetch fal.ai model schema and extract input parameter mappings + * Uses the Model Search API with OpenAPI expansion (same as /api/models/[modelId]) + * Results are cached in-memory for 30 minutes per model. + */ +async function getFalInputMapping(modelId: string, apiKey: string | null): Promise { + // Check cache first + const cached = falInputMappingCache.get(modelId); + if (cached && Date.now() - cached.timestamp < FAL_MAPPING_CACHE_TTL) { + return cached.result; + } + const paramMap: Record = {}; + const arrayParams = new Set(); + const schemaArrayParams = new Set(); + const parameterTypes: ParameterTypeInfo = {}; + + try { + // Use fal.ai Model Search API with OpenAPI expansion + const headers: Record = {}; + if (apiKey) { + headers["Authorization"] = `Key ${apiKey}`; + } + + const url = `https://api.fal.ai/v1/models?endpoint_id=${encodeURIComponent(modelId)}&expand=openapi-3.0`; + const response = await fetch(url, { headers }); + + if (!response.ok) { + return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; + } + + const data = await response.json(); + const modelData = data.models?.[0]; + if (!modelData?.openapi) { + return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; + } + + // Extract input schema from OpenAPI spec (same logic as /api/models/[modelId]) + const spec = modelData.openapi; + let inputSchema: Record | null = null; + + for (const pathObj of Object.values(spec.paths || {})) { + const postOp = (pathObj as Record)?.post as Record | undefined; + const reqBody = postOp?.requestBody as Record | undefined; + const content = reqBody?.content as Record> | undefined; + const jsonContent = content?.["application/json"]; + + if (jsonContent?.schema) { + const schema = jsonContent.schema as Record; + if (schema.$ref && typeof schema.$ref === "string") { + const refPath = schema.$ref.replace("#/components/schemas/", ""); + inputSchema = spec.components?.schemas?.[refPath] as Record; + break; + } else if (schema.properties) { + inputSchema = schema; + break; + } + } + } + + if (!inputSchema) { + return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; + } + + const properties = inputSchema.properties as Record | undefined; + if (!properties) return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; + + // First pass: detect all array-typed properties and extract parameter types + // This is used for dynamicInputs which use schema names directly + for (const [propName, prop] of Object.entries(properties)) { + const property = prop as Record; + if (property?.type === "array") { + schemaArrayParams.add(propName); + } + // Extract parameter type for type coercion + const type = property?.type as string | undefined; + if (type && ["string", "integer", "number", "boolean", "array", "object"].includes(type)) { + parameterTypes[propName] = type as ParameterTypeInfo[string]; + } + } + + // Second pass: match properties to INPUT_PATTERNS and detect array types + const propertyNames = Object.keys(properties); + for (const [genericName, patterns] of Object.entries(INPUT_PATTERNS)) { + for (const pattern of patterns) { + let matchedParam: string | null = null; + + // Check for exact match first + if (properties[pattern]) { + matchedParam = pattern; + } else { + // Check for case-insensitive partial match + const match = propertyNames.find(name => + name.toLowerCase().includes(pattern.toLowerCase()) || + pattern.toLowerCase().includes(name.toLowerCase()) + ); + if (match) { + matchedParam = match; + } + } + + if (matchedParam) { + paramMap[genericName] = matchedParam; + // Check if this property expects an array type + const property = properties[matchedParam] as Record; + if (property?.type === "array") { + arrayParams.add(genericName); + } + break; + } + } + } + + const result = { paramMap, arrayParams, schemaArrayParams, parameterTypes }; + falInputMappingCache.set(modelId, { result, timestamp: Date.now() }); + return result; + } catch { + // Schema parsing failed - return defaults without caching so next call retries + return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; + } +} + +export const MAX_UPLOAD_SIZE = 20 * 1024 * 1024; // 20 MB + +/** + * Upload a base64 data URL image to fal.ai CDN storage. + * Returns the CDN URL to use in API requests instead of inline base64. + * If the input is already a URL (not base64), returns it as-is. + */ +export async function uploadImageToFal(base64DataUrl: string, apiKey: string | null): Promise { + // Already a URL, not base64 + if (!base64DataUrl.startsWith("data:")) return base64DataUrl; + + const match = base64DataUrl.match(/^data:([^;]+);base64,(.+)$/); + if (!match) return base64DataUrl; + + const estimatedBytes = Math.ceil(match[2].length * 3 / 4); + if (estimatedBytes > MAX_UPLOAD_SIZE) { + throw new Error(`Image too large to upload (${(estimatedBytes / (1024 * 1024)).toFixed(1)} MB, max ${MAX_UPLOAD_SIZE / (1024 * 1024)} MB)`); + } + + const contentType = match[1]; + const binaryData = Buffer.from(match[2], "base64"); + + const authHeaders: Record = {}; + if (apiKey) authHeaders["Authorization"] = `Key ${apiKey}`; + + // Step 1: Initiate upload to get a signed PUT URL + const ext = contentType.split("/")[1] || "png"; + const initiateResponse = await fetch( + "https://rest.alpha.fal.ai/storage/upload/initiate?storage_type=fal-cdn-v3", + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...authHeaders, + }, + body: JSON.stringify({ + content_type: contentType, + file_name: `${Date.now()}.${ext}`, + }), + } + ); + + if (!initiateResponse.ok) { + throw new Error(`Failed to initiate fal CDN upload: ${initiateResponse.status}`); + } + + const { upload_url: uploadUrl, file_url: fileUrl } = await initiateResponse.json(); + + // Validate both URLs before using them (SSRF protection) + if (!uploadUrl || !fileUrl) { + throw new Error("fal CDN initiate response missing upload_url or file_url"); + } + + const uploadUrlCheck = validateMediaUrl(uploadUrl); + if (!uploadUrlCheck.valid || !uploadUrl.startsWith('https://')) { + throw new Error(`fal CDN upload_url failed validation: ${uploadUrlCheck.error || 'not HTTPS'}`); + } + + const fileUrlCheck = validateMediaUrl(fileUrl); + if (!fileUrlCheck.valid || !fileUrl.startsWith('https://')) { + throw new Error(`fal CDN file_url failed validation: ${fileUrlCheck.error || 'not HTTPS'}`); + } + + // Step 2: PUT the binary data to the validated signed URL + const putResponse = await fetch(uploadUrl, { + method: "PUT", + headers: { "Content-Type": contentType }, + body: binaryData, + }); + + if (!putResponse.ok) { + throw new Error(`Failed to upload to fal CDN: ${putResponse.status}`); + } + + return fileUrl; +} + +/** + * Generate using fal.ai Queue API + * Uses async queue submission + polling (1s interval) instead of blocking fal.run. + * Images are uploaded to fal CDN before submission to avoid payload size issues. + */ +export async function generateWithFalQueue( + requestId: string, + apiKey: string | null, + input: GenerationInput +): Promise { + console.log(`[API:${requestId}] fal.ai queue generation - Model: ${input.model.id}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); + + const modelId = input.model.id; + const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; + console.log(`[API:${requestId}] Dynamic inputs: ${hasDynamicInputs ? Object.keys(input.dynamicInputs!).join(", ") : "none"}, API key: ${apiKey ? "yes" : "no"}`); + + // Fetch schema for type coercion and input mapping (cached) + const { paramMap, arrayParams, schemaArrayParams, parameterTypes } = await getFalInputMapping(modelId, apiKey); + + // Build request body - parameters are applied per-path below to avoid double-spreading + const requestBody: Record = {}; + + // Upload base64 images to fal CDN to avoid sending large payloads inline + const uploadImage = async (value: string | string[]): Promise => { + if (Array.isArray(value)) { + return Promise.all(value.map(v => typeof v === "string" && v.startsWith("data:") ? uploadImageToFal(v, apiKey) : Promise.resolve(v))); + } + if (typeof value === "string" && value.startsWith("data:")) { + return uploadImageToFal(value, apiKey); + } + return value; + }; + + if (hasDynamicInputs) { + // Apply coerced parameters first, then dynamic inputs override + Object.assign(requestBody, coerceParameterTypes(input.parameters, parameterTypes)); + const filteredInputs: Record = {}; + for (const [key, value] of Object.entries(input.dynamicInputs!)) { + if (value !== null && value !== undefined && value !== '') { + let processedValue: unknown = value; + // Upload base64 images to CDN + if (typeof value === "string" || Array.isArray(value)) { + processedValue = await uploadImage(value); + } + // Wrap in array if schema expects array but we have a single value + if (schemaArrayParams.has(key) && !Array.isArray(processedValue)) { + filteredInputs[key] = [processedValue]; + } else if (!schemaArrayParams.has(key) && Array.isArray(processedValue)) { + // Unwrap array to single value if schema expects a string (e.g. image_url) + if (processedValue.length > 0) { + filteredInputs[key] = processedValue[0]; + } + } else { + filteredInputs[key] = processedValue; + } + } + } + Object.assign(requestBody, filteredInputs); + } else { + // Fallback: use schema to map generic input names to model-specific parameter names + if (input.prompt) { + const promptParam = paramMap.prompt || "prompt"; + requestBody[promptParam] = input.prompt; + } + + if (input.images && input.images.length > 0) { + // Upload images to CDN before sending + const uploadedImages = await Promise.all( + input.images.map(img => uploadImageToFal(img, apiKey)) + ); + const imageParam = paramMap.image || "image_url"; + if (arrayParams.has("image")) { + requestBody[imageParam] = uploadedImages; + } else { + requestBody[imageParam] = uploadedImages[0]; + } + } + + // Map any parameters that might need renaming (use coerced values) + const coercedParams = coerceParameterTypes(input.parameters, parameterTypes); + for (const [key, value] of Object.entries(coercedParams)) { + const mappedKey = paramMap[key] || key; + requestBody[mappedKey] = value; + } + } + + // Build headers + const headers: Record = { + "Content-Type": "application/json", + }; + if (apiKey) { + headers["Authorization"] = `Key ${apiKey}`; + } + + // Submit to queue + console.log(`[API:${requestId}] Submitting to fal.ai queue with inputs: ${Object.keys(requestBody).join(", ")}`); + const submitResponse = await fetch(`https://queue.fal.run/${modelId}`, { + method: "POST", + headers, + body: JSON.stringify(requestBody), + }); + + if (!submitResponse.ok) { + const errorText = await submitResponse.text(); + let errorDetail = errorText || `HTTP ${submitResponse.status}`; + try { + const errorJson = JSON.parse(errorText); + if (typeof errorJson.error === 'object' && errorJson.error?.message) { + errorDetail = errorJson.error.message; + } else if (errorJson.detail) { + if (Array.isArray(errorJson.detail)) { + errorDetail = errorJson.detail.map((d: { msg?: string; loc?: string[] }) => + d.msg || JSON.stringify(d) + ).join('; '); + } else { + errorDetail = errorJson.detail; + } + } else if (errorJson.message) { + errorDetail = errorJson.message; + } else if (typeof errorJson.error === 'string') { + errorDetail = errorJson.error; + } + } catch { + // Keep original text if not JSON + } + + if (submitResponse.status === 429) { + return { + success: false, + error: `${input.model.name}: Rate limit exceeded. ${apiKey ? "Try again in a moment." : "Add an API key in settings for higher limits."}`, + }; + } + + return { + success: false, + error: `${input.model.name}: ${errorDetail}`, + }; + } + + const submitResult = await submitResponse.json(); + console.log(`[API:${requestId}] Queue submit response:`, JSON.stringify(submitResult).substring(0, 500)); + const falRequestId = submitResult.request_id; + + if (!falRequestId) { + console.error(`[API:${requestId}] No request_id in queue submit response`); + return { + success: false, + error: "No request_id in queue response", + }; + } + + // Use URLs from response if provided, with SSRF validation; fall back to constructed URLs + const fallbackStatusUrl = `https://queue.fal.run/${modelId}/requests/${falRequestId}/status`; + const fallbackResponseUrl = `https://queue.fal.run/${modelId}/requests/${falRequestId}`; + let statusUrl = fallbackStatusUrl; + let responseUrl = fallbackResponseUrl; + + if (submitResult.status_url) { + const statusCheck = validateMediaUrl(submitResult.status_url); + if (statusCheck.valid && submitResult.status_url.startsWith('https://queue.fal.run/')) { + statusUrl = submitResult.status_url; + } else { + console.warn(`[API:${requestId}] fal.ai provided invalid status URL: ${submitResult.status_url} — falling back to constructed URL`); + } + } + if (submitResult.response_url) { + const responseCheck = validateMediaUrl(submitResult.response_url); + if (responseCheck.valid && submitResult.response_url.startsWith('https://queue.fal.run/')) { + responseUrl = submitResult.response_url; + } else { + console.warn(`[API:${requestId}] fal.ai provided invalid response URL: ${submitResult.response_url} — falling back to constructed URL`); + } + } + + console.log(`[API:${requestId}] Queue request submitted: ${falRequestId}, status URL: ${statusUrl}`); + + // Poll for completion + const maxWaitTime = 10 * 60 * 1000; // 10 minutes for video + const pollInterval = 1000; // 1 second (matches Replicate/WaveSpeed) + const startTime = Date.now(); + let lastStatus = ""; + + while (true) { + if (Date.now() - startTime > maxWaitTime) { + console.error(`[API:${requestId}] Queue request timed out after 10 minutes`); + return { + success: false, + error: `${input.model.name}: Video generation timed out after 10 minutes`, + }; + } + + await new Promise(resolve => setTimeout(resolve, pollInterval)); + + const statusResponse = await fetch( + statusUrl, + { headers: apiKey ? { "Authorization": `Key ${apiKey}` } : {} } + ); + + if (!statusResponse.ok) { + console.error(`[API:${requestId}] Failed to poll status: ${statusResponse.status}`); + return { + success: false, + error: `Failed to poll status: ${statusResponse.status}`, + }; + } + + const statusResult = await statusResponse.json(); + const status = statusResult.status; + + if (status !== lastStatus) { + console.log(`[API:${requestId}] Queue status: ${status}`); + lastStatus = status; + } + + if (status === "COMPLETED") { + // Fetch the result + const resultResponse = await fetch( + responseUrl, + { headers: apiKey ? { "Authorization": `Key ${apiKey}` } : {} } + ); + + if (!resultResponse.ok) { + console.error(`[API:${requestId}] Failed to fetch result: ${resultResponse.status}`); + return { + success: false, + error: `Failed to fetch result: ${resultResponse.status}`, + }; + } + + const result = await resultResponse.json(); + + // Extract media URL from result + let mediaUrl: string | null = null; + + // Check for 3D model output (GLB mesh) — must check before images + if (result.model_mesh?.url) { + mediaUrl = result.model_mesh.url; + } else if (result.mesh?.url) { + mediaUrl = result.mesh.url; + } else if (result.glb?.url) { + mediaUrl = result.glb.url; + } else if (result.video && result.video.url) { + mediaUrl = result.video.url; + } else if (result.images && Array.isArray(result.images) && result.images.length > 0) { + mediaUrl = result.images[0].url; + } else if (result.image && result.image.url) { + mediaUrl = result.image.url; + } else if (result.output && typeof result.output === "string") { + mediaUrl = result.output; + } + + if (!mediaUrl) { + console.error(`[API:${requestId}] No media URL found in queue result`); + return { + success: false, + error: "No media URL in response", + }; + } + + // Validate URL before fetching (SSRF protection) + const mediaUrlCheck = validateMediaUrl(mediaUrl); + if (!mediaUrlCheck.valid) { + return { success: false, error: `Invalid media URL: ${mediaUrlCheck.error}` }; + } + + // Fetch the media and convert to base64 + console.log(`[API:${requestId}] Fetching output from: ${mediaUrl.substring(0, 80)}...`); + const mediaResponse = await fetch(mediaUrl); + + if (!mediaResponse.ok) { + return { + success: false, + error: `Failed to fetch output: ${mediaResponse.status}`, + }; + } + + const is3DModel = input.model.capabilities.some(c => c.includes("3d")); + const isVideoModel = input.model.capabilities.some(c => c.includes("video")); + + // For 3D models, return URL directly (GLB files are binary — don't base64 encode) + if (is3DModel) { + console.log(`[API:${requestId}] SUCCESS - Returning 3D model URL`); + return { + success: true, + outputs: [ + { + type: "3d", + data: "", + url: mediaUrl, + }, + ], + }; + } + + const contentType = mediaResponse.headers.get("content-type") || (isVideoModel ? "video/mp4" : "image/png"); + const isVideo = contentType.startsWith("video/"); + + const mediaArrayBuffer = await mediaResponse.arrayBuffer(); + const mediaSizeBytes = mediaArrayBuffer.byteLength; + const mediaSizeMB = mediaSizeBytes / (1024 * 1024); + + console.log(`[API:${requestId}] Output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); + + // For very large videos (>20MB), return URL only (data left empty for consumers) + if (isVideo && mediaSizeMB > 20) { + console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); + return { + success: true, + outputs: [ + { + type: "video", + data: "", + url: mediaUrl, + }, + ], + }; + } + + const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); + console.log(`[API:${requestId}] SUCCESS - Returning ${isVideo ? "video" : "image"}`); + + return { + success: true, + outputs: [ + { + type: isVideo ? "video" : "image", + data: `data:${contentType};base64,${mediaBase64}`, + url: mediaUrl, + }, + ], + }; + } + + if (status === "FAILED") { + const errorMessage = statusResult.error || "Video generation failed"; + console.error(`[API:${requestId}] Queue request failed: ${errorMessage}`); + return { + success: false, + error: `${input.model.name}: ${errorMessage}`, + }; + } + + // Continue polling for IN_QUEUE, IN_PROGRESS, etc. + } +} diff --git a/src/app/api/generate/providers/gemini.ts b/src/app/api/generate/providers/gemini.ts new file mode 100644 index 00000000..58bca493 --- /dev/null +++ b/src/app/api/generate/providers/gemini.ts @@ -0,0 +1,183 @@ +/** + * Gemini Provider for Generate API Route + * + * Handles image generation using Google's Gemini API models. + */ + +import { NextResponse } from "next/server"; +import { GoogleGenAI } from "@google/genai"; +import { GenerateResponse, ModelType } from "@/types"; + +/** + * Map model types to Gemini model IDs + */ +export const MODEL_MAP: Record = { + "nano-banana": "gemini-2.5-flash-preview-image-generation", + "nano-banana-pro": "gemini-3-pro-image-preview", +}; + +/** + * Generate image using Gemini API (legacy/default path) + */ +export async function generateWithGemini( + requestId: string, + apiKey: string, + prompt: string, + images: string[], + model: ModelType, + aspectRatio?: string, + resolution?: string, + useGoogleSearch?: boolean +): Promise> { + console.log(`[API:${requestId}] Gemini generation - Model: ${model}, Images: ${images?.length || 0}, Prompt: ${prompt?.length || 0} chars`); + + // Extract base64 data and MIME types from data URLs + const imageData = (images || []).map((image, idx) => { + if (image.includes("base64,")) { + const [header, data] = image.split("base64,"); + // Extract MIME type from header (e.g., "data:image/png;" -> "image/png") + const mimeMatch = header.match(/data:([^;]+)/); + const mimeType = mimeMatch ? mimeMatch[1] : "image/png"; + console.log(`[API:${requestId}] Image ${idx + 1}: ${mimeType}, ${(data.length / 1024).toFixed(1)}KB`); + return { data, mimeType }; + } + console.log(`[API:${requestId}] Image ${idx + 1}: raw, ${(image.length / 1024).toFixed(1)}KB`); + return { data: image, mimeType: "image/png" }; + }); + + // Initialize Gemini client + const ai = new GoogleGenAI({ apiKey }); + + // Build request parts array with prompt and all images + const requestParts: Array<{ text: string } | { inlineData: { mimeType: string; data: string } }> = [ + { text: prompt }, + ...imageData.map(({ data, mimeType }) => ({ + inlineData: { + mimeType, + data, + }, + })), + ]; + + // Build config object based on model capabilities + const config: Record = { + responseModalities: ["IMAGE", "TEXT"], + }; + + // Add imageConfig for both models (both support aspect ratio) + if (aspectRatio) { + config.imageConfig = { + aspectRatio, + }; + } + + // Add resolution only for Nano Banana Pro + if (model === "nano-banana-pro" && resolution) { + if (!config.imageConfig) { + config.imageConfig = {}; + } + (config.imageConfig as Record).imageSize = resolution; + } + + // Add tools array for Google Search (only Nano Banana Pro) + const tools = []; + if (model === "nano-banana-pro" && useGoogleSearch) { + tools.push({ googleSearch: {} }); + } + + console.log(`[API:${requestId}] Config: ${JSON.stringify(config)}`); + + // Make request to Gemini + const geminiStartTime = Date.now(); + + const response = await ai.models.generateContent({ + model: MODEL_MAP[model], + contents: [ + { + role: "user", + parts: requestParts, + }, + ], + config, + ...(tools.length > 0 && { tools }), + }); + + const geminiDuration = Date.now() - geminiStartTime; + console.log(`[API:${requestId}] Gemini API completed in ${geminiDuration}ms`); + + // Extract image from response + const candidates = response.candidates; + + if (!candidates || candidates.length === 0) { + console.error(`[API:${requestId}] No candidates in Gemini response`); + return NextResponse.json( + { + success: false, + error: "No response from AI model", + }, + { status: 500 } + ); + } + + const parts = candidates[0].content?.parts; + console.log(`[API:${requestId}] Response parts: ${parts?.length || 0}`); + + if (!parts) { + console.error(`[API:${requestId}] No parts in Gemini candidate content`); + return NextResponse.json( + { + success: false, + error: "No content in response", + }, + { status: 500 } + ); + } + + // Find image part in response + for (const part of parts) { + if (part.inlineData && part.inlineData.data) { + const mimeType = part.inlineData.mimeType || "image/png"; + const imgData = part.inlineData.data; + const imageSizeKB = (imgData.length / 1024).toFixed(1); + + console.log(`[API:${requestId}] Output image: ${mimeType}, ${imageSizeKB}KB`); + + const dataUrl = `data:${mimeType};base64,${imgData}`; + + const responsePayload = { success: true, image: dataUrl }; + const responseSize = JSON.stringify(responsePayload).length; + const responseSizeMB = (responseSize / (1024 * 1024)).toFixed(2); + + if (responseSize > 4.5 * 1024 * 1024) { + console.warn(`[API:${requestId}] Response size (${responseSizeMB}MB) approaching Next.js 5MB limit`); + } + + console.log(`[API:${requestId}] SUCCESS - Returning ${responseSizeMB}MB payload`); + + return NextResponse.json(responsePayload); + } + } + + // If no image found, check for text error + for (const part of parts) { + if (part.text) { + console.error(`[API:${requestId}] Gemini returned text instead of image: ${part.text.substring(0, 100)}`); + return NextResponse.json( + { + success: false, + error: `Model returned text instead of image: ${part.text.substring(0, 200)}`, + }, + { status: 500 } + ); + } + } + + console.error(`[API:${requestId}] No image or text found in Gemini response`); + return NextResponse.json( + { + success: false, + error: "No image in response", + }, + { status: 500 } + ); +} diff --git a/src/app/api/generate/providers/index.ts b/src/app/api/generate/providers/index.ts new file mode 100644 index 00000000..9a1d9ebd --- /dev/null +++ b/src/app/api/generate/providers/index.ts @@ -0,0 +1,9 @@ +/** + * Provider barrel exports for Generate API Route + */ + +export { generateWithGemini } from "./gemini"; +export { generateWithReplicate } from "./replicate"; +export { clearFalInputMappingCache, generateWithFalQueue } from "./fal"; +export { generateWithKie } from "./kie"; +export { generateWithWaveSpeed } from "./wavespeed"; diff --git a/src/app/api/generate/providers/kie.ts b/src/app/api/generate/providers/kie.ts new file mode 100644 index 00000000..9ed5be40 --- /dev/null +++ b/src/app/api/generate/providers/kie.ts @@ -0,0 +1,823 @@ +/** + * Kie.ai Provider for Generate API Route + * + * Handles image/video generation using Kie.ai API. + * Supports standard createTask endpoint and Veo-specific endpoints. + */ + +import { GenerationInput, GenerationOutput } from "@/lib/providers/types"; +import { validateMediaUrl } from "@/utils/urlValidation"; + +const MAX_MEDIA_SIZE = 500 * 1024 * 1024; // 500MB +const MAX_UPLOAD_SIZE = 20 * 1024 * 1024; // 20MB + +/** + * Get default required parameters for a Kie model + * Many Kie models require specific parameters to be present even if not user-specified + */ +export function getKieModelDefaults(modelId: string): Record { + switch (modelId) { + // GPT Image models + case "gpt-image/1.5-text-to-image": + case "gpt-image/1.5-image-to-image": + return { + aspect_ratio: "3:2", + quality: "medium", + }; + + // Z-Image model + case "z-image": + return { + aspect_ratio: "1:1", + }; + + // Seedream models + case "seedream/4.5-text-to-image": + case "seedream/4.5-edit": + return { + aspect_ratio: "1:1", + quality: "basic", + }; + + // Nano Banana Pro (Kie) + case "nano-banana-pro": + return { + aspect_ratio: "1:1", + resolution: "1K", + }; + + // Flux-2 models + case "flux-2/pro-text-to-image": + case "flux-2/pro-image-to-image": + case "flux-2/flex-text-to-image": + case "flux-2/flex-image-to-image": + return { + aspect_ratio: "1:1", + }; + + // Grok Imagine image models + case "grok-imagine/text-to-image": + return { + aspect_ratio: "1:1", + }; + + case "grok-imagine/image-to-image": + return {}; + + // Grok Imagine video models + case "grok-imagine/text-to-video": + return { + aspect_ratio: "2:3", + duration: "6", + mode: "normal", + }; + + case "grok-imagine/image-to-video": + return { + aspect_ratio: "2:3", + duration: "6", + mode: "normal", + }; + + // Kling 2.6 video models + case "kling-2.6/text-to-video": + case "kling-2.6/image-to-video": + return { + aspect_ratio: "16:9", + duration: "5", + sound: true, + }; + + // Kling 2.6 motion control + case "kling-2.6/motion-control": + return { + mode: "720p", + character_orientation: "video", + }; + + // Kling 2.5 turbo models + case "kling/v2-5-turbo-text-to-video-pro": + case "kling/v2-5-turbo-image-to-video-pro": + return { + aspect_ratio: "16:9", + duration: "5", + cfg_scale: 0.5, + }; + + // Wan video models + case "wan/2-6-text-to-video": + case "wan/2-6-image-to-video": + return { + duration: "5", + resolution: "1080p", + }; + + case "wan/2-6-video-to-video": + return { + duration: "5", + resolution: "1080p", + }; + + // Topaz video upscale + case "topaz/video-upscale": + return { + upscale_factor: "2", + }; + + // Veo 3 models + case "veo3/text-to-video": + case "veo3/image-to-video": + case "veo3-fast/text-to-video": + case "veo3-fast/image-to-video": + return { + aspect_ratio: "16:9", + }; + + default: + return {}; + } +} + +/** + * Get the correct image input parameter name for a Kie model + */ +export function getKieImageInputKey(modelId: string): string { + // Model-specific parameter names + if (modelId === "nano-banana-pro") return "image_input"; + if (modelId === "seedream/4.5-edit") return "image_urls"; + if (modelId === "gpt-image/1.5-image-to-image") return "input_urls"; + // Flux-2 I2I models use input_urls + if (modelId === "flux-2/pro-image-to-image" || modelId === "flux-2/flex-image-to-image") return "input_urls"; + // Kling 2.5 turbo I2V uses singular image_url + if (modelId === "kling/v2-5-turbo-image-to-video-pro") return "image_url"; + // Kling 2.6 motion control uses input_urls + if (modelId === "kling-2.6/motion-control") return "input_urls"; + // Topaz video upscale uses video_url (singular) + if (modelId === "topaz/video-upscale") return "video_url"; + // Veo 3 models use imageUrls + if (modelId.startsWith("veo3")) return "imageUrls"; + // Default for most models + return "image_urls"; +} + + +/** + * Detect actual image type from binary data (magic bytes) + */ +export function detectImageType(buffer: Buffer): { mimeType: string; ext: string } { + // Check magic bytes + if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4E && buffer[3] === 0x47) { + return { mimeType: "image/png", ext: "png" }; + } + if (buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) { + return { mimeType: "image/jpeg", ext: "jpg" }; + } + if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46 && + buffer[8] === 0x57 && buffer[9] === 0x45 && buffer[10] === 0x42 && buffer[11] === 0x50) { + return { mimeType: "image/webp", ext: "webp" }; + } + if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) { + return { mimeType: "image/gif", ext: "gif" }; + } + // Default to PNG + return { mimeType: "image/png", ext: "png" }; +} + +/** + * Upload a base64 image to Kie.ai and get a URL + * Required for image-to-image models since Kie doesn't accept base64 directly + * Uses base64 upload endpoint (same as official Kie client) + */ +export async function uploadImageToKie( + requestId: string, + apiKey: string, + base64Image: string +): Promise { + // Extract mime type and data from data URL + let declaredMimeType = "image/png"; + let imageData = base64Image; + + if (base64Image.startsWith("data:")) { + const matches = base64Image.match(/^data:([^;]+);base64,(.+)$/); + if (matches) { + declaredMimeType = matches[1]; + imageData = matches[2]; + } + } + + // Convert base64 to binary to detect actual type + const binaryData = Buffer.from(imageData, "base64"); + + if (binaryData.length > MAX_UPLOAD_SIZE) { + throw new Error(`[API:${requestId}] Image too large to upload (${(binaryData.length / (1024 * 1024)).toFixed(1)}MB, max ${MAX_UPLOAD_SIZE / (1024 * 1024)}MB)`); + } + + // Detect actual image type from magic bytes (don't trust the declared MIME type) + const detected = detectImageType(binaryData); + const mimeType = detected.mimeType; + const ext = detected.ext; + + const filename = `upload_${Date.now()}.${ext}`; + + console.log(`[API:${requestId}] Uploading image to Kie.ai: ${filename} (${(binaryData.length / 1024).toFixed(1)}KB) [declared: ${declaredMimeType}, actual: ${mimeType}]`); + + // Use base64 upload endpoint (same as official Kie client) + // Format: data:{mime_type};base64,{data} + const dataUrl = `data:${mimeType};base64,${imageData}`; + + const response = await fetch("https://kieai.redpandaai.co/api/file-base64-upload", { + method: "POST", + headers: { + "Authorization": `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + base64Data: dataUrl, + uploadPath: "images", + fileName: filename, + }), + }); + + if (!response.ok) { + const errorText = await response.text(); + throw new Error(`Failed to upload image: ${response.status} - ${errorText}`); + } + + const result = await response.json(); + console.log(`[API:${requestId}] Kie upload response:`, JSON.stringify(result).substring(0, 300)); + + // Check for error in response + if (result.code && result.code !== 200 && !result.success) { + throw new Error(`Upload failed: ${result.msg || 'Unknown error'}`); + } + + // Response format: { success: true, code: 200, data: { downloadUrl: "...", fileName: "...", fileSize: 123 } } + const downloadUrl = result.data?.downloadUrl || result.downloadUrl || result.url; + + if (!downloadUrl) { + console.error(`[API:${requestId}] Upload response has no URL:`, result); + throw new Error(`No download URL in upload response. Response: ${JSON.stringify(result).substring(0, 200)}`); + } + + console.log(`[API:${requestId}] Image uploaded: ${downloadUrl.substring(0, 80)}...`); + return downloadUrl; +} + +/** + * Poll Kie.ai task status until completion + */ +export async function pollKieTaskCompletion( + requestId: string, + apiKey: string, + taskId: string, +): Promise<{ success: boolean; data?: Record; error?: string }> { + const maxWaitTime = 10 * 60 * 1000; // 10 minutes for video + const pollInterval = 2000; // 2 seconds + const startTime = Date.now(); + let lastStatus = ""; + + const pollUrl = `https://api.kie.ai/api/v1/jobs/recordInfo?taskId=${encodeURIComponent(taskId)}`; + + while (true) { + if (Date.now() - startTime > maxWaitTime) { + return { success: false, error: "Generation timed out after 10 minutes" }; + } + + await new Promise(resolve => setTimeout(resolve, pollInterval)); + + const response = await fetch(pollUrl, { + headers: { + "Authorization": `Bearer ${apiKey}`, + }, + }); + + if (!response.ok) { + return { success: false, error: `Failed to poll status: ${response.status}` }; + } + + const result = await response.json(); + // Kie API returns "state" in result.data.state (not "status") + const state = (result.data?.state || result.state || result.status || "").toUpperCase(); + + if (state !== lastStatus) { + console.log(`[API:${requestId}] Kie task state: ${state}`); + lastStatus = state; + } + + if (state === "SUCCESS" || state === "COMPLETED") { + return { success: true, data: result.data || result }; + } + + if (state === "FAIL" || state === "FAILED" || state === "ERROR") { + const errorMessage = result.data?.failMsg || result.data?.errorMessage || result.error || result.message || "Generation failed"; + return { success: false, error: errorMessage }; + } + + // Continue polling for: WAITING, QUEUING, GENERATING, PROCESSING, etc. + } +} + + +export function isVeoModel(modelId: string): boolean { + return modelId.startsWith("veo3/") || modelId.startsWith("veo3-fast/"); +} + +export function getVeoApiModelId(modelId: string): string { + if (modelId.startsWith("veo3-fast/")) return "veo3_fast"; + return "veo3"; +} + +export async function pollVeoTaskCompletion( + requestId: string, + apiKey: string, + taskId: string, +): Promise<{ success: boolean; data?: Record; error?: string }> { + const maxWaitTime = 10 * 60 * 1000; + const pollInterval = 2000; + const startTime = Date.now(); + let lastStatus = -1; + + const pollUrl = `https://api.kie.ai/api/v1/veo/record-info?taskId=${encodeURIComponent(taskId)}`; + + while (true) { + if (Date.now() - startTime > maxWaitTime) { + return { success: false, error: "Generation timed out after 10 minutes" }; + } + await new Promise(resolve => setTimeout(resolve, pollInterval)); + + const response = await fetch(pollUrl, { + headers: { "Authorization": `Bearer ${apiKey}` }, + }); + if (!response.ok) { + return { success: false, error: `Failed to poll status: ${response.status}` }; + } + + const result = await response.json(); + const successFlag = result.data?.successFlag ?? -1; + + if (successFlag !== lastStatus) { + console.log(`[API:${requestId}] Veo task successFlag: ${successFlag}`); + lastStatus = successFlag; + } + + if (successFlag === 1) { + return { success: true, data: result.data }; + } + if (successFlag === 2 || successFlag === 3) { + const errorMessage = result.data?.errorMessage || "Generation failed"; + return { success: false, error: errorMessage }; + } + // successFlag === 0 means still generating, continue polling + } +} + +/** + * Generate image/video using Kie.ai API + */ +export async function generateWithKie( + requestId: string, + apiKey: string, + input: GenerationInput +): Promise { + const modelId = input.model.id; + + console.log(`[API:${requestId}] Kie.ai generation - Model: ${modelId}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); + + // Build the input object (all parameters go inside "input" for Kie API) + // Start with model-specific required defaults + const modelDefaults = getKieModelDefaults(modelId); + const inputParams: Record = { ...modelDefaults }; + + // Add prompt + if (input.prompt) { + inputParams.prompt = input.prompt; + } + + // Add model parameters (user params override defaults) + if (input.parameters) { + Object.assign(inputParams, input.parameters); + } + + // GPT Image 1.5 does NOT support 'size' parameter - only 'aspect_ratio' + // Remove any stale 'size' values from old workflow data + if (modelId.startsWith("gpt-image/1.5")) { + delete inputParams.size; + } + + // Handle dynamic inputs FIRST (from schema-mapped connections) - these take priority + // Track which image keys dynamicInputs already handled to avoid double-uploads + const handledImageKeys = new Set(); + + if (input.dynamicInputs) { + for (const [key, value] of Object.entries(input.dynamicInputs)) { + if (value !== null && value !== undefined && value !== '') { + // Check if this is an image input that needs uploading + if (typeof value === 'string' && value.startsWith('data:image')) { + // Single data URL - upload it + const url = await uploadImageToKie(requestId, apiKey, value); + // Singular keys get a string, plural keys get an array + if (key === "image_url" || key === "video_url" || key === "tail_image_url") { + inputParams[key] = url; + } else { + inputParams[key] = [url]; + } + handledImageKeys.add(key); + } else if (Array.isArray(value)) { + // Array of values - check if they're data URLs that need uploading + const processedArray: string[] = []; + for (const item of value) { + if (typeof item === 'string' && item.startsWith('data:image')) { + const url = await uploadImageToKie(requestId, apiKey, item); + processedArray.push(url); + } else if (typeof item === 'string' && item.startsWith('http')) { + processedArray.push(item); + } else if (typeof item === 'string') { + processedArray.push(item); + } + } + if (processedArray.length > 0) { + // Singular keys get first element, plural keys get full array + if (key === "image_url" || key === "video_url" || key === "tail_image_url") { + inputParams[key] = processedArray[0]; + } else { + inputParams[key] = processedArray; + } + handledImageKeys.add(key); + } + } else { + inputParams[key] = value; + } + } + } + } + + // Handle image inputs (fallback - only if dynamicInputs didn't already set the image key) + const imageKey = getKieImageInputKey(modelId); + if (input.images && input.images.length > 0 && !handledImageKeys.has(imageKey)) { + // Upload images to get URLs (Kie requires URLs, not base64) + const imageUrls: string[] = []; + for (const image of input.images) { + if (image.startsWith("http")) { + imageUrls.push(image); + } else { + // Upload base64 image + const url = await uploadImageToKie(requestId, apiKey, image); + imageUrls.push(url); + } + } + + // Some models use singular string, others use arrays + if (imageKey === "image_url" || imageKey === "video_url") { + inputParams[imageKey] = imageUrls[0]; + } else { + inputParams[imageKey] = imageUrls; + } + } + + // Veo 3 models use a different API endpoint and request format + if (isVeoModel(modelId)) { + const veoBody: Record = { + prompt: inputParams.prompt, + model: getVeoApiModelId(modelId), + aspect_ratio: inputParams.aspect_ratio || "16:9", + }; + + // Add image URLs if present (for image-to-video) + if (inputParams.imageUrls) { + veoBody.imageUrls = Array.isArray(inputParams.imageUrls) + ? inputParams.imageUrls + : [inputParams.imageUrls]; + } + + // Add optional seed + if (inputParams.seeds !== undefined) { + veoBody.seeds = inputParams.seeds; + } + + const veoUrl = "https://api.kie.ai/api/v1/veo/generate"; + console.log(`[API:${requestId}] Calling Veo API: ${veoUrl}`); + console.log(`[API:${requestId}] Veo request body:`, JSON.stringify(veoBody, null, 2)); + + const createResponse = await fetch(veoUrl, { + method: "POST", + headers: { + "Authorization": `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(veoBody), + }); + + if (!createResponse.ok) { + const errorText = await createResponse.text(); + let errorDetail = errorText; + try { + const errorJson = JSON.parse(errorText); + errorDetail = errorJson.message || errorJson.error || errorJson.detail || errorText; + } catch { + // Keep original text + } + if (createResponse.status === 429) { + return { success: false, error: `${input.model.name}: Rate limit exceeded. Try again in a moment.` }; + } + return { success: false, error: `${input.model.name}: ${errorDetail}` }; + } + + const createResult = await createResponse.json(); + if (createResult.code && createResult.code !== 200) { + return { success: false, error: `${input.model.name}: ${createResult.msg || "API error"}` }; + } + + const taskId = createResult.data?.taskId || createResult.taskId; + if (!taskId) { + console.error(`[API:${requestId}] No taskId in Veo response:`, createResult); + return { success: false, error: "No task ID in Veo response" }; + } + + console.log(`[API:${requestId}] Veo task created: ${taskId}`); + + // Poll with Veo-specific polling + const pollResult = await pollVeoTaskCompletion(requestId, apiKey, taskId); + if (!pollResult.success) { + return { success: false, error: `${input.model.name}: ${pollResult.error}` }; + } + + // Extract video URL from Veo response format + const data = pollResult.data; + let mediaUrl: string | null = null; + + const responseObj = data?.response as Record | undefined; + const resultUrls = (responseObj?.resultUrls || data?.resultUrls) as string[] | undefined; + if (resultUrls && resultUrls.length > 0) { + mediaUrl = resultUrls[0]; + } + + if (!mediaUrl) { + console.error(`[API:${requestId}] No media URL found in Veo response:`, data); + return { success: false, error: "No output URL in Veo response" }; + } + + // Validate URL before fetching + const mediaUrlCheck = validateMediaUrl(mediaUrl); + if (!mediaUrlCheck.valid) { + return { success: false, error: `Invalid media URL: ${mediaUrlCheck.error}` }; + } + + // Fetch the video and convert to base64 + console.log(`[API:${requestId}] Fetching Veo output from: ${mediaUrl.substring(0, 80)}...`); + const mediaResponse = await fetch(mediaUrl); + if (!mediaResponse.ok) { + return { success: false, error: `Failed to fetch output: ${mediaResponse.status}` }; + } + + const mediaContentLength = parseInt(mediaResponse.headers.get("content-length") || "0", 10); + if (mediaContentLength > MAX_MEDIA_SIZE) { + return { success: false, error: `Media too large: ${(mediaContentLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; + } + + const contentType = mediaResponse.headers.get("content-type") || "video/mp4"; + const mediaArrayBuffer = await mediaResponse.arrayBuffer(); + if (mediaArrayBuffer.byteLength > MAX_MEDIA_SIZE) { + return { success: false, error: `Media too large: ${(mediaArrayBuffer.byteLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; + } + const mediaSizeMB = mediaArrayBuffer.byteLength / (1024 * 1024); + + console.log(`[API:${requestId}] Veo output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); + + // For very large videos (>20MB), return URL only (data left empty for consumers) + if (mediaSizeMB > 20) { + console.log(`[API:${requestId}] SUCCESS - Returning URL for large Veo video`); + return { + success: true, + outputs: [{ type: "video", data: "", url: mediaUrl }], + }; + } + + const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); + console.log(`[API:${requestId}] SUCCESS - Returning Veo video`); + return { + success: true, + outputs: [{ type: "video", data: `data:${contentType};base64,${mediaBase64}`, url: mediaUrl }], + }; + } + + // All remaining Kie models use the standard createTask endpoint + const requestBody: Record = { + model: modelId, + input: inputParams, + }; + + const createUrl = "https://api.kie.ai/api/v1/jobs/createTask"; + + console.log(`[API:${requestId}] Calling Kie.ai API: ${createUrl}`); + // Log full request body for debugging (truncate very long prompts) + const bodyForLogging = { ...requestBody }; + if (bodyForLogging.input && typeof bodyForLogging.input === 'object') { + const inputForLogging = { ...(bodyForLogging.input as Record) }; + if (typeof inputForLogging.prompt === 'string' && (inputForLogging.prompt as string).length > 200) { + inputForLogging.prompt = (inputForLogging.prompt as string).substring(0, 200) + '...[truncated]'; + } + bodyForLogging.input = inputForLogging; + } + console.log(`[API:${requestId}] Request body:`, JSON.stringify(bodyForLogging, null, 2)); + + // Create task + const createResponse = await fetch(createUrl, { + method: "POST", + headers: { + "Authorization": `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(requestBody), + }); + + if (!createResponse.ok) { + const errorText = await createResponse.text(); + let errorDetail = errorText; + try { + const errorJson = JSON.parse(errorText); + errorDetail = errorJson.message || errorJson.error || errorJson.detail || errorText; + } catch { + // Keep original text + } + + if (createResponse.status === 429) { + return { + success: false, + error: `${input.model.name}: Rate limit exceeded. Try again in a moment.`, + }; + } + + return { + success: false, + error: `${input.model.name}: ${errorDetail}`, + }; + } + + const createResult = await createResponse.json(); + + // Kie API returns HTTP 200 even on errors, check the response code + if (createResult.code && createResult.code !== 200) { + const errorMsg = createResult.msg || createResult.message || "API error"; + console.error(`[API:${requestId}] Kie API error (code ${createResult.code}):`, errorMsg); + return { + success: false, + error: `${input.model.name}: ${errorMsg}`, + }; + } + + const taskId = createResult.taskId || createResult.data?.taskId || createResult.id; + + if (!taskId) { + console.error(`[API:${requestId}] No taskId in Kie response:`, createResult); + return { + success: false, + error: "No task ID in response", + }; + } + + console.log(`[API:${requestId}] Kie task created: ${taskId}`); + + // Poll for completion + const pollResult = await pollKieTaskCompletion(requestId, apiKey, taskId); + + if (!pollResult.success) { + return { + success: false, + error: `${input.model.name}: ${pollResult.error}`, + }; + } + + // Extract output URL from result + // Kie API returns: { data: { status: "success", resultJson: { resultUrls: ["url1", "url2"] } } } + const data = pollResult.data; + let mediaUrl: string | null = null; + let isVideo = false; + + console.log(`[API:${requestId}] Kie poll result data:`, JSON.stringify(data).substring(0, 500)); + + // Try various response formats - Kie uses resultJson.resultUrls + // Note: resultJson is often a JSON string that needs parsing + if (data) { + let resultJson = data.resultJson as Record | string | undefined; + + // Parse resultJson if it's a string (Kie API returns it as escaped JSON string) + if (typeof resultJson === 'string') { + try { + resultJson = JSON.parse(resultJson) as Record; + } catch { + // Not valid JSON, keep as-is + resultJson = undefined; + } + } + + const resultUrls = ((resultJson as Record | undefined)?.resultUrls || data.resultUrls) as string[] | undefined; + + if (resultUrls && resultUrls.length > 0) { + mediaUrl = resultUrls[0]; + // Check if it's a video based on URL + isVideo = mediaUrl.includes('.mp4') || mediaUrl.includes('.webm') || mediaUrl.includes('video'); + } + // Fallback to other formats + else if (data.videoUrl) { + mediaUrl = data.videoUrl as string; + isVideo = true; + } else if (data.video_url) { + mediaUrl = data.video_url as string; + isVideo = true; + } else if (data.output && typeof data.output === 'string' && (data.output as string).includes('.mp4')) { + mediaUrl = data.output as string; + isVideo = true; + } + // Image outputs + else if (data.imageUrl) { + mediaUrl = data.imageUrl as string; + } else if (data.image_url) { + mediaUrl = data.image_url as string; + } else if (data.output && typeof data.output === 'string') { + mediaUrl = data.output as string; + } else if (data.url) { + mediaUrl = data.url as string; + } else if (Array.isArray(data.images) && data.images.length > 0) { + mediaUrl = (data.images[0] as { url?: string })?.url || data.images[0] as string; + } + } + + if (!mediaUrl) { + console.error(`[API:${requestId}] No media URL found in Kie response:`, data); + return { + success: false, + error: "No output URL in response", + }; + } + + // Detect video from URL if not already detected + if (!isVideo && (mediaUrl.includes('.mp4') || mediaUrl.includes('.webm') || mediaUrl.includes('video'))) { + isVideo = true; + } + + // Validate URL before fetching + const mediaUrlCheck = validateMediaUrl(mediaUrl); + if (!mediaUrlCheck.valid) { + return { success: false, error: `Invalid media URL: ${mediaUrlCheck.error}` }; + } + + // Fetch the media and convert to base64 + console.log(`[API:${requestId}] Fetching output from: ${mediaUrl.substring(0, 80)}...`); + const mediaResponse = await fetch(mediaUrl); + + if (!mediaResponse.ok) { + return { + success: false, + error: `Failed to fetch output: ${mediaResponse.status}`, + }; + } + + // Check file size before downloading body + const mediaContentLength = parseInt(mediaResponse.headers.get("content-length") || "0", 10); + if (mediaContentLength > MAX_MEDIA_SIZE) { + return { success: false, error: `Media too large: ${(mediaContentLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; + } + + const contentType = mediaResponse.headers.get("content-type") || (isVideo ? "video/mp4" : "image/png"); + if (contentType.startsWith("video/")) { + isVideo = true; + } + + const mediaArrayBuffer = await mediaResponse.arrayBuffer(); + if (mediaArrayBuffer.byteLength > MAX_MEDIA_SIZE) { + return { success: false, error: `Media too large: ${(mediaArrayBuffer.byteLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; + } + const mediaSizeBytes = mediaArrayBuffer.byteLength; + const mediaSizeMB = mediaSizeBytes / (1024 * 1024); + + console.log(`[API:${requestId}] Output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); + + // For very large videos (>20MB), return URL only (data left empty for consumers) + if (isVideo && mediaSizeMB > 20) { + console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); + return { + success: true, + outputs: [ + { + type: "video", + data: "", + url: mediaUrl, + }, + ], + }; + } + + const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); + console.log(`[API:${requestId}] SUCCESS - Returning ${isVideo ? "video" : "image"}`); + + return { + success: true, + outputs: [ + { + type: isVideo ? "video" : "image", + data: `data:${contentType};base64,${mediaBase64}`, + url: mediaUrl, + }, + ], + }; +} diff --git a/src/app/api/generate/providers/replicate.ts b/src/app/api/generate/providers/replicate.ts new file mode 100644 index 00000000..482d4d78 --- /dev/null +++ b/src/app/api/generate/providers/replicate.ts @@ -0,0 +1,318 @@ +/** + * Replicate Provider for Generate API Route + * + * Handles image/video generation using Replicate's prediction API. + */ + +import { GenerationInput, GenerationOutput } from "@/lib/providers/types"; +import { validateMediaUrl } from "@/utils/urlValidation"; +import { + getParameterTypesFromSchema, + coerceParameterTypes, + getInputMappingFromSchema, +} from "../schemaUtils"; + +/** + * Generate image using Replicate API + */ +export async function generateWithReplicate( + requestId: string, + apiKey: string, + input: GenerationInput +): Promise { + console.log(`[API:${requestId}] Replicate generation - Model: ${input.model.id}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); + + const REPLICATE_API_BASE = "https://api.replicate.com/v1"; + + // Get the latest version of the model + const modelId = input.model.id; + const [owner, name] = modelId.split("/"); + + if (!owner || !name) { + return { + success: false, + error: `Invalid Replicate model ID "${modelId}": expected "owner/name" format`, + }; + } + + // First, get the model to find the latest version + const modelResponse = await fetch( + `${REPLICATE_API_BASE}/models/${owner}/${name}`, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + } + ); + + if (!modelResponse.ok) { + return { + success: false, + error: `Failed to get model info: ${modelResponse.status}`, + }; + } + + const modelData = await modelResponse.json(); + const version = modelData.latest_version?.id; + + if (!version) { + return { + success: false, + error: "Model has no available version", + }; + } + + const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; + console.log(`[API:${requestId}] Model version: ${version}, Dynamic inputs: ${hasDynamicInputs ? Object.keys(input.dynamicInputs!).join(", ") : "none"}`); + + // Get schema for type coercion and input mapping + const schema = modelData.latest_version?.openapi_schema as Record | undefined; + const parameterTypes = getParameterTypesFromSchema(schema); + + // Build input for the prediction - parameters are applied per-path below to avoid double-spreading + const predictionInput: Record = {}; + + // Add dynamic inputs if provided (these come from schema-mapped connections) + if (hasDynamicInputs) { + // Apply coerced parameters first, then dynamic inputs override + Object.assign(predictionInput, coerceParameterTypes(input.parameters, parameterTypes)); + const { schemaArrayParams } = getInputMappingFromSchema(schema); + + // Apply array wrapping based on schema type + for (const [key, value] of Object.entries(input.dynamicInputs!)) { + if (value !== null && value !== undefined && value !== '') { + if (schemaArrayParams.has(key) && !Array.isArray(value)) { + predictionInput[key] = [value]; // Wrap in array + } else if (!schemaArrayParams.has(key) && Array.isArray(value)) { + predictionInput[key] = value[0]; // Unwrap array to single value + } else { + predictionInput[key] = value; + } + } + } + } else { + // Fallback: use schema to map generic input names to model-specific parameter names + const { paramMap, arrayParams } = getInputMappingFromSchema(schema); + + // Map prompt input + if (input.prompt) { + const promptParam = paramMap.prompt || "prompt"; + predictionInput[promptParam] = input.prompt; + } + + // Map image input - use array or string format based on schema + if (input.images && input.images.length > 0) { + const imageParam = paramMap.image || "image"; + if (arrayParams.has("image")) { + predictionInput[imageParam] = input.images; + } else { + predictionInput[imageParam] = input.images[0]; + } + } + + // Map any parameters that might need renaming (use coerced values) + const coercedParams = coerceParameterTypes(input.parameters, parameterTypes); + for (const [key, value] of Object.entries(coercedParams)) { + const mappedKey = paramMap[key] || key; + predictionInput[mappedKey] = value; + } + } + + // Create a prediction + const createResponse = await fetch(`${REPLICATE_API_BASE}/predictions`, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + version, + input: predictionInput, + }), + }); + + if (!createResponse.ok) { + const errorText = await createResponse.text(); + let errorDetail = errorText; + try { + const errorJson = JSON.parse(errorText); + errorDetail = errorJson.detail || errorJson.message || errorJson.error || errorText; + } catch { + // Keep original text if not JSON + } + + // Handle rate limits + if (createResponse.status === 429) { + return { + success: false, + error: `${input.model.name}: Rate limit exceeded. Try again in a moment.`, + }; + } + + return { + success: false, + error: `${input.model.name}: ${errorDetail}`, + }; + } + + const prediction = await createResponse.json(); + console.log(`[API:${requestId}] Prediction created: ${prediction.id}`); + + // Poll for completion — video models get a longer timeout + const isVideoModel = input.model.capabilities.some(c => c.includes("video")); + const maxWaitTime = isVideoModel ? 10 * 60 * 1000 : 5 * 60 * 1000; + const pollInterval = 1000; // 1 second + const startTime = Date.now(); + + let currentPrediction = prediction; + let lastStatus = ""; + + while ( + currentPrediction.status !== "succeeded" && + currentPrediction.status !== "failed" && + currentPrediction.status !== "canceled" + ) { + if (Date.now() - startTime > maxWaitTime) { + return { + success: false, + error: `${input.model.name}: Generation timed out after ${maxWaitTime / 60000} minutes.`, + }; + } + + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + + const pollResponse = await fetch( + `${REPLICATE_API_BASE}/predictions/${currentPrediction.id}`, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + } + ); + + if (!pollResponse.ok) { + return { + success: false, + error: `Failed to poll prediction: ${pollResponse.status}`, + }; + } + + currentPrediction = await pollResponse.json(); + if (currentPrediction.status !== lastStatus) { + console.log(`[API:${requestId}] Prediction status: ${currentPrediction.status}`); + lastStatus = currentPrediction.status; + } + } + + if (currentPrediction.status === "failed") { + const failureReason = currentPrediction.error || "Prediction failed"; + return { + success: false, + error: `${input.model.name}: ${failureReason}`, + }; + } + + if (currentPrediction.status === "canceled") { + return { + success: false, + error: "Prediction was canceled", + }; + } + + // Extract output + const output = currentPrediction.output; + if (!output) { + return { + success: false, + error: "No output from prediction", + }; + } + + // Output can be a single URL string or an array — filter to valid strings only + const rawOutputs = Array.isArray(output) ? output : [output]; + const outputUrls: string[] = rawOutputs.filter( + (v): v is string => typeof v === "string" && v.length > 0 + ); + + if (outputUrls.length === 0) { + return { + success: false, + error: "No output from prediction", + }; + } + + // Fetch the first output and convert to base64 + const mediaUrl = outputUrls[0]; + + // Validate URL before fetching (SSRF protection) + const mediaUrlCheck = validateMediaUrl(mediaUrl); + if (!mediaUrlCheck.valid) { + console.error(`[API:${requestId}] Invalid media URL from Replicate: ${mediaUrl}`); + return { success: false, error: `Invalid media URL: ${mediaUrlCheck.error}` }; + } + + console.log(`[API:${requestId}] Fetching output from: ${mediaUrl.substring(0, 80)}...`); + const mediaResponse = await fetch(mediaUrl); + + if (!mediaResponse.ok) { + return { + success: false, + error: `Failed to fetch output: ${mediaResponse.status}`, + }; + } + + // Check if this is a 3D model — return URL directly (GLB files are binary) + const is3DModel = input.model.capabilities.some(c => c.includes("3d")); + if (is3DModel) { + console.log(`[API:${requestId}] SUCCESS - Returning 3D model URL`); + return { + success: true, + outputs: [ + { + type: "3d", + data: "", + url: mediaUrl, + }, + ], + }; + } + + // Determine MIME type from response + const contentType = mediaResponse.headers.get("content-type") || "image/png"; + const isVideo = contentType.startsWith("video/"); + + const mediaArrayBuffer = await mediaResponse.arrayBuffer(); + const mediaSizeBytes = mediaArrayBuffer.byteLength; + const mediaSizeMB = mediaSizeBytes / (1024 * 1024); + + console.log(`[API:${requestId}] Output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); + + // For very large videos (>20MB), return URL only (data left empty for consumers) + if (isVideo && mediaSizeMB > 20) { + console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); + return { + success: true, + outputs: [ + { + type: "video", + data: "", + url: mediaUrl, + }, + ], + }; + } + + const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); + console.log(`[API:${requestId}] SUCCESS - Returning ${isVideo ? "video" : "image"}`); + + return { + success: true, + outputs: [ + { + type: isVideo ? "video" : "image", + data: `data:${contentType};base64,${mediaBase64}`, + url: mediaUrl, + }, + ], + }; +} diff --git a/src/app/api/generate/providers/wavespeed.ts b/src/app/api/generate/providers/wavespeed.ts new file mode 100644 index 00000000..82febf22 --- /dev/null +++ b/src/app/api/generate/providers/wavespeed.ts @@ -0,0 +1,412 @@ +/** + * WaveSpeed Provider for Generate API Route + * + * Handles image/video generation using WaveSpeed API. + * Uses async task submission + polling. + */ + +import { GenerationInput, GenerationOutput } from "@/lib/providers/types"; +import { validateMediaUrl } from "@/utils/urlValidation"; + +type WaveSpeedStatus = "created" | "pending" | "processing" | "completed" | "failed"; + +/** + * WaveSpeed submit response + * Format: { code: 200, message: "success", data: { id, model, status, urls, created_at } } + */ +interface WaveSpeedSubmitResponse { + code?: number; + message?: string; + data?: { + id: string; + model?: string; + status?: WaveSpeedStatus; + urls?: { + get?: string; + }; + created_at?: string; + }; + // Fallback fields for other response formats + id?: string; + status?: WaveSpeedStatus; + error?: string; +} + +/** + * WaveSpeed prediction/poll response (inner data object) + */ +interface WaveSpeedPredictionData { + id: string; + status: WaveSpeedStatus; + outputs?: string[]; + output?: { + images?: string[]; + videos?: string[]; + }; + timings?: { + inference?: number; + }; + created_at?: string; + error?: string; +} + +/** + * WaveSpeed prediction/poll response wrapper + * Format: { code: 200, message: "success", data: { id, status, outputs, ... } } + */ +interface WaveSpeedPredictionResponse { + code?: number; + message?: string; + data?: WaveSpeedPredictionData; + // Fallback: some responses might have fields at top level + id?: string; + status?: WaveSpeedStatus; + outputs?: string[]; + error?: string; +} + +/** + * Generate image/video using WaveSpeed API + * Uses async task submission + polling + */ +export async function generateWithWaveSpeed( + requestId: string, + apiKey: string, + input: GenerationInput +): Promise { + console.log(`[API:${requestId}] WaveSpeed generation - Model: ${input.model.id}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); + + const WAVESPEED_API_BASE = "https://api.wavespeed.ai/api/v3"; + const modelId = input.model.id; + + // Validate modelId to prevent path traversal + if (/[^a-zA-Z0-9\-_/.]/.test(modelId) || modelId.includes('..')) { + return { success: false, error: `Invalid model ID: ${modelId}` }; + } + + const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; + console.log(`[API:${requestId}] Dynamic inputs: ${hasDynamicInputs ? Object.keys(input.dynamicInputs!).join(", ") : "none"}`); + + // Determine output type from model capabilities + const is3DModel = input.model.capabilities.some(c => c.includes("3d")); + const isVideoModel = input.model.capabilities.includes("text-to-video") || + input.model.capabilities.includes("image-to-video"); + + // Build WaveSpeed payload — spread parameters first so explicit prompt wins + const payload: Record = { + ...input.parameters, + prompt: input.prompt, + }; + + // Apply dynamic inputs (schema-mapped connections) + // These have the correct parameter names from the schema (e.g., "images" for edit models) + if (hasDynamicInputs) { + for (const [key, value] of Object.entries(input.dynamicInputs!)) { + if (value !== null && value !== undefined && value !== '') { + // If the key is "images" and value is not an array, wrap it + if (key === "images" && !Array.isArray(value)) { + payload[key] = [value]; + } else if (key !== "images" && Array.isArray(value)) { + // Unwrap array to single value for non-array params + payload[key] = value[0]; + } else { + payload[key] = value; + } + } + } + } else if (input.images && input.images.length > 0) { + // Fallback: if no dynamic inputs but images array is provided + // Use "image" for single image (default WaveSpeed format) + payload.image = input.images[0]; + } + + console.log(`[API:${requestId}] Submitting to WaveSpeed with inputs: ${Object.keys(payload).join(", ")}`); + + // Submit task + // Model ID goes directly in the URL path (slashes are part of the path) + const submitUrl = `${WAVESPEED_API_BASE}/${modelId}`; + console.log(`[API:${requestId}] WaveSpeed submit URL: ${submitUrl}`); + + const submitResponse = await fetch(submitUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify(payload), + }); + + if (!submitResponse.ok) { + const errorText = await submitResponse.text(); + let errorDetail = errorText || `HTTP ${submitResponse.status}`; + try { + const errorJson = JSON.parse(errorText); + errorDetail = errorJson.error || errorJson.message || errorJson.detail || errorText || `HTTP ${submitResponse.status}`; + } catch { + // Keep original text + } + + console.error(`[API:${requestId}] WaveSpeed submit failed: ${submitResponse.status} - ${errorDetail}`); + + if (submitResponse.status === 429) { + return { + success: false, + error: `${input.model.name || 'WaveSpeed'}: Rate limit exceeded. Try again in a moment.`, + }; + } + + return { + success: false, + error: `${input.model.name || 'WaveSpeed'}: ${errorDetail}`, + }; + } + + const submitResult: WaveSpeedSubmitResponse = await submitResponse.json(); + console.log(`[API:${requestId}] WaveSpeed submit response:`, JSON.stringify(submitResult).substring(0, 500)); + + const taskId = submitResult.data?.id || submitResult.id; + // Use the polling URL provided by the API if available, with SSRF validation + let providedPollUrl: string | undefined = submitResult.data?.urls?.get; + if (providedPollUrl) { + const pollUrlCheck = validateMediaUrl(providedPollUrl); + if (!pollUrlCheck.valid || !providedPollUrl.startsWith('https://api.wavespeed.ai')) { + console.warn(`[API:${requestId}] WaveSpeed provided invalid poll URL: ${providedPollUrl} — falling back to constructed URL`); + providedPollUrl = undefined; + } + } + + if (!taskId) { + console.error(`[API:${requestId}] No task ID in WaveSpeed submit response`); + return { + success: false, + error: "WaveSpeed: No task ID returned from API", + }; + } + + console.log(`[API:${requestId}] WaveSpeed task submitted: ${taskId}`); + if (providedPollUrl) { + console.log(`[API:${requestId}] WaveSpeed provided poll URL: ${providedPollUrl}`); + } + + // Poll for completion using the URL from the API response, or construct it + // Status flow: created → processing → completed/failed + const maxWaitTime = 5 * 60 * 1000; // 5 minutes + const pollInterval = 1000; // 1 second + const startTime = Date.now(); + let lastStatus = ""; + + let resultData: WaveSpeedPredictionResponse | null = null; + + while (true) { + if (Date.now() - startTime > maxWaitTime) { + console.error(`[API:${requestId}] WaveSpeed task timed out after 5 minutes`); + return { + success: false, + error: `${input.model.name}: Generation timed out after 5 minutes`, + }; + } + + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + + try { + // Use provided poll URL if available, otherwise construct it + const pollUrl = providedPollUrl || `${WAVESPEED_API_BASE}/predictions/${taskId}/result`; + const pollResponse = await fetch( + pollUrl, + { + headers: { + Authorization: `Bearer ${apiKey}`, + }, + } + ); + + // Log poll response status for debugging + const elapsedSec = Math.round((Date.now() - startTime) / 1000); + console.log(`[API:${requestId}] WaveSpeed poll (${elapsedSec}s): ${pollResponse.status} from ${pollUrl}`); + + // 404 means result not ready yet - continue polling + if (pollResponse.status === 404) { + lastStatus = "pending"; + continue; + } + + if (!pollResponse.ok) { + const errorText = await pollResponse.text(); + let errorDetail = errorText || `HTTP ${pollResponse.status}`; + try { + const errorJson = JSON.parse(errorText); + errorDetail = errorJson.error || errorJson.message || errorJson.detail || errorDetail; + } catch { + // Keep original text + } + console.error(`[API:${requestId}] WaveSpeed poll failed: ${pollResponse.status} - ${errorDetail}`); + return { + success: false, + error: `${input.model.name}: ${errorDetail}`, + }; + } + + const pollData: WaveSpeedPredictionResponse = await pollResponse.json(); + console.log(`[API:${requestId}] WaveSpeed poll data:`, JSON.stringify(pollData).substring(0, 300)); + + // Extract status from nested data object (WaveSpeed wraps response in { code, message, data: {...} }) + const currentStatus = pollData.data?.status || pollData.status; + const currentError = pollData.data?.error || pollData.error; + + // Log status changes + if (currentStatus !== lastStatus) { + console.log(`[API:${requestId}] WaveSpeed status changed: ${lastStatus} → ${currentStatus}`); + lastStatus = currentStatus || ""; + } + + // Check if task is complete + if (currentStatus === "completed") { + console.log(`[API:${requestId}] WaveSpeed task completed`); + resultData = pollData; + break; + } + + // Check if task failed + if (currentStatus === "failed") { + const failureReason = currentError || pollData.message || "Generation failed"; + console.error(`[API:${requestId}] WaveSpeed task failed: ${failureReason}`); + return { + success: false, + error: `${input.model.name}: ${failureReason}`, + }; + } + + // Continue polling for "created" or "processing" status + } catch (pollError) { + const message = pollError instanceof Error ? pollError.message : String(pollError); + console.error(`[API:${requestId}] WaveSpeed poll error: ${message}`); + return { + success: false, + error: `${input.model.name}: ${message}`, + }; + } + } + + // Safety check (should never happen since we break on completed) + if (!resultData) { + return { + success: false, + error: `${input.model.name}: No result received`, + }; + } + + // Extract outputs - WaveSpeed wraps response in { code, message, data: { outputs: [...] } } + let outputUrls: string[] = []; + const resultDataInner = resultData.data; + + // Format 1: data.outputs array (standard WaveSpeed format) + if (resultDataInner?.outputs && Array.isArray(resultDataInner.outputs) && resultDataInner.outputs.length > 0) { + outputUrls = resultDataInner.outputs; + } + // Format 2: data.output object with images/videos arrays + else if (resultDataInner?.output) { + if (isVideoModel && resultDataInner.output.videos && resultDataInner.output.videos.length > 0) { + outputUrls = resultDataInner.output.videos; + } else if (resultDataInner.output.images && resultDataInner.output.images.length > 0) { + outputUrls = resultDataInner.output.images; + } + } + // Format 3: Fallback - outputs at top level (unlikely but safe) + else if (resultData.outputs && Array.isArray(resultData.outputs) && resultData.outputs.length > 0) { + outputUrls = resultData.outputs; + } + + if (outputUrls.length === 0) { + console.error(`[API:${requestId}] No outputs in WaveSpeed result. Response:`, JSON.stringify(resultData).substring(0, 500)); + return { + success: false, + error: `${input.model.name}: No outputs in generation result`, + }; + } + + // Fetch the first output and convert to base64 + const outputUrl = outputUrls[0]; + + // Validate URL before fetching + const outputUrlCheck = validateMediaUrl(outputUrl); + if (!outputUrlCheck.valid) { + return { success: false, error: `Invalid output URL: ${outputUrlCheck.error}` }; + } + + // For 3D models, return URL directly (GLB files are binary — skip downloading/buffering) + if (is3DModel) { + console.log(`[API:${requestId}] SUCCESS - Returning 3D model URL`); + return { + success: true, + outputs: [ + { + type: "3d", + data: "", + url: outputUrl, + }, + ], + }; + } + + console.log(`[API:${requestId}] Fetching WaveSpeed output from: ${outputUrl.substring(0, 80)}...`); + + const outputResponse = await fetch(outputUrl); + + if (!outputResponse.ok) { + return { + success: false, + error: `Failed to fetch output: ${outputResponse.status}`, + }; + } + + // Check file size before downloading body + const MAX_MEDIA_SIZE_WS = 500 * 1024 * 1024; // 500MB + const wsContentLength = parseInt(outputResponse.headers.get("content-length") || "0", 10); + if (!isNaN(wsContentLength) && wsContentLength > MAX_MEDIA_SIZE_WS) { + return { success: false, error: `Media too large: ${(wsContentLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; + } + + const outputArrayBuffer = await outputResponse.arrayBuffer(); + if (outputArrayBuffer.byteLength > MAX_MEDIA_SIZE_WS) { + return { success: false, error: `Media too large: ${(outputArrayBuffer.byteLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; + } + const outputSizeMB = outputArrayBuffer.byteLength / (1024 * 1024); + + const rawContentType = outputResponse.headers.get("content-type"); + const contentType = + (rawContentType && (rawContentType.startsWith("video/") || rawContentType.startsWith("image/"))) + ? rawContentType + : (isVideoModel ? "video/mp4" : "image/png"); + + console.log(`[API:${requestId}] Output: ${contentType}, ${outputSizeMB.toFixed(2)}MB`); + + // For very large videos (>20MB), return URL only (data left empty for consumers) + if (isVideoModel && outputSizeMB > 20) { + console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); + return { + success: true, + outputs: [ + { + type: "video", + data: "", + url: outputUrl, + }, + ], + }; + } + + const outputBase64 = Buffer.from(outputArrayBuffer).toString("base64"); + console.log(`[API:${requestId}] SUCCESS - Returning ${isVideoModel ? "video" : "image"}`); + + return { + success: true, + outputs: [ + { + type: isVideoModel ? "video" : "image", + data: `data:${contentType};base64,${outputBase64}`, + url: outputUrl, + }, + ], + }; +} + diff --git a/src/app/api/generate/route.ts b/src/app/api/generate/route.ts index b72d48ae..9b1a17f7 100644 --- a/src/app/api/generate/route.ts +++ b/src/app/api/generate/route.ts @@ -11,2351 +11,31 @@ * Images are uploaded to fal CDN before submission to avoid payload size issues. */ import { NextRequest, NextResponse } from "next/server"; -import { GoogleGenAI } from "@google/genai"; import { GenerateRequest, GenerateResponse, ModelType, SelectedModel, ProviderType } from "@/types"; -import { GenerationInput, GenerationOutput, ProviderModel } from "@/lib/providers/types"; -import { uploadImageForUrl, shouldUseImageUrl, deleteImages } from "@/lib/images"; -import { validateMediaUrl } from "@/utils/urlValidation"; +import { GenerationInput } from "@/lib/providers/types"; +import { generateWithGemini } from "./providers/gemini"; +import { generateWithReplicate } from "./providers/replicate"; +import { clearFalInputMappingCache as _clearFalInputMappingCache, generateWithFalQueue } from "./providers/fal"; +import { generateWithKie } from "./providers/kie"; +import { generateWithWaveSpeed } from "./providers/wavespeed"; -export const maxDuration = 300; // 5 minute timeout (Vercel hobby plan limit) -export const dynamic = 'force-dynamic'; // Ensure this route is always dynamic - -// Map model types to Gemini model IDs -const MODEL_MAP: Record = { - "nano-banana": "gemini-2.5-flash-image", // Updated to correct model name - "nano-banana-pro": "gemini-3-pro-image-preview", -}; - -/** - * Extended request format that supports both legacy and multi-provider requests - */ -interface MultiProviderGenerateRequest extends GenerateRequest { - selectedModel?: SelectedModel; - parameters?: Record; - /** Dynamic inputs from schema-based connections (e.g., image_url, tail_image_url, prompt) */ - dynamicInputs?: Record; -} - -/** - * Generate image using Gemini API (legacy/default path) - */ -async function generateWithGemini( - requestId: string, - apiKey: string, - prompt: string, - images: string[], - model: ModelType, - aspectRatio?: string, - resolution?: string, - useGoogleSearch?: boolean -): Promise> { - console.log(`[API:${requestId}] Gemini generation - Model: ${model}, Images: ${images?.length || 0}, Prompt: ${prompt?.length || 0} chars`); - - // Extract base64 data and MIME types from data URLs - const imageData = (images || []).map((image, idx) => { - if (image.includes("base64,")) { - const [header, data] = image.split("base64,"); - // Extract MIME type from header (e.g., "data:image/png;" -> "image/png") - const mimeMatch = header.match(/data:([^;]+)/); - const mimeType = mimeMatch ? mimeMatch[1] : "image/png"; - console.log(`[API:${requestId}] Image ${idx + 1}: ${mimeType}, ${(data.length / 1024).toFixed(1)}KB`); - return { data, mimeType }; - } - console.log(`[API:${requestId}] Image ${idx + 1}: raw, ${(image.length / 1024).toFixed(1)}KB`); - return { data: image, mimeType: "image/png" }; - }); - - // Initialize Gemini client - const ai = new GoogleGenAI({ apiKey }); - - // Build request parts array with prompt and all images - const requestParts: Array<{ text: string } | { inlineData: { mimeType: string; data: string } }> = [ - { text: prompt }, - ...imageData.map(({ data, mimeType }) => ({ - inlineData: { - mimeType, - data, - }, - })), - ]; - - // Build config object based on model capabilities - const config: Record = { - responseModalities: ["IMAGE", "TEXT"], - }; - - // Add imageConfig for both models (both support aspect ratio) - if (aspectRatio) { - config.imageConfig = { - aspectRatio, - }; - } - - // Add resolution only for Nano Banana Pro - if (model === "nano-banana-pro" && resolution) { - if (!config.imageConfig) { - config.imageConfig = {}; - } - (config.imageConfig as Record).imageSize = resolution; - } - - // Add tools array for Google Search (only Nano Banana Pro) - const tools = []; - if (model === "nano-banana-pro" && useGoogleSearch) { - tools.push({ googleSearch: {} }); - } - - console.log(`[API:${requestId}] Config: ${JSON.stringify(config)}`); - - // Make request to Gemini - const geminiStartTime = Date.now(); - - const response = await ai.models.generateContent({ - model: MODEL_MAP[model], - contents: [ - { - role: "user", - parts: requestParts, - }, - ], - config, - ...(tools.length > 0 && { tools }), - }); - - const geminiDuration = Date.now() - geminiStartTime; - console.log(`[API:${requestId}] Gemini API completed in ${geminiDuration}ms`); - - // Extract image from response - const candidates = response.candidates; - - if (!candidates || candidates.length === 0) { - console.error(`[API:${requestId}] No candidates in Gemini response`); - return NextResponse.json( - { - success: false, - error: "No response from AI model", - }, - { status: 500 } - ); - } - - const parts = candidates[0].content?.parts; - console.log(`[API:${requestId}] Response parts: ${parts?.length || 0}`); - - if (!parts) { - console.error(`[API:${requestId}] No parts in Gemini candidate content`); - return NextResponse.json( - { - success: false, - error: "No content in response", - }, - { status: 500 } - ); - } - - // Find image part in response - for (const part of parts) { - if (part.inlineData && part.inlineData.data) { - const mimeType = part.inlineData.mimeType || "image/png"; - const imgData = part.inlineData.data; - const imageSizeKB = (imgData.length / 1024).toFixed(1); - - console.log(`[API:${requestId}] Output image: ${mimeType}, ${imageSizeKB}KB`); - - const dataUrl = `data:${mimeType};base64,${imgData}`; - - const responsePayload = { success: true, image: dataUrl }; - const responseSize = JSON.stringify(responsePayload).length; - const responseSizeMB = (responseSize / (1024 * 1024)).toFixed(2); - - if (responseSize > 4.5 * 1024 * 1024) { - console.warn(`[API:${requestId}] Response size (${responseSizeMB}MB) approaching Next.js 5MB limit`); - } - - console.log(`[API:${requestId}] SUCCESS - Returning ${responseSizeMB}MB payload`); - - // Create response with explicit headers to handle large payloads - const resp = NextResponse.json(responsePayload); - resp.headers.set('Content-Type', 'application/json'); - resp.headers.set('Content-Length', responseSize.toString()); - - return resp; - } - } - - // If no image found, check for text error - for (const part of parts) { - if (part.text) { - console.error(`[API:${requestId}] Gemini returned text instead of image: ${part.text.substring(0, 100)}`); - return NextResponse.json( - { - success: false, - error: `Model returned text instead of image: ${part.text.substring(0, 200)}`, - }, - { status: 500 } - ); - } - } - - console.error(`[API:${requestId}] No image or text found in Gemini response`); - return NextResponse.json( - { - success: false, - error: "No image in response", - }, - { status: 500 } - ); -} - -/** - * Input parameter patterns - maps generic input types to possible schema parameter names - */ -const INPUT_PATTERNS: Record = { - // Text/prompt inputs - prompt: ["prompt", "text", "caption", "input_text", "description", "query"], - negativePrompt: ["negative_prompt", "negative", "neg_prompt", "negative_text"], - - // Image inputs - image: ["image_url", "image_urls", "image", "first_frame", "start_image", "init_image", - "reference_image", "input_image", "image_input", "source_image", "img", "photo"], - - // Video/media settings - aspectRatio: ["aspect_ratio", "ratio", "size", "dimensions", "output_size"], - duration: ["duration", "length", "num_frames", "seconds", "video_length"], - fps: ["fps", "frame_rate", "framerate", "frames_per_second"], - - // Audio settings - audio: ["audio_enabled", "with_audio", "enable_audio", "audio", "sound"], - - // Generation settings - seed: ["seed", "random_seed", "noise_seed"], - steps: ["steps", "num_steps", "num_inference_steps", "inference_steps"], - guidance: ["guidance_scale", "guidance", "cfg_scale", "cfg"], - - // Model-specific - scheduler: ["scheduler", "sampler", "sampler_name"], - strength: ["strength", "denoise", "denoising_strength"], -}; - -/** - * Input mapping result from schema parsing - */ -interface InputMapping { - // Maps our generic names to model-specific parameter names - paramMap: Record; - // Track which generic params expect array types (e.g., "image") - arrayParams: Set; - // Track actual schema param names that expect array types (e.g., "image_urls") - schemaArrayParams: Set; -} - -/** - * Parameter type information extracted from OpenAPI schema - */ -interface ParameterTypeInfo { - [paramName: string]: "string" | "integer" | "number" | "boolean" | "array" | "object"; -} - -/** - * Extract parameter types from OpenAPI schema - */ -function getParameterTypesFromSchema(schema: Record | undefined): ParameterTypeInfo { - const typeInfo: ParameterTypeInfo = {}; - - if (!schema) return typeInfo; - - try { - const components = schema.components as Record | undefined; - const schemas = components?.schemas as Record | undefined; - const input = schemas?.Input as Record | undefined; - const properties = input?.properties as Record | undefined; - - if (!properties) return typeInfo; - - for (const [propName, prop] of Object.entries(properties)) { - const property = prop as Record; - const type = property?.type as string | undefined; - if (type && ["string", "integer", "number", "boolean", "array", "object"].includes(type)) { - typeInfo[propName] = type as ParameterTypeInfo[string]; - } - } - } catch { - // Schema parsing failed - } - - return typeInfo; -} - -/** - * Coerce parameter values to their expected types based on schema - * This handles cases where values were incorrectly stored as strings (e.g., from UI enum selects) - */ -function coerceParameterTypes( - parameters: Record | undefined, - typeInfo: ParameterTypeInfo -): Record { - if (!parameters) return {}; - - const result = { ...parameters }; - - for (const [key, value] of Object.entries(result)) { - if (value === undefined || value === null) continue; - - const expectedType = typeInfo[key]; - if (!expectedType) continue; - - // Coerce string values to their expected types - if (typeof value === "string") { - if (expectedType === "integer") { - const parsed = parseInt(value, 10); - if (!isNaN(parsed)) result[key] = parsed; - } else if (expectedType === "number") { - const parsed = parseFloat(value); - if (!isNaN(parsed)) result[key] = parsed; - } else if (expectedType === "boolean") { - result[key] = value === "true"; - } - } - } - - return result; -} - -/** - * Extract input parameter mappings from OpenAPI schema - * Returns a mapping of generic parameter names to model-specific names - */ -function getInputMappingFromSchema(schema: Record | undefined): InputMapping { - const paramMap: Record = {}; - const arrayParams = new Set(); - const schemaArrayParams = new Set(); - - if (!schema) return { paramMap, arrayParams, schemaArrayParams }; - - try { - // Navigate to input schema properties - const components = schema.components as Record | undefined; - const schemas = components?.schemas as Record | undefined; - const input = schemas?.Input as Record | undefined; - const properties = input?.properties as Record | undefined; - - if (!properties) return { paramMap, arrayParams, schemaArrayParams }; - - // First pass: detect all array-typed properties by their actual schema name - for (const [propName, prop] of Object.entries(properties)) { - const property = prop as Record; - if (property?.type === "array") { - schemaArrayParams.add(propName); - } - } - - const propertyNames = Object.keys(properties); - - // For each input type pattern, find the matching schema property - for (const [genericName, patterns] of Object.entries(INPUT_PATTERNS)) { - for (const pattern of patterns) { - let matchedParam: string | null = null; - - // Check for exact match first - if (properties[pattern]) { - matchedParam = pattern; - } else { - // Check for case-insensitive partial match - const match = propertyNames.find(name => - name.toLowerCase().includes(pattern.toLowerCase()) || - pattern.toLowerCase().includes(name.toLowerCase()) - ); - if (match) { - matchedParam = match; - } - } - - if (matchedParam) { - paramMap[genericName] = matchedParam; - // Check if this property expects an array type - const property = properties[matchedParam] as Record; - if (property?.type === "array") { - arrayParams.add(genericName); - } - break; - } - } - } - } catch { - // Schema parsing failed - } - - return { paramMap, arrayParams, schemaArrayParams }; -} - -/** - * Generate image using Replicate API - */ -async function generateWithReplicate( - requestId: string, - apiKey: string, - input: GenerationInput -): Promise { - console.log(`[API:${requestId}] Replicate generation - Model: ${input.model.id}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); - - const REPLICATE_API_BASE = "https://api.replicate.com/v1"; - - // Get the latest version of the model - const modelId = input.model.id; - const [owner, name] = modelId.split("/"); - - // First, get the model to find the latest version - const modelResponse = await fetch( - `${REPLICATE_API_BASE}/models/${owner}/${name}`, - { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - } - ); - - if (!modelResponse.ok) { - return { - success: false, - error: `Failed to get model info: ${modelResponse.status}`, - }; - } - - const modelData = await modelResponse.json(); - const version = modelData.latest_version?.id; - - if (!version) { - return { - success: false, - error: "Model has no available version", - }; - } - - const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; - console.log(`[API:${requestId}] Model version: ${version}, Dynamic inputs: ${hasDynamicInputs ? Object.keys(input.dynamicInputs!).join(", ") : "none"}`); - - // Get schema for type coercion and input mapping - const schema = modelData.latest_version?.openapi_schema as Record | undefined; - const parameterTypes = getParameterTypesFromSchema(schema); - - // Build input for the prediction, coercing parameter types from schema - const predictionInput: Record = { - ...coerceParameterTypes(input.parameters, parameterTypes), - }; - - // Add dynamic inputs if provided (these come from schema-mapped connections) - if (hasDynamicInputs) { - const { schemaArrayParams } = getInputMappingFromSchema(schema); - - // Apply array wrapping based on schema type - for (const [key, value] of Object.entries(input.dynamicInputs!)) { - if (value !== null && value !== undefined && value !== '') { - if (schemaArrayParams.has(key) && !Array.isArray(value)) { - predictionInput[key] = [value]; // Wrap in array - } else { - predictionInput[key] = value; - } - } - } - } else { - // Fallback: use schema to map generic input names to model-specific parameter names - const { paramMap, arrayParams } = getInputMappingFromSchema(schema); - - // Map prompt input - if (input.prompt) { - const promptParam = paramMap.prompt || "prompt"; - predictionInput[promptParam] = input.prompt; - } - - // Map image input - use array or string format based on schema - if (input.images && input.images.length > 0) { - const imageParam = paramMap.image || "image"; - if (arrayParams.has("image")) { - predictionInput[imageParam] = input.images; - } else { - predictionInput[imageParam] = input.images[0]; - } - } - - // Map any parameters that might need renaming (use coerced values) - const coercedParams = coerceParameterTypes(input.parameters, parameterTypes); - for (const [key, value] of Object.entries(coercedParams)) { - const mappedKey = paramMap[key] || key; - predictionInput[mappedKey] = value; - } - } - - // Create a prediction - const createResponse = await fetch(`${REPLICATE_API_BASE}/predictions`, { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - version, - input: predictionInput, - }), - }); - - if (!createResponse.ok) { - const errorText = await createResponse.text(); - let errorDetail = errorText; - try { - const errorJson = JSON.parse(errorText); - errorDetail = errorJson.detail || errorJson.message || errorJson.error || errorText; - } catch { - // Keep original text if not JSON - } - - // Handle rate limits - if (createResponse.status === 429) { - return { - success: false, - error: `${input.model.name}: Rate limit exceeded. Try again in a moment.`, - }; - } - - return { - success: false, - error: `${input.model.name}: ${errorDetail}`, - }; - } - - const prediction = await createResponse.json(); - console.log(`[API:${requestId}] Prediction created: ${prediction.id}`); - - // Poll for completion - const maxWaitTime = 5 * 60 * 1000; // 5 minutes - const pollInterval = 1000; // 1 second - const startTime = Date.now(); - - let currentPrediction = prediction; - let lastStatus = ""; - - while ( - currentPrediction.status !== "succeeded" && - currentPrediction.status !== "failed" && - currentPrediction.status !== "canceled" - ) { - if (Date.now() - startTime > maxWaitTime) { - return { - success: false, - error: `${input.model.name}: Generation timed out after 5 minutes. Video models may take longer - try again.`, - }; - } - - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - - const pollResponse = await fetch( - `${REPLICATE_API_BASE}/predictions/${currentPrediction.id}`, - { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - } - ); - - if (!pollResponse.ok) { - return { - success: false, - error: `Failed to poll prediction: ${pollResponse.status}`, - }; - } - - currentPrediction = await pollResponse.json(); - if (currentPrediction.status !== lastStatus) { - console.log(`[API:${requestId}] Prediction status: ${currentPrediction.status}`); - lastStatus = currentPrediction.status; - } - } - - if (currentPrediction.status === "failed") { - const failureReason = currentPrediction.error || "Prediction failed"; - return { - success: false, - error: `${input.model.name}: ${failureReason}`, - }; - } - - if (currentPrediction.status === "canceled") { - return { - success: false, - error: "Prediction was canceled", - }; - } - - // Extract output - const output = currentPrediction.output; - if (!output) { - return { - success: false, - error: "No output from prediction", - }; - } - - // Output can be a single URL string or an array of URLs - const outputUrls: string[] = Array.isArray(output) ? output : [output]; - - if (outputUrls.length === 0) { - return { - success: false, - error: "No output from prediction", - }; - } - - // Fetch the first output and convert to base64 - const mediaUrl = outputUrls[0]; - console.log(`[API:${requestId}] Fetching output from: ${mediaUrl.substring(0, 80)}...`); - const mediaResponse = await fetch(mediaUrl); - - if (!mediaResponse.ok) { - return { - success: false, - error: `Failed to fetch output: ${mediaResponse.status}`, - }; - } - - // Determine MIME type from response - const contentType = mediaResponse.headers.get("content-type") || "image/png"; - const isVideo = contentType.startsWith("video/"); - - const mediaArrayBuffer = await mediaResponse.arrayBuffer(); - const mediaSizeBytes = mediaArrayBuffer.byteLength; - const mediaSizeMB = mediaSizeBytes / (1024 * 1024); - - console.log(`[API:${requestId}] Output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); - - // For very large videos (>20MB), return URL directly instead of base64 - if (isVideo && mediaSizeMB > 20) { - console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); - return { - success: true, - outputs: [ - { - type: "video", - data: mediaUrl, // Return URL directly for very large videos - url: mediaUrl, - }, - ], - }; - } - - const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); - console.log(`[API:${requestId}] SUCCESS - Returning ${isVideo ? "video" : "image"}`); - - return { - success: true, - outputs: [ - { - type: isVideo ? "video" : "image", - data: `data:${contentType};base64,${mediaBase64}`, - url: mediaUrl, - }, - ], - }; -} - -/** - * Extended input mapping with parameter types for fal.ai - */ -interface FalInputMapping extends InputMapping { - parameterTypes: ParameterTypeInfo; -} - -/** - * In-memory cache for fal.ai schema mappings to avoid extra API call per generation - */ -const falInputMappingCache = new Map(); -const FAL_MAPPING_CACHE_TTL = 30 * 60 * 1000; // 30 minutes - -/** Clear the fal schema mapping cache (exported for testing) */ -export function clearFalInputMappingCache() { - falInputMappingCache.clear(); -} - -/** - * Fetch fal.ai model schema and extract input parameter mappings - * Uses the Model Search API with OpenAPI expansion (same as /api/models/[modelId]) - * Results are cached in-memory for 30 minutes per model. - */ -async function getFalInputMapping(modelId: string, apiKey: string | null): Promise { - // Check cache first - const cached = falInputMappingCache.get(modelId); - if (cached && Date.now() - cached.timestamp < FAL_MAPPING_CACHE_TTL) { - return cached.result; - } - const paramMap: Record = {}; - const arrayParams = new Set(); - const schemaArrayParams = new Set(); - const parameterTypes: ParameterTypeInfo = {}; - - try { - // Use fal.ai Model Search API with OpenAPI expansion - const headers: Record = {}; - if (apiKey) { - headers["Authorization"] = `Key ${apiKey}`; - } - - const url = `https://api.fal.ai/v1/models?endpoint_id=${encodeURIComponent(modelId)}&expand=openapi-3.0`; - const response = await fetch(url, { headers }); - - if (!response.ok) { - return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; - } - - const data = await response.json(); - const modelData = data.models?.[0]; - if (!modelData?.openapi) { - return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; - } - - // Extract input schema from OpenAPI spec (same logic as /api/models/[modelId]) - const spec = modelData.openapi; - let inputSchema: Record | null = null; - - for (const pathObj of Object.values(spec.paths || {})) { - const postOp = (pathObj as Record)?.post as Record | undefined; - const reqBody = postOp?.requestBody as Record | undefined; - const content = reqBody?.content as Record> | undefined; - const jsonContent = content?.["application/json"]; - - if (jsonContent?.schema) { - const schema = jsonContent.schema as Record; - if (schema.$ref && typeof schema.$ref === "string") { - const refPath = schema.$ref.replace("#/components/schemas/", ""); - inputSchema = spec.components?.schemas?.[refPath] as Record; - break; - } else if (schema.properties) { - inputSchema = schema; - break; - } - } - } - - if (!inputSchema) { - return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; - } - - const properties = inputSchema.properties as Record | undefined; - if (!properties) return { paramMap, arrayParams, schemaArrayParams, parameterTypes }; - - // First pass: detect all array-typed properties and extract parameter types - // This is used for dynamicInputs which use schema names directly - for (const [propName, prop] of Object.entries(properties)) { - const property = prop as Record; - if (property?.type === "array") { - schemaArrayParams.add(propName); - } - // Extract parameter type for type coercion - const type = property?.type as string | undefined; - if (type && ["string", "integer", "number", "boolean", "array", "object"].includes(type)) { - parameterTypes[propName] = type as ParameterTypeInfo[string]; - } - } - - // Second pass: match properties to INPUT_PATTERNS and detect array types - const propertyNames = Object.keys(properties); - for (const [genericName, patterns] of Object.entries(INPUT_PATTERNS)) { - for (const pattern of patterns) { - let matchedParam: string | null = null; - - // Check for exact match first - if (properties[pattern]) { - matchedParam = pattern; - } else { - // Check for case-insensitive partial match - const match = propertyNames.find(name => - name.toLowerCase().includes(pattern.toLowerCase()) || - pattern.toLowerCase().includes(name.toLowerCase()) - ); - if (match) { - matchedParam = match; - } - } - - if (matchedParam) { - paramMap[genericName] = matchedParam; - // Check if this property expects an array type - const property = properties[matchedParam] as Record; - if (property?.type === "array") { - arrayParams.add(genericName); - } - break; - } - } - } - } catch { - // Schema parsing failed - continue with empty mapping - } - - const result = { paramMap, arrayParams, schemaArrayParams, parameterTypes }; - falInputMappingCache.set(modelId, { result, timestamp: Date.now() }); - return result; -} - -const MAX_UPLOAD_SIZE = 20 * 1024 * 1024; // 20 MB - -/** - * Upload a base64 data URL image to fal.ai CDN storage. - * Returns the CDN URL to use in API requests instead of inline base64. - * If the input is already a URL (not base64), returns it as-is. - */ -async function uploadImageToFal(base64DataUrl: string, apiKey: string | null): Promise { - // Already a URL, not base64 - if (!base64DataUrl.startsWith("data:")) return base64DataUrl; - - const match = base64DataUrl.match(/^data:([^;]+);base64,(.+)$/); - if (!match) return base64DataUrl; - - const estimatedBytes = Math.ceil(match[2].length * 3 / 4); - if (estimatedBytes > MAX_UPLOAD_SIZE) { - throw new Error(`Image too large to upload (${(estimatedBytes / (1024 * 1024)).toFixed(1)} MB, max ${MAX_UPLOAD_SIZE / (1024 * 1024)} MB)`); - } - - const contentType = match[1]; - const binaryData = Buffer.from(match[2], "base64"); - - const authHeaders: Record = {}; - if (apiKey) authHeaders["Authorization"] = `Key ${apiKey}`; - - // Step 1: Initiate upload to get a signed PUT URL - const ext = contentType.split("/")[1] || "png"; - const initiateResponse = await fetch( - "https://rest.alpha.fal.ai/storage/upload/initiate?storage_type=fal-cdn-v3", - { - method: "POST", - headers: { - "Content-Type": "application/json", - ...authHeaders, - }, - body: JSON.stringify({ - content_type: contentType, - file_name: `${Date.now()}.${ext}`, - }), - } - ); - - if (!initiateResponse.ok) { - throw new Error(`Failed to initiate fal CDN upload: ${initiateResponse.status}`); - } - - const { upload_url: uploadUrl, file_url: fileUrl } = await initiateResponse.json(); - - // Validate both URLs before using them (SSRF protection) - if (!uploadUrl || !fileUrl) { - throw new Error("fal CDN initiate response missing upload_url or file_url"); - } - - const uploadUrlCheck = validateMediaUrl(uploadUrl); - if (!uploadUrlCheck.valid || !uploadUrl.startsWith('https://')) { - throw new Error(`fal CDN upload_url failed validation: ${uploadUrlCheck.error || 'not HTTPS'}`); - } - - const fileUrlCheck = validateMediaUrl(fileUrl); - if (!fileUrlCheck.valid || !fileUrl.startsWith('https://')) { - throw new Error(`fal CDN file_url failed validation: ${fileUrlCheck.error || 'not HTTPS'}`); - } - - // Step 2: PUT the binary data to the validated signed URL - const putResponse = await fetch(uploadUrl, { - method: "PUT", - headers: { "Content-Type": contentType }, - body: binaryData, - }); - - if (!putResponse.ok) { - throw new Error(`Failed to upload to fal CDN: ${putResponse.status}`); - } - - return fileUrl; -} - -/** - * Generate using fal.ai Queue API - * Uses async queue submission + polling (1s interval) instead of blocking fal.run. - * Images are uploaded to fal CDN before submission to avoid payload size issues. - */ -async function generateWithFalQueue( - requestId: string, - apiKey: string | null, - input: GenerationInput -): Promise { - console.log(`[API:${requestId}] fal.ai queue generation - Model: ${input.model.id}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); - - const modelId = input.model.id; - const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; - console.log(`[API:${requestId}] Dynamic inputs: ${hasDynamicInputs ? Object.keys(input.dynamicInputs!).join(", ") : "none"}, API key: ${apiKey ? "yes" : "no"}`); - - // Fetch schema for type coercion and input mapping (cached) - const { paramMap, arrayParams, schemaArrayParams, parameterTypes } = await getFalInputMapping(modelId, apiKey); - - // Build request body, coercing parameter types from schema - const requestBody: Record = { - ...coerceParameterTypes(input.parameters, parameterTypes), - }; - - // Upload base64 images to fal CDN to avoid sending large payloads inline - const uploadImage = async (value: string | string[]): Promise => { - if (Array.isArray(value)) { - return Promise.all(value.map(v => typeof v === "string" && v.startsWith("data:") ? uploadImageToFal(v, apiKey) : Promise.resolve(v))); - } - if (typeof value === "string" && value.startsWith("data:")) { - return uploadImageToFal(value, apiKey); - } - return value; - }; - - if (hasDynamicInputs) { - const filteredInputs: Record = {}; - for (const [key, value] of Object.entries(input.dynamicInputs!)) { - if (value !== null && value !== undefined && value !== '') { - let processedValue: unknown = value; - // Upload base64 images to CDN - if (typeof value === "string" || Array.isArray(value)) { - processedValue = await uploadImage(value); - } - // Wrap in array if schema expects array but we have a single value - if (schemaArrayParams.has(key) && !Array.isArray(processedValue)) { - filteredInputs[key] = [processedValue]; - } else { - filteredInputs[key] = processedValue; - } - } - } - Object.assign(requestBody, filteredInputs); - } else { - // Fallback: use schema to map generic input names to model-specific parameter names - if (input.prompt) { - const promptParam = paramMap.prompt || "prompt"; - requestBody[promptParam] = input.prompt; - } - - if (input.images && input.images.length > 0) { - // Upload images to CDN before sending - const uploadedImages = await Promise.all( - input.images.map(img => uploadImageToFal(img, apiKey)) - ); - const imageParam = paramMap.image || "image_url"; - if (arrayParams.has("image")) { - requestBody[imageParam] = uploadedImages; - } else { - requestBody[imageParam] = uploadedImages[0]; - } - } - - // Map any parameters that might need renaming (use coerced values) - const coercedParams = coerceParameterTypes(input.parameters, parameterTypes); - for (const [key, value] of Object.entries(coercedParams)) { - const mappedKey = paramMap[key] || key; - requestBody[mappedKey] = value; - } - } - - // Build headers - const headers: Record = { - "Content-Type": "application/json", - }; - if (apiKey) { - headers["Authorization"] = `Key ${apiKey}`; - } - - // Submit to queue - console.log(`[API:${requestId}] Submitting to fal.ai queue with inputs: ${Object.keys(requestBody).join(", ")}`); - const submitResponse = await fetch(`https://queue.fal.run/${modelId}`, { - method: "POST", - headers, - body: JSON.stringify(requestBody), - }); - - if (!submitResponse.ok) { - const errorText = await submitResponse.text(); - let errorDetail = errorText || `HTTP ${submitResponse.status}`; - try { - const errorJson = JSON.parse(errorText); - if (typeof errorJson.error === 'object' && errorJson.error?.message) { - errorDetail = errorJson.error.message; - } else if (errorJson.detail) { - if (Array.isArray(errorJson.detail)) { - errorDetail = errorJson.detail.map((d: { msg?: string; loc?: string[] }) => - d.msg || JSON.stringify(d) - ).join('; '); - } else { - errorDetail = errorJson.detail; - } - } else if (errorJson.message) { - errorDetail = errorJson.message; - } else if (typeof errorJson.error === 'string') { - errorDetail = errorJson.error; - } - } catch { - // Keep original text if not JSON - } - - if (submitResponse.status === 429) { - return { - success: false, - error: `${input.model.name}: Rate limit exceeded. ${apiKey ? "Try again in a moment." : "Add an API key in settings for higher limits."}`, - }; - } - - return { - success: false, - error: `${input.model.name}: ${errorDetail}`, - }; - } - - const submitResult = await submitResponse.json(); - console.log(`[API:${requestId}] Queue submit response:`, JSON.stringify(submitResult).substring(0, 500)); - const falRequestId = submitResult.request_id; - - if (!falRequestId) { - console.error(`[API:${requestId}] No request_id in queue submit response`); - return { - success: false, - error: "No request_id in queue response", - }; - } - - // Use URLs from response if provided, with SSRF validation; fall back to constructed URLs - const fallbackStatusUrl = `https://queue.fal.run/${modelId}/requests/${falRequestId}/status`; - const fallbackResponseUrl = `https://queue.fal.run/${modelId}/requests/${falRequestId}`; - let statusUrl = fallbackStatusUrl; - let responseUrl = fallbackResponseUrl; - - if (submitResult.status_url) { - const statusCheck = validateMediaUrl(submitResult.status_url); - if (statusCheck.valid && submitResult.status_url.startsWith('https://queue.fal.run/')) { - statusUrl = submitResult.status_url; - } else { - console.warn(`[API:${requestId}] fal.ai provided invalid status URL: ${submitResult.status_url} — falling back to constructed URL`); - } - } - if (submitResult.response_url) { - const responseCheck = validateMediaUrl(submitResult.response_url); - if (responseCheck.valid && submitResult.response_url.startsWith('https://queue.fal.run/')) { - responseUrl = submitResult.response_url; - } else { - console.warn(`[API:${requestId}] fal.ai provided invalid response URL: ${submitResult.response_url} — falling back to constructed URL`); - } - } - - console.log(`[API:${requestId}] Queue request submitted: ${falRequestId}, status URL: ${statusUrl}`); - - // Poll for completion - const maxWaitTime = 10 * 60 * 1000; // 10 minutes for video - const pollInterval = 1000; // 1 second (matches Replicate/WaveSpeed) - const startTime = Date.now(); - let lastStatus = ""; - - while (true) { - if (Date.now() - startTime > maxWaitTime) { - console.error(`[API:${requestId}] Queue request timed out after 10 minutes`); - return { - success: false, - error: `${input.model.name}: Video generation timed out after 10 minutes`, - }; - } - - await new Promise(resolve => setTimeout(resolve, pollInterval)); - - const statusResponse = await fetch( - statusUrl, - { headers: apiKey ? { "Authorization": `Key ${apiKey}` } : {} } - ); - - if (!statusResponse.ok) { - console.error(`[API:${requestId}] Failed to poll status: ${statusResponse.status}`); - return { - success: false, - error: `Failed to poll status: ${statusResponse.status}`, - }; - } - - const statusResult = await statusResponse.json(); - const status = statusResult.status; - - if (status !== lastStatus) { - console.log(`[API:${requestId}] Queue status: ${status}`); - lastStatus = status; - } - - if (status === "COMPLETED") { - // Fetch the result - const resultResponse = await fetch( - responseUrl, - { headers: apiKey ? { "Authorization": `Key ${apiKey}` } : {} } - ); - - if (!resultResponse.ok) { - console.error(`[API:${requestId}] Failed to fetch result: ${resultResponse.status}`); - return { - success: false, - error: `Failed to fetch result: ${resultResponse.status}`, - }; - } - - const result = await resultResponse.json(); - - // Extract video URL from result - let mediaUrl: string | null = null; - - if (result.video && result.video.url) { - mediaUrl = result.video.url; - } else if (result.images && Array.isArray(result.images) && result.images.length > 0) { - mediaUrl = result.images[0].url; - } else if (result.image && result.image.url) { - mediaUrl = result.image.url; - } else if (result.output && typeof result.output === "string") { - mediaUrl = result.output; - } - - if (!mediaUrl) { - console.error(`[API:${requestId}] No media URL found in queue result`); - return { - success: false, - error: "No media URL in response", - }; - } - - // Fetch the media and convert to base64 - console.log(`[API:${requestId}] Fetching output from: ${mediaUrl.substring(0, 80)}...`); - const mediaResponse = await fetch(mediaUrl); - - if (!mediaResponse.ok) { - return { - success: false, - error: `Failed to fetch output: ${mediaResponse.status}`, - }; - } - - const contentType = mediaResponse.headers.get("content-type") || "video/mp4"; - const isVideo = contentType.startsWith("video/"); - - const mediaArrayBuffer = await mediaResponse.arrayBuffer(); - const mediaSizeBytes = mediaArrayBuffer.byteLength; - const mediaSizeMB = mediaSizeBytes / (1024 * 1024); - - console.log(`[API:${requestId}] Output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); - - // For very large videos (>20MB), return URL directly instead of base64 - if (isVideo && mediaSizeMB > 20) { - console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); - return { - success: true, - outputs: [ - { - type: "video", - data: mediaUrl, - url: mediaUrl, - }, - ], - }; - } - - const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); - console.log(`[API:${requestId}] SUCCESS - Returning ${isVideo ? "video" : "image"}`); - - return { - success: true, - outputs: [ - { - type: isVideo ? "video" : "image", - data: `data:${contentType};base64,${mediaBase64}`, - url: mediaUrl, - }, - ], - }; - } - - if (status === "FAILED") { - const errorMessage = statusResult.error || "Video generation failed"; - console.error(`[API:${requestId}] Queue request failed: ${errorMessage}`); - return { - success: false, - error: `${input.model.name}: ${errorMessage}`, - }; - } - - // Continue polling for IN_QUEUE, IN_PROGRESS, etc. - } -} - -// ============ Kie.ai Helpers ============ - -/** - * Get default required parameters for a Kie model - * Many Kie models require specific parameters to be present even if not user-specified - */ -function getKieModelDefaults(modelId: string): Record { - switch (modelId) { - // GPT Image models - case "gpt-image/1.5-text-to-image": - case "gpt-image/1.5-image-to-image": - return { - aspect_ratio: "3:2", - quality: "medium", - }; - - // Z-Image model - case "z-image": - return { - aspect_ratio: "1:1", - }; - - // Seedream models - case "seedream/4.5-text-to-image": - case "seedream/4.5-edit": - return { - aspect_ratio: "1:1", - quality: "basic", - }; - - // Nano Banana Pro (Kie) - case "nano-banana-pro": - return { - aspect_ratio: "1:1", - resolution: "1K", - }; - - // Flux-2 models - case "flux-2/pro-text-to-image": - case "flux-2/pro-image-to-image": - case "flux-2/flex-text-to-image": - case "flux-2/flex-image-to-image": - return { - aspect_ratio: "1:1", - }; - - // Grok Imagine image models - case "grok-imagine/text-to-image": - return { - aspect_ratio: "1:1", - }; - - case "grok-imagine/image-to-image": - return {}; - - // Grok Imagine video models - case "grok-imagine/text-to-video": - return { - aspect_ratio: "2:3", - duration: "6", - mode: "normal", - }; - - case "grok-imagine/image-to-video": - return { - aspect_ratio: "2:3", - duration: "6", - mode: "normal", - }; - - // Kling 2.6 video models - case "kling-2.6/text-to-video": - case "kling-2.6/image-to-video": - return { - aspect_ratio: "16:9", - duration: "5", - sound: true, - }; - - // Kling 2.6 motion control - case "kling-2.6/motion-control": - return { - mode: "720p", - character_orientation: "video", - }; - - // Kling 2.5 turbo models - case "kling/v2-5-turbo-text-to-video-pro": - case "kling/v2-5-turbo-image-to-video-pro": - return { - aspect_ratio: "16:9", - duration: "5", - cfg_scale: 0.5, - }; - - // Wan video models - case "wan/2-6-text-to-video": - case "wan/2-6-image-to-video": - return { - duration: "5", - resolution: "1080p", - }; - - case "wan/2-6-video-to-video": - return { - duration: "5", - resolution: "1080p", - }; - - // Topaz video upscale - case "topaz/video-upscale": - return { - upscale_factor: "2", - }; - - // Veo 3 models - case "veo3/text-to-video": - case "veo3/image-to-video": - case "veo3-fast/text-to-video": - case "veo3-fast/image-to-video": - return { - aspect_ratio: "16:9", - }; - - default: - return {}; - } -} - -/** - * Get the correct image input parameter name for a Kie model - */ -function getKieImageInputKey(modelId: string): string { - // Model-specific parameter names - if (modelId === "nano-banana-pro") return "image_input"; - if (modelId === "seedream/4.5-edit") return "image_urls"; - if (modelId === "gpt-image/1.5-image-to-image") return "input_urls"; - // Flux-2 I2I models use input_urls - if (modelId === "flux-2/pro-image-to-image" || modelId === "flux-2/flex-image-to-image") return "input_urls"; - // Kling 2.5 turbo I2V uses singular image_url - if (modelId === "kling/v2-5-turbo-image-to-video-pro") return "image_url"; - // Kling 2.6 motion control uses input_urls - if (modelId === "kling-2.6/motion-control") return "input_urls"; - // Topaz video upscale uses video_url (singular) - if (modelId === "topaz/video-upscale") return "video_url"; - // Veo 3 models use imageUrls - if (modelId.startsWith("veo3")) return "imageUrls"; - // Default for most models - return "image_urls"; -} - - -/** - * Detect actual image type from binary data (magic bytes) - */ -function detectImageType(buffer: Buffer): { mimeType: string; ext: string } { - // Check magic bytes - if (buffer[0] === 0x89 && buffer[1] === 0x50 && buffer[2] === 0x4E && buffer[3] === 0x47) { - return { mimeType: "image/png", ext: "png" }; - } - if (buffer[0] === 0xFF && buffer[1] === 0xD8 && buffer[2] === 0xFF) { - return { mimeType: "image/jpeg", ext: "jpg" }; - } - if (buffer[0] === 0x52 && buffer[1] === 0x49 && buffer[2] === 0x46 && buffer[3] === 0x46 && - buffer[8] === 0x57 && buffer[9] === 0x45 && buffer[10] === 0x42 && buffer[11] === 0x50) { - return { mimeType: "image/webp", ext: "webp" }; - } - if (buffer[0] === 0x47 && buffer[1] === 0x49 && buffer[2] === 0x46) { - return { mimeType: "image/gif", ext: "gif" }; - } - // Default to PNG - return { mimeType: "image/png", ext: "png" }; -} - -/** - * Upload a base64 image to Kie.ai and get a URL - * Required for image-to-image models since Kie doesn't accept base64 directly - * Uses base64 upload endpoint (same as official Kie client) - */ -async function uploadImageToKie( - requestId: string, - apiKey: string, - base64Image: string -): Promise { - // Extract mime type and data from data URL - let declaredMimeType = "image/png"; - let imageData = base64Image; - - if (base64Image.startsWith("data:")) { - const matches = base64Image.match(/^data:([^;]+);base64,(.+)$/); - if (matches) { - declaredMimeType = matches[1]; - imageData = matches[2]; - } - } - - // Convert base64 to binary to detect actual type - const binaryData = Buffer.from(imageData, "base64"); - - // Detect actual image type from magic bytes (don't trust the declared MIME type) - const detected = detectImageType(binaryData); - const mimeType = detected.mimeType; - const ext = detected.ext; - - const filename = `upload_${Date.now()}.${ext}`; - - console.log(`[API:${requestId}] Uploading image to Kie.ai: ${filename} (${(binaryData.length / 1024).toFixed(1)}KB) [declared: ${declaredMimeType}, actual: ${mimeType}]`); - - // Use base64 upload endpoint (same as official Kie client) - // Format: data:{mime_type};base64,{data} - const dataUrl = `data:${mimeType};base64,${imageData}`; - - const response = await fetch("https://kieai.redpandaai.co/api/file-base64-upload", { - method: "POST", - headers: { - "Authorization": `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - base64Data: dataUrl, - uploadPath: "images", - fileName: filename, - }), - }); - - if (!response.ok) { - const errorText = await response.text(); - throw new Error(`Failed to upload image: ${response.status} - ${errorText}`); - } - - const result = await response.json(); - console.log(`[API:${requestId}] Kie upload response:`, JSON.stringify(result).substring(0, 300)); - - // Check for error in response - if (result.code && result.code !== 200 && !result.success) { - throw new Error(`Upload failed: ${result.msg || 'Unknown error'}`); - } - - // Response format: { success: true, code: 200, data: { downloadUrl: "...", fileName: "...", fileSize: 123 } } - const downloadUrl = result.data?.downloadUrl || result.downloadUrl || result.url; - - if (!downloadUrl) { - console.error(`[API:${requestId}] Upload response has no URL:`, result); - throw new Error(`No download URL in upload response. Response: ${JSON.stringify(result).substring(0, 200)}`); - } - - console.log(`[API:${requestId}] Image uploaded: ${downloadUrl.substring(0, 80)}...`); - return downloadUrl; -} - -/** - * Poll Kie.ai task status until completion - */ -async function pollKieTaskCompletion( - requestId: string, - apiKey: string, - taskId: string, -): Promise<{ success: boolean; data?: Record; error?: string }> { - const maxWaitTime = 10 * 60 * 1000; // 10 minutes for video - const pollInterval = 2000; // 2 seconds - const startTime = Date.now(); - let lastStatus = ""; - - const pollUrl = `https://api.kie.ai/api/v1/jobs/recordInfo?taskId=${encodeURIComponent(taskId)}`; - - while (true) { - if (Date.now() - startTime > maxWaitTime) { - return { success: false, error: "Generation timed out after 10 minutes" }; - } - - await new Promise(resolve => setTimeout(resolve, pollInterval)); - - const response = await fetch(pollUrl, { - headers: { - "Authorization": `Bearer ${apiKey}`, - }, - }); - - if (!response.ok) { - return { success: false, error: `Failed to poll status: ${response.status}` }; - } - - const result = await response.json(); - // Kie API returns "state" in result.data.state (not "status") - const state = (result.data?.state || result.state || result.status || "").toUpperCase(); - - if (state !== lastStatus) { - console.log(`[API:${requestId}] Kie task state: ${state}`); - lastStatus = state; - } - - if (state === "SUCCESS" || state === "COMPLETED") { - return { success: true, data: result.data || result }; - } - - if (state === "FAIL" || state === "FAILED" || state === "ERROR") { - const errorMessage = result.data?.failMsg || result.data?.errorMessage || result.error || result.message || "Generation failed"; - return { success: false, error: errorMessage }; - } - - // Continue polling for: WAITING, QUEUING, GENERATING, PROCESSING, etc. - } -} - -// ============ Veo 3 Helpers ============ - -function isVeoModel(modelId: string): boolean { - return modelId.startsWith("veo3/") || modelId.startsWith("veo3-fast/"); -} - -function getVeoApiModelId(modelId: string): string { - if (modelId.startsWith("veo3-fast/")) return "veo3_fast"; - return "veo3"; -} - -async function pollVeoTaskCompletion( - requestId: string, - apiKey: string, - taskId: string, -): Promise<{ success: boolean; data?: Record; error?: string }> { - const maxWaitTime = 10 * 60 * 1000; - const pollInterval = 2000; - const startTime = Date.now(); - let lastStatus = -1; - - const pollUrl = `https://api.kie.ai/api/v1/veo/record-info?taskId=${encodeURIComponent(taskId)}`; - - while (true) { - if (Date.now() - startTime > maxWaitTime) { - return { success: false, error: "Generation timed out after 10 minutes" }; - } - await new Promise(resolve => setTimeout(resolve, pollInterval)); - - const response = await fetch(pollUrl, { - headers: { "Authorization": `Bearer ${apiKey}` }, - }); - if (!response.ok) { - return { success: false, error: `Failed to poll status: ${response.status}` }; - } - - const result = await response.json(); - const successFlag = result.data?.successFlag ?? -1; - - if (successFlag !== lastStatus) { - console.log(`[API:${requestId}] Veo task successFlag: ${successFlag}`); - lastStatus = successFlag; - } - - if (successFlag === 1) { - return { success: true, data: result.data }; - } - if (successFlag === 2 || successFlag === 3) { - const errorMessage = result.data?.errorMessage || "Generation failed"; - return { success: false, error: errorMessage }; - } - // successFlag === 0 means still generating, continue polling - } -} - -/** - * Generate image/video using Kie.ai API - */ -async function generateWithKie( - requestId: string, - apiKey: string, - input: GenerationInput -): Promise { - const modelId = input.model.id; - - console.log(`[API:${requestId}] Kie.ai generation - Model: ${modelId}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); - - // Build the input object (all parameters go inside "input" for Kie API) - // Start with model-specific required defaults - const modelDefaults = getKieModelDefaults(modelId); - const inputParams: Record = { ...modelDefaults }; - - // Add prompt - if (input.prompt) { - inputParams.prompt = input.prompt; - } - - // Add model parameters (user params override defaults) - if (input.parameters) { - Object.assign(inputParams, input.parameters); - } - - // GPT Image 1.5 does NOT support 'size' parameter - only 'aspect_ratio' - // Remove any stale 'size' values from old workflow data - if (modelId.startsWith("gpt-image/1.5")) { - delete inputParams.size; - } - - // Handle dynamic inputs FIRST (from schema-mapped connections) - these take priority - // Track which image keys dynamicInputs already handled to avoid double-uploads - const handledImageKeys = new Set(); - - if (input.dynamicInputs) { - for (const [key, value] of Object.entries(input.dynamicInputs)) { - if (value !== null && value !== undefined && value !== '') { - // Check if this is an image input that needs uploading - if (typeof value === 'string' && value.startsWith('data:image')) { - // Single data URL - upload it - const url = await uploadImageToKie(requestId, apiKey, value); - // Singular keys get a string, plural keys get an array - if (key === "image_url" || key === "video_url" || key === "tail_image_url") { - inputParams[key] = url; - } else { - inputParams[key] = [url]; - } - handledImageKeys.add(key); - } else if (Array.isArray(value)) { - // Array of values - check if they're data URLs that need uploading - const processedArray: string[] = []; - for (const item of value) { - if (typeof item === 'string' && item.startsWith('data:image')) { - const url = await uploadImageToKie(requestId, apiKey, item); - processedArray.push(url); - } else if (typeof item === 'string' && item.startsWith('http')) { - processedArray.push(item); - } else if (typeof item === 'string') { - processedArray.push(item); - } - } - if (processedArray.length > 0) { - inputParams[key] = processedArray; - handledImageKeys.add(key); - } - } else { - inputParams[key] = value; - } - } - } - } - - // Handle image inputs (fallback - only if dynamicInputs didn't already set the image key) - const imageKey = getKieImageInputKey(modelId); - if (input.images && input.images.length > 0 && !handledImageKeys.has(imageKey)) { - // Upload images to get URLs (Kie requires URLs, not base64) - const imageUrls: string[] = []; - for (const image of input.images) { - if (image.startsWith("http")) { - imageUrls.push(image); - } else { - // Upload base64 image - const url = await uploadImageToKie(requestId, apiKey, image); - imageUrls.push(url); - } - } - - // Some models use singular string, others use arrays - if (imageKey === "image_url" || imageKey === "video_url") { - inputParams[imageKey] = imageUrls[0]; - } else { - inputParams[imageKey] = imageUrls; - } - } - - // Veo 3 models use a different API endpoint and request format - if (isVeoModel(modelId)) { - const veoBody: Record = { - prompt: inputParams.prompt, - model: getVeoApiModelId(modelId), - aspect_ratio: inputParams.aspect_ratio || "16:9", - }; - - // Add image URLs if present (for image-to-video) - if (inputParams.imageUrls) { - veoBody.imageUrls = Array.isArray(inputParams.imageUrls) - ? inputParams.imageUrls - : [inputParams.imageUrls]; - } - - // Add optional seed - if (inputParams.seeds !== undefined) { - veoBody.seeds = inputParams.seeds; - } - - const veoUrl = "https://api.kie.ai/api/v1/veo/generate"; - console.log(`[API:${requestId}] Calling Veo API: ${veoUrl}`); - console.log(`[API:${requestId}] Veo request body:`, JSON.stringify(veoBody, null, 2)); - - const createResponse = await fetch(veoUrl, { - method: "POST", - headers: { - "Authorization": `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(veoBody), - }); - - if (!createResponse.ok) { - const errorText = await createResponse.text(); - let errorDetail = errorText; - try { - const errorJson = JSON.parse(errorText); - errorDetail = errorJson.message || errorJson.error || errorJson.detail || errorText; - } catch { - // Keep original text - } - if (createResponse.status === 429) { - return { success: false, error: `${input.model.name}: Rate limit exceeded. Try again in a moment.` }; - } - return { success: false, error: `${input.model.name}: ${errorDetail}` }; - } - - const createResult = await createResponse.json(); - if (createResult.code && createResult.code !== 200) { - return { success: false, error: `${input.model.name}: ${createResult.msg || "API error"}` }; - } - - const taskId = createResult.data?.taskId || createResult.taskId; - if (!taskId) { - console.error(`[API:${requestId}] No taskId in Veo response:`, createResult); - return { success: false, error: "No task ID in Veo response" }; - } - - console.log(`[API:${requestId}] Veo task created: ${taskId}`); - - // Poll with Veo-specific polling - const pollResult = await pollVeoTaskCompletion(requestId, apiKey, taskId); - if (!pollResult.success) { - return { success: false, error: `${input.model.name}: ${pollResult.error}` }; - } - - // Extract video URL from Veo response format - const data = pollResult.data; - let mediaUrl: string | null = null; - - const responseObj = data?.response as Record | undefined; - const resultUrls = (responseObj?.resultUrls || data?.resultUrls) as string[] | undefined; - if (resultUrls && resultUrls.length > 0) { - mediaUrl = resultUrls[0]; - } - - if (!mediaUrl) { - console.error(`[API:${requestId}] No media URL found in Veo response:`, data); - return { success: false, error: "No output URL in Veo response" }; - } - - // Validate URL before fetching - const mediaUrlCheck = validateMediaUrl(mediaUrl); - if (!mediaUrlCheck.valid) { - return { success: false, error: `Invalid media URL: ${mediaUrlCheck.error}` }; - } - - // Fetch the video and convert to base64 - console.log(`[API:${requestId}] Fetching Veo output from: ${mediaUrl.substring(0, 80)}...`); - const mediaResponse = await fetch(mediaUrl); - if (!mediaResponse.ok) { - return { success: false, error: `Failed to fetch output: ${mediaResponse.status}` }; - } - - const MAX_MEDIA_SIZE = 500 * 1024 * 1024; - const mediaContentLength = parseInt(mediaResponse.headers.get("content-length") || "0", 10); - if (mediaContentLength > MAX_MEDIA_SIZE) { - return { success: false, error: `Media too large: ${(mediaContentLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; - } - - const contentType = mediaResponse.headers.get("content-type") || "video/mp4"; - const mediaArrayBuffer = await mediaResponse.arrayBuffer(); - const mediaSizeMB = mediaArrayBuffer.byteLength / (1024 * 1024); - - console.log(`[API:${requestId}] Veo output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); - - // For very large videos (>20MB), return URL directly - if (mediaSizeMB > 20) { - console.log(`[API:${requestId}] SUCCESS - Returning URL for large Veo video`); - return { - success: true, - outputs: [{ type: "video", data: mediaUrl, url: mediaUrl }], - }; - } - - const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); - console.log(`[API:${requestId}] SUCCESS - Returning Veo video`); - return { - success: true, - outputs: [{ type: "video", data: `data:${contentType};base64,${mediaBase64}`, url: mediaUrl }], - }; - } - - // All remaining Kie models use the standard createTask endpoint - const requestBody: Record = { - model: modelId, - input: inputParams, - }; - - const createUrl = "https://api.kie.ai/api/v1/jobs/createTask"; - - console.log(`[API:${requestId}] Calling Kie.ai API: ${createUrl}`); - // Log full request body for debugging (truncate very long prompts) - const bodyForLogging = { ...requestBody }; - if (bodyForLogging.input && typeof bodyForLogging.input === 'object') { - const inputForLogging = { ...(bodyForLogging.input as Record) }; - if (typeof inputForLogging.prompt === 'string' && (inputForLogging.prompt as string).length > 200) { - inputForLogging.prompt = (inputForLogging.prompt as string).substring(0, 200) + '...[truncated]'; - } - bodyForLogging.input = inputForLogging; - } - console.log(`[API:${requestId}] Request body:`, JSON.stringify(bodyForLogging, null, 2)); - - // Create task - const createResponse = await fetch(createUrl, { - method: "POST", - headers: { - "Authorization": `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(requestBody), - }); - - if (!createResponse.ok) { - const errorText = await createResponse.text(); - let errorDetail = errorText; - try { - const errorJson = JSON.parse(errorText); - errorDetail = errorJson.message || errorJson.error || errorJson.detail || errorText; - } catch { - // Keep original text - } - - if (createResponse.status === 429) { - return { - success: false, - error: `${input.model.name}: Rate limit exceeded. Try again in a moment.`, - }; - } - - return { - success: false, - error: `${input.model.name}: ${errorDetail}`, - }; - } - - const createResult = await createResponse.json(); - - // Kie API returns HTTP 200 even on errors, check the response code - if (createResult.code && createResult.code !== 200) { - const errorMsg = createResult.msg || createResult.message || "API error"; - console.error(`[API:${requestId}] Kie API error (code ${createResult.code}):`, errorMsg); - return { - success: false, - error: `${input.model.name}: ${errorMsg}`, - }; - } - - const taskId = createResult.taskId || createResult.data?.taskId || createResult.id; - - if (!taskId) { - console.error(`[API:${requestId}] No taskId in Kie response:`, createResult); - return { - success: false, - error: "No task ID in response", - }; - } - - console.log(`[API:${requestId}] Kie task created: ${taskId}`); - - // Poll for completion - const pollResult = await pollKieTaskCompletion(requestId, apiKey, taskId); - - if (!pollResult.success) { - return { - success: false, - error: `${input.model.name}: ${pollResult.error}`, - }; - } - - // Extract output URL from result - // Kie API returns: { data: { status: "success", resultJson: { resultUrls: ["url1", "url2"] } } } - const data = pollResult.data; - let mediaUrl: string | null = null; - let isVideo = false; - - console.log(`[API:${requestId}] Kie poll result data:`, JSON.stringify(data).substring(0, 500)); - - // Try various response formats - Kie uses resultJson.resultUrls - // Note: resultJson is often a JSON string that needs parsing - if (data) { - let resultJson = data.resultJson as Record | string | undefined; - - // Parse resultJson if it's a string (Kie API returns it as escaped JSON string) - if (typeof resultJson === 'string') { - try { - resultJson = JSON.parse(resultJson) as Record; - } catch { - // Not valid JSON, keep as-is - resultJson = undefined; - } - } - - const resultUrls = ((resultJson as Record | undefined)?.resultUrls || data.resultUrls) as string[] | undefined; - - if (resultUrls && resultUrls.length > 0) { - mediaUrl = resultUrls[0]; - // Check if it's a video based on URL - isVideo = mediaUrl.includes('.mp4') || mediaUrl.includes('.webm') || mediaUrl.includes('video'); - } - // Fallback to other formats - else if (data.videoUrl) { - mediaUrl = data.videoUrl as string; - isVideo = true; - } else if (data.video_url) { - mediaUrl = data.video_url as string; - isVideo = true; - } else if (data.output && typeof data.output === 'string' && (data.output as string).includes('.mp4')) { - mediaUrl = data.output as string; - isVideo = true; - } - // Image outputs - else if (data.imageUrl) { - mediaUrl = data.imageUrl as string; - } else if (data.image_url) { - mediaUrl = data.image_url as string; - } else if (data.output && typeof data.output === 'string') { - mediaUrl = data.output as string; - } else if (data.url) { - mediaUrl = data.url as string; - } else if (Array.isArray(data.images) && data.images.length > 0) { - mediaUrl = (data.images[0] as { url?: string })?.url || data.images[0] as string; - } - } - - if (!mediaUrl) { - console.error(`[API:${requestId}] No media URL found in Kie response:`, data); - return { - success: false, - error: "No output URL in response", - }; - } - - // Detect video from URL if not already detected - if (!isVideo && (mediaUrl.includes('.mp4') || mediaUrl.includes('.webm') || mediaUrl.includes('video'))) { - isVideo = true; - } - - // Validate URL before fetching - const mediaUrlCheck = validateMediaUrl(mediaUrl); - if (!mediaUrlCheck.valid) { - return { success: false, error: `Invalid media URL: ${mediaUrlCheck.error}` }; - } - - // Fetch the media and convert to base64 - console.log(`[API:${requestId}] Fetching output from: ${mediaUrl.substring(0, 80)}...`); - const mediaResponse = await fetch(mediaUrl); - - if (!mediaResponse.ok) { - return { - success: false, - error: `Failed to fetch output: ${mediaResponse.status}`, - }; - } - - // Check file size before downloading body - const MAX_MEDIA_SIZE = 500 * 1024 * 1024; // 500MB - const mediaContentLength = parseInt(mediaResponse.headers.get("content-length") || "0", 10); - if (mediaContentLength > MAX_MEDIA_SIZE) { - return { success: false, error: `Media too large: ${(mediaContentLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; - } - - const contentType = mediaResponse.headers.get("content-type") || (isVideo ? "video/mp4" : "image/png"); - if (contentType.startsWith("video/")) { - isVideo = true; - } - - const mediaArrayBuffer = await mediaResponse.arrayBuffer(); - const mediaSizeBytes = mediaArrayBuffer.byteLength; - const mediaSizeMB = mediaSizeBytes / (1024 * 1024); - - console.log(`[API:${requestId}] Output: ${contentType}, ${mediaSizeMB.toFixed(2)}MB`); - - // For very large videos (>20MB), return URL directly - if (isVideo && mediaSizeMB > 20) { - console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); - return { - success: true, - outputs: [ - { - type: "video", - data: mediaUrl, - url: mediaUrl, - }, - ], - }; - } - - const mediaBase64 = Buffer.from(mediaArrayBuffer).toString("base64"); - console.log(`[API:${requestId}] SUCCESS - Returning ${isVideo ? "video" : "image"}`); +// Re-export for backward compatibility (test file imports from route) +export const clearFalInputMappingCache = _clearFalInputMappingCache; - return { - success: true, - outputs: [ - { - type: isVideo ? "video" : "image", - data: `data:${contentType};base64,${mediaBase64}`, - url: mediaUrl, - }, - ], - }; -} - -/** - * WaveSpeed task status from API - * Values: created → processing → completed/failed - */ -type WaveSpeedStatus = "created" | "pending" | "processing" | "completed" | "failed"; - -/** - * WaveSpeed submit response - * Format: { code: 200, message: "success", data: { id, model, status, urls, created_at } } - */ -interface WaveSpeedSubmitResponse { - code?: number; - message?: string; - data?: { - id: string; - model?: string; - status?: WaveSpeedStatus; - urls?: { - get?: string; - }; - created_at?: string; - }; - // Fallback fields for other response formats - id?: string; - status?: WaveSpeedStatus; - error?: string; -} +export const maxDuration = 300; // 5 minute timeout (Vercel hobby plan limit) +export const dynamic = 'force-dynamic'; // Ensure this route is always dynamic -/** - * WaveSpeed prediction/poll response (inner data object) - */ -interface WaveSpeedPredictionData { - id: string; - status: WaveSpeedStatus; - outputs?: string[]; - output?: { - images?: string[]; - videos?: string[]; - }; - timings?: { - inference?: number; - }; - created_at?: string; - error?: string; -} /** - * WaveSpeed prediction/poll response wrapper - * Format: { code: 200, message: "success", data: { id, status, outputs, ... } } + * Extended request format that supports both legacy and multi-provider requests */ -interface WaveSpeedPredictionResponse { - code?: number; - message?: string; - data?: WaveSpeedPredictionData; - // Fallback: some responses might have fields at top level - id?: string; - status?: WaveSpeedStatus; - outputs?: string[]; - error?: string; +interface MultiProviderGenerateRequest extends GenerateRequest { + selectedModel?: SelectedModel; + parameters?: Record; + /** Dynamic inputs from schema-based connections (e.g., image_url, tail_image_url, prompt) */ + dynamicInputs?: Record; } -/** - * Generate image/video using WaveSpeed API - * Uses async task submission + polling - */ -async function generateWithWaveSpeed( - requestId: string, - apiKey: string, - input: GenerationInput -): Promise { - console.log(`[API:${requestId}] WaveSpeed generation - Model: ${input.model.id}, Images: ${input.images?.length || 0}, Prompt: ${input.prompt.length} chars`); - - const WAVESPEED_API_BASE = "https://api.wavespeed.ai/api/v3"; - const modelId = input.model.id; - - // Validate modelId to prevent path traversal - if (/[^a-zA-Z0-9\-_/.]/.test(modelId) || modelId.includes('..')) { - return { success: false, error: `Invalid model ID: ${modelId}` }; - } - - const hasDynamicInputs = input.dynamicInputs && Object.keys(input.dynamicInputs).length > 0; - console.log(`[API:${requestId}] Dynamic inputs: ${hasDynamicInputs ? Object.keys(input.dynamicInputs!).join(", ") : "none"}`); - - // Determine output type from model capabilities - const isVideoModel = input.model.capabilities.includes("text-to-video") || - input.model.capabilities.includes("image-to-video"); - - // Build WaveSpeed payload - const payload: Record = { - prompt: input.prompt, - ...input.parameters, - }; - - // Apply dynamic inputs (schema-mapped connections) - // These have the correct parameter names from the schema (e.g., "images" for edit models) - if (hasDynamicInputs) { - for (const [key, value] of Object.entries(input.dynamicInputs!)) { - if (value !== null && value !== undefined && value !== '') { - // If the key is "images" and value is not an array, wrap it - if (key === "images" && !Array.isArray(value)) { - payload[key] = [value]; - } else { - payload[key] = value; - } - } - } - } else if (input.images && input.images.length > 0) { - // Fallback: if no dynamic inputs but images array is provided - // Use "image" for single image (default WaveSpeed format) - payload.image = input.images[0]; - } - - console.log(`[API:${requestId}] Submitting to WaveSpeed with inputs: ${Object.keys(payload).join(", ")}`); - - // Submit task - // Model ID goes directly in the URL path (slashes are part of the path) - const submitUrl = `${WAVESPEED_API_BASE}/${modelId}`; - console.log(`[API:${requestId}] WaveSpeed submit URL: ${submitUrl}`); - - const submitResponse = await fetch(submitUrl, { - method: "POST", - headers: { - Authorization: `Bearer ${apiKey}`, - "Content-Type": "application/json", - }, - body: JSON.stringify(payload), - }); - - if (!submitResponse.ok) { - const errorText = await submitResponse.text(); - let errorDetail = errorText || `HTTP ${submitResponse.status}`; - try { - const errorJson = JSON.parse(errorText); - errorDetail = errorJson.error || errorJson.message || errorJson.detail || errorText || `HTTP ${submitResponse.status}`; - } catch { - // Keep original text - } - - console.error(`[API:${requestId}] WaveSpeed submit failed: ${submitResponse.status} - ${errorDetail}`); - - if (submitResponse.status === 429) { - return { - success: false, - error: `${input.model.name || 'WaveSpeed'}: Rate limit exceeded. Try again in a moment.`, - }; - } - - return { - success: false, - error: `${input.model.name || 'WaveSpeed'}: ${errorDetail}`, - }; - } - - const submitResult: WaveSpeedSubmitResponse = await submitResponse.json(); - console.log(`[API:${requestId}] WaveSpeed submit response:`, JSON.stringify(submitResult).substring(0, 500)); - - const taskId = submitResult.data?.id || submitResult.id; - // Use the polling URL provided by the API if available, with SSRF validation - let providedPollUrl: string | undefined = submitResult.data?.urls?.get; - if (providedPollUrl) { - const pollUrlCheck = validateMediaUrl(providedPollUrl); - if (!pollUrlCheck.valid || !providedPollUrl.startsWith('https://api.wavespeed.ai')) { - console.warn(`[API:${requestId}] WaveSpeed provided invalid poll URL: ${providedPollUrl} — falling back to constructed URL`); - providedPollUrl = undefined; - } - } - - if (!taskId) { - console.error(`[API:${requestId}] No task ID in WaveSpeed submit response`); - return { - success: false, - error: "WaveSpeed: No task ID returned from API", - }; - } - - console.log(`[API:${requestId}] WaveSpeed task submitted: ${taskId}`); - if (providedPollUrl) { - console.log(`[API:${requestId}] WaveSpeed provided poll URL: ${providedPollUrl}`); - } - - // Poll for completion using the URL from the API response, or construct it - // Status flow: created → processing → completed/failed - const maxWaitTime = 5 * 60 * 1000; // 5 minutes - const pollInterval = 1000; // 1 second - const startTime = Date.now(); - let lastStatus = ""; - - let resultData: WaveSpeedPredictionResponse | null = null; - - while (true) { - if (Date.now() - startTime > maxWaitTime) { - console.error(`[API:${requestId}] WaveSpeed task timed out after 5 minutes`); - return { - success: false, - error: `${input.model.name}: Generation timed out after 5 minutes`, - }; - } - - await new Promise((resolve) => setTimeout(resolve, pollInterval)); - - // Use provided poll URL if available, otherwise construct it - const pollUrl = providedPollUrl || `${WAVESPEED_API_BASE}/predictions/${taskId}/result`; - const pollResponse = await fetch( - pollUrl, - { - headers: { - Authorization: `Bearer ${apiKey}`, - }, - } - ); - - // Log poll response status for debugging - const elapsedSec = Math.round((Date.now() - startTime) / 1000); - console.log(`[API:${requestId}] WaveSpeed poll (${elapsedSec}s): ${pollResponse.status} from ${pollUrl}`); - - // 404 means result not ready yet - continue polling - if (pollResponse.status === 404) { - lastStatus = "pending"; - continue; - } - - if (!pollResponse.ok) { - const errorText = await pollResponse.text(); - let errorDetail = errorText || `HTTP ${pollResponse.status}`; - try { - const errorJson = JSON.parse(errorText); - errorDetail = errorJson.error || errorJson.message || errorJson.detail || errorDetail; - } catch { - // Keep original text - } - console.error(`[API:${requestId}] WaveSpeed poll failed: ${pollResponse.status} - ${errorDetail}`); - return { - success: false, - error: `${input.model.name}: ${errorDetail}`, - }; - } - - const pollData: WaveSpeedPredictionResponse = await pollResponse.json(); - console.log(`[API:${requestId}] WaveSpeed poll data:`, JSON.stringify(pollData).substring(0, 300)); - - // Extract status from nested data object (WaveSpeed wraps response in { code, message, data: {...} }) - const currentStatus = pollData.data?.status || pollData.status; - const currentError = pollData.data?.error || pollData.error; - - // Log status changes - if (currentStatus !== lastStatus) { - console.log(`[API:${requestId}] WaveSpeed status changed: ${lastStatus} → ${currentStatus}`); - lastStatus = currentStatus || ""; - } - - // Check if task is complete - if (currentStatus === "completed") { - console.log(`[API:${requestId}] WaveSpeed task completed`); - resultData = pollData; - break; - } - - // Check if task failed - if (currentStatus === "failed") { - const failureReason = currentError || pollData.message || "Generation failed"; - console.error(`[API:${requestId}] WaveSpeed task failed: ${failureReason}`); - return { - success: false, - error: `${input.model.name}: ${failureReason}`, - }; - } - - // Continue polling for "created" or "processing" status - } - - // Safety check (should never happen since we break on completed) - if (!resultData) { - return { - success: false, - error: `${input.model.name}: No result received`, - }; - } - - // Extract outputs - WaveSpeed wraps response in { code, message, data: { outputs: [...] } } - let outputUrls: string[] = []; - const resultDataInner = resultData.data; - - // Format 1: data.outputs array (standard WaveSpeed format) - if (resultDataInner?.outputs && Array.isArray(resultDataInner.outputs) && resultDataInner.outputs.length > 0) { - outputUrls = resultDataInner.outputs; - } - // Format 2: data.output object with images/videos arrays - else if (resultDataInner?.output) { - if (isVideoModel && resultDataInner.output.videos && resultDataInner.output.videos.length > 0) { - outputUrls = resultDataInner.output.videos; - } else if (resultDataInner.output.images && resultDataInner.output.images.length > 0) { - outputUrls = resultDataInner.output.images; - } - } - // Format 3: Fallback - outputs at top level (unlikely but safe) - else if (resultData.outputs && Array.isArray(resultData.outputs) && resultData.outputs.length > 0) { - outputUrls = resultData.outputs; - } - - if (outputUrls.length === 0) { - console.error(`[API:${requestId}] No outputs in WaveSpeed result. Response:`, JSON.stringify(resultData).substring(0, 500)); - return { - success: false, - error: `${input.model.name}: No outputs in generation result`, - }; - } - - // Fetch the first output and convert to base64 - const outputUrl = outputUrls[0]; - - // Validate URL before fetching - const outputUrlCheck = validateMediaUrl(outputUrl); - if (!outputUrlCheck.valid) { - return { success: false, error: `Invalid output URL: ${outputUrlCheck.error}` }; - } - - console.log(`[API:${requestId}] Fetching WaveSpeed output from: ${outputUrl.substring(0, 80)}...`); - - const outputResponse = await fetch(outputUrl); - - if (!outputResponse.ok) { - return { - success: false, - error: `Failed to fetch output: ${outputResponse.status}`, - }; - } - - // Check file size before downloading body - const MAX_MEDIA_SIZE_WS = 500 * 1024 * 1024; // 500MB - const wsContentLength = parseInt(outputResponse.headers.get("content-length") || "0", 10); - if (wsContentLength > MAX_MEDIA_SIZE_WS) { - return { success: false, error: `Media too large: ${(wsContentLength / (1024 * 1024)).toFixed(0)}MB > 500MB limit` }; - } - - const outputArrayBuffer = await outputResponse.arrayBuffer(); - const outputSizeMB = outputArrayBuffer.byteLength / (1024 * 1024); - - const rawContentType = outputResponse.headers.get("content-type"); - const contentType = - (rawContentType && (rawContentType.startsWith("video/") || rawContentType.startsWith("image/"))) - ? rawContentType - : (isVideoModel ? "video/mp4" : "image/png"); - - console.log(`[API:${requestId}] Output: ${contentType}, ${outputSizeMB.toFixed(2)}MB`); - - // For very large videos (>20MB), return URL directly instead of base64 - if (isVideoModel && outputSizeMB > 20) { - console.log(`[API:${requestId}] SUCCESS - Returning URL for large video`); - return { - success: true, - outputs: [ - { - type: "video", - data: outputUrl, - url: outputUrl, - }, - ], - }; - } - - const outputBase64 = Buffer.from(outputArrayBuffer).toString("base64"); - console.log(`[API:${requestId}] SUCCESS - Returning ${isVideoModel ? "video" : "image"}`); - - return { - success: true, - outputs: [ - { - type: isVideoModel ? "video" : "image", - data: `data:${contentType};base64,${outputBase64}`, - url: outputUrl, - }, - ], - }; -} export async function POST(request: NextRequest) { const requestId = Math.random().toString(36).substring(7); @@ -2445,7 +125,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "replicate", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -2468,7 +148,7 @@ export async function POST(request: NextRequest) { // Return first output (image or video) const output = result.outputs?.[0]; - if (!output?.data) { + if (!output?.data && !output?.url) { return NextResponse.json( { success: false, @@ -2479,13 +159,21 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { - // Check if data is a URL (for large videos) or base64 - const isUrl = output.data.startsWith("http"); + // Large videos have data="" with url set; normal videos have base64 data + const isLargeVideo = !output.data && output.url; return NextResponse.json({ success: true, - video: isUrl ? undefined : output.data, - videoUrl: isUrl ? output.data : undefined, + video: isLargeVideo ? undefined : output.data, + videoUrl: isLargeVideo ? output.url : undefined, contentType: "video", }); } @@ -2532,7 +220,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "fal", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -2555,7 +243,7 @@ export async function POST(request: NextRequest) { // Return first output (image or video) const output = result.outputs?.[0]; - if (!output?.data) { + if (!output?.data && !output?.url) { return NextResponse.json( { success: false, @@ -2566,13 +254,21 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { - // Check if data is a URL (for large videos) or base64 - const isUrl = output.data.startsWith("http"); + // Large videos have data="" with url set; normal videos have base64 data + const isLargeVideo = !output.data && output.url; return NextResponse.json({ success: true, - video: isUrl ? undefined : output.data, - videoUrl: isUrl ? output.data : undefined, + video: isLargeVideo ? undefined : output.data, + videoUrl: isLargeVideo ? output.url : undefined, contentType: "video", }); } @@ -2623,7 +319,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "kie", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -2646,7 +342,7 @@ export async function POST(request: NextRequest) { // Return first output (image or video) const output = result.outputs?.[0]; - if (!output?.data) { + if (!output?.data && !output?.url) { return NextResponse.json( { success: false, @@ -2657,13 +353,21 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { - // Check if data is a URL (for large videos) or base64 - const isUrl = output.data.startsWith("http"); + // Large videos have data="" with url set; normal videos have base64 data + const isLargeVideo = !output.data && output.url; return NextResponse.json({ success: true, - video: isUrl ? undefined : output.data, - videoUrl: isUrl ? output.data : undefined, + video: isLargeVideo ? undefined : output.data, + videoUrl: isLargeVideo ? output.url : undefined, contentType: "video", }); } @@ -2714,7 +418,7 @@ export async function POST(request: NextRequest) { id: selectedModel!.modelId, name: selectedModel!.displayName, provider: "wavespeed", - capabilities: mediaType === "video" ? ["text-to-video"] : ["text-to-image"], + capabilities: mediaType === "video" ? ["text-to-video"] : mediaType === "3d" ? ["text-to-3d"] : ["text-to-image"], description: null, }, prompt: prompt || "", @@ -2737,7 +441,7 @@ export async function POST(request: NextRequest) { // Return first output (image or video) const output = result.outputs?.[0]; - if (!output?.data) { + if (!output?.data && !output?.url) { return NextResponse.json( { success: false, @@ -2748,12 +452,21 @@ export async function POST(request: NextRequest) { } // Return appropriate fields based on output type + if (output.type === "3d") { + return NextResponse.json({ + success: true, + model3dUrl: output.url, + contentType: "3d", + }); + } + if (output.type === "video") { - const isUrl = output.data.startsWith("http"); + // Large videos have data="" with url set; normal videos have base64 data + const isLargeVideo = !output.data && output.url; return NextResponse.json({ success: true, - video: isUrl ? undefined : output.data, - videoUrl: isUrl ? output.data : undefined, + video: isLargeVideo ? undefined : output.data, + videoUrl: isLargeVideo ? output.url : undefined, contentType: "video", }); } diff --git a/src/app/api/generate/schemaUtils.ts b/src/app/api/generate/schemaUtils.ts new file mode 100644 index 00000000..ac66e378 --- /dev/null +++ b/src/app/api/generate/schemaUtils.ts @@ -0,0 +1,193 @@ +/** + * Schema Utilities for Generate API Route + * + * Provides input parameter pattern matching, type extraction, and coercion + * from OpenAPI schemas used by multi-provider generation. + */ + +/** + * Input parameter patterns - maps generic input types to possible schema parameter names + */ +export const INPUT_PATTERNS: Record = { + // Text/prompt inputs + prompt: ["prompt", "text", "caption", "input_text", "description", "query"], + negativePrompt: ["negative_prompt", "negative", "neg_prompt", "negative_text"], + + // Image inputs + image: ["image_url", "image_urls", "image", "first_frame", "start_image", "init_image", + "reference_image", "input_image", "image_input", "source_image", "img", "photo"], + + // Video/media settings + aspectRatio: ["aspect_ratio", "ratio", "size", "dimensions", "output_size"], + duration: ["duration", "length", "num_frames", "seconds", "video_length"], + fps: ["fps", "frame_rate", "framerate", "frames_per_second"], + + // Audio settings + audio: ["audio_enabled", "with_audio", "enable_audio", "audio", "sound"], + + // Generation settings + seed: ["seed", "random_seed", "noise_seed"], + steps: ["steps", "num_steps", "num_inference_steps", "inference_steps"], + guidance: ["guidance_scale", "guidance", "cfg_scale", "cfg"], + + // Model-specific + scheduler: ["scheduler", "sampler", "sampler_name"], + strength: ["strength", "denoise", "denoising_strength"], +}; + +/** + * Input mapping result from schema parsing + */ +export interface InputMapping { + // Maps our generic names to model-specific parameter names + paramMap: Record; + // Track which generic params expect array types (e.g., "image") + arrayParams: Set; + // Track actual schema param names that expect array types (e.g., "image_urls") + schemaArrayParams: Set; +} + +/** + * Parameter type information extracted from OpenAPI schema + */ +export interface ParameterTypeInfo { + [paramName: string]: "string" | "integer" | "number" | "boolean" | "array" | "object"; +} + +/** + * Extract parameter types from OpenAPI schema + */ +export function getParameterTypesFromSchema(schema: Record | undefined): ParameterTypeInfo { + const typeInfo: ParameterTypeInfo = {}; + + if (!schema) return typeInfo; + + try { + const components = schema.components as Record | undefined; + const schemas = components?.schemas as Record | undefined; + const input = schemas?.Input as Record | undefined; + const properties = input?.properties as Record | undefined; + + if (!properties) return typeInfo; + + for (const [propName, prop] of Object.entries(properties)) { + const property = prop as Record; + const type = property?.type as string | undefined; + if (type && ["string", "integer", "number", "boolean", "array", "object"].includes(type)) { + typeInfo[propName] = type as ParameterTypeInfo[string]; + } + } + } catch { + // Schema parsing failed + } + + return typeInfo; +} + +/** + * Coerce parameter values to their expected types based on schema + * This handles cases where values were incorrectly stored as strings (e.g., from UI enum selects) + */ +export function coerceParameterTypes( + parameters: Record | undefined, + typeInfo: ParameterTypeInfo +): Record { + if (!parameters) return {}; + + const result = { ...parameters }; + + for (const [key, value] of Object.entries(result)) { + if (value === undefined || value === null) continue; + + const expectedType = typeInfo[key]; + if (!expectedType) continue; + + // Coerce string values to their expected types + if (typeof value === "string") { + if (expectedType === "integer") { + const parsed = parseInt(value, 10); + if (!isNaN(parsed)) result[key] = parsed; + } else if (expectedType === "number") { + const parsed = parseFloat(value); + if (!isNaN(parsed)) result[key] = parsed; + } else if (expectedType === "boolean") { + result[key] = value === "true"; + } + } + } + + return result; +} + +/** + * Extract input parameter mappings from OpenAPI schema + * Returns a mapping of generic parameter names to model-specific names + */ +export function getInputMappingFromSchema(schema: Record | undefined): InputMapping { + const paramMap: Record = {}; + const arrayParams = new Set(); + const schemaArrayParams = new Set(); + + if (!schema) return { paramMap, arrayParams, schemaArrayParams }; + + try { + // Navigate to input schema properties + const components = schema.components as Record | undefined; + const schemas = components?.schemas as Record | undefined; + const input = schemas?.Input as Record | undefined; + const properties = input?.properties as Record | undefined; + + if (!properties) return { paramMap, arrayParams, schemaArrayParams }; + + // First pass: detect all array-typed properties by their actual schema name + for (const [propName, prop] of Object.entries(properties)) { + const property = prop as Record; + if (property?.type === "array") { + schemaArrayParams.add(propName); + } + } + + const propertyNames = Object.keys(properties); + + // For each input type pattern, find the matching schema property + for (const [genericName, patterns] of Object.entries(INPUT_PATTERNS)) { + for (const pattern of patterns) { + let matchedParam: string | null = null; + + // Check for exact match first + if (properties[pattern]) { + matchedParam = pattern; + } else { + // Check for case-insensitive partial match + const patternLower = pattern.toLowerCase(); + const match = propertyNames.find(name => { + const nameLower = name.toLowerCase(); + // Property name contains the pattern (intended direction) + if (nameLower.includes(patternLower)) return true; + // Pattern contains the property name — only allow for longer patterns + // to prevent short property names (e.g. "id") matching everything + if (patternLower.length >= 3 && patternLower.includes(nameLower)) return true; + return false; + }); + if (match) { + matchedParam = match; + } + } + + if (matchedParam) { + paramMap[genericName] = matchedParam; + // Check if this property expects an array type + const property = properties[matchedParam] as Record; + if (property?.type === "array") { + arrayParams.add(genericName); + } + break; + } + } + } + } catch { + // Schema parsing failed + } + + return { paramMap, arrayParams, schemaArrayParams }; +} diff --git a/src/app/api/models/route.ts b/src/app/api/models/route.ts index b95a87e5..7b1a8b91 100644 --- a/src/app/api/models/route.ts +++ b/src/app/api/models/route.ts @@ -43,12 +43,14 @@ const REPLICATE_API_BASE = "https://api.replicate.com/v1"; const FAL_API_BASE = "https://api.fal.ai/v1"; const WAVESPEED_API_BASE = "https://api.wavespeed.ai/api/v3"; -// Categories we care about for image/video generation (fal.ai) +// Categories we care about for image/video/3D generation (fal.ai) const RELEVANT_CATEGORIES = [ "text-to-image", "image-to-image", "text-to-video", "image-to-video", + "text-to-3d", + "image-to-3d", ]; // Kie.ai models (hardcoded - no discovery API available) @@ -420,7 +422,32 @@ function inferReplicateCapabilities(model: ReplicateModel): ModelCapability[] { const capabilities: ModelCapability[] = []; const searchText = `${model.name} ${model.description ?? ""}`.toLowerCase(); - // Check for video-related keywords first + // Check for 3D-related keywords first + const is3DModel = + searchText.includes("3d") || + searchText.includes("mesh") || + searchText.includes("triposr") || + searchText.includes("tripo") || + searchText.includes("hunyuan3d") || + searchText.includes("instant-mesh") || + searchText.includes("point-e") || + searchText.includes("shap-e"); + + if (is3DModel) { + // 3D model - determine if image-to-3d or text-to-3d + const hasImageInput = + searchText.includes("image") || + searchText.includes("img") || + searchText.includes("photo"); + if (hasImageInput) { + capabilities.push("image-to-3d"); + } else { + capabilities.push("text-to-3d"); + } + return capabilities; + } + + // Check for video-related keywords const isVideoModel = searchText.includes("video") || searchText.includes("animate") || @@ -559,6 +586,27 @@ function inferWaveSpeedCapabilities(model: WaveSpeedModel): ModelCapability[] { const category = (model.category || model.type || "").toLowerCase(); const searchText = `${modelId} ${name} ${description} ${category}`; + // Check for 3D-related keywords first + const is3DModel = + searchText.includes("3d") || + searchText.includes("mesh") || + searchText.includes("tripo") || + searchText.includes("hunyuan3d") || + category.includes("3d"); + + if (is3DModel) { + const hasImageInput = + searchText.includes("image") || + searchText.includes("img") || + searchText.includes("photo"); + if (hasImageInput) { + capabilities.push("image-to-3d"); + } else { + capabilities.push("text-to-3d"); + } + return capabilities; + } + // Check for video-related keywords const isVideoModel = searchText.includes("video") || diff --git a/src/app/api/save-generation/route.ts b/src/app/api/save-generation/route.ts index 9c17be2b..01443f35 100644 --- a/src/app/api/save-generation/route.ts +++ b/src/app/api/save-generation/route.ts @@ -15,6 +15,7 @@ function getExtensionFromMime(mimeType: string): string { "video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", + "model/gltf-binary": "glb", }; // Check explicit mapping first @@ -29,6 +30,9 @@ function getExtensionFromMime(mimeType: string): string { if (mimeType.startsWith("video/")) { return "mp4"; } + if (mimeType.startsWith("model/")) { + return "glb"; + } // Unknown type - use generic binary extension return "bin"; @@ -69,18 +73,21 @@ export async function POST(request: NextRequest) { directoryPath = body.directoryPath; const image = body.image; const video = body.video; + const model3d = body.model3d; const prompt = body.prompt; const imageId = body.imageId; // Optional ID for carousel support const customFilename = body.customFilename; // Optional custom filename (without extension) const createDirectory = body.createDirectory; // Optional flag to create directory if it doesn't exist const isVideo = !!video; - const content = video || image; + const isModel = !!model3d; + const content = video || model3d || image; logger.info('file.save', 'Generation auto-save request received', { directoryPath, hasImage: !!image, hasVideo: !!video, + hasModel3d: !!model3d, prompt, customFilename, }); @@ -166,9 +173,9 @@ export async function POST(request: NextRequest) { } const rawSaveContentType = response.headers.get("content-type"); - const contentType = (rawSaveContentType && (rawSaveContentType.startsWith("video/") || rawSaveContentType.startsWith("image/"))) + const contentType = (rawSaveContentType && (rawSaveContentType.startsWith("video/") || rawSaveContentType.startsWith("image/") || rawSaveContentType.startsWith("model/"))) ? rawSaveContentType - : (isVideo ? "video/mp4" : "image/png"); + : (isModel ? "model/gltf-binary" : isVideo ? "video/mp4" : "image/png"); extension = getExtensionFromMime(contentType); const arrayBuffer = await response.arrayBuffer(); @@ -203,7 +210,7 @@ export async function POST(request: NextRequest) { // Safety net: if extension resolved to "bin" but we know the media type, use correct extension if (extension === "bin") { - extension = isVideo ? "mp4" : "png"; + extension = isModel ? "glb" : isVideo ? "mp4" : "png"; } // Compute content hash for deduplication @@ -258,6 +265,7 @@ export async function POST(request: NextRequest) { filename, fileSize: buffer.length, isVideo, + isModel, contentHash, }); diff --git a/src/app/globals.css b/src/app/globals.css index 5c2b9936..ad52a5df 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -10,6 +10,7 @@ /* Handle colors (used by labels) */ --handle-color-image: #10b981; --handle-color-text: #3b82f6; + --handle-color-3d: #f97316; } html { @@ -87,6 +88,11 @@ body { background: #3b82f6; } +/* 3D handles - orange */ +.react-flow__handle[data-handletype="3d"] { + background: #f97316; +} + .react-flow__edge-path { stroke: #94a3b8; stroke-width: 3; diff --git a/src/components/AnnotationModal.tsx b/src/components/AnnotationModal.tsx index 351ca479..90cdf5c3 100644 --- a/src/components/AnnotationModal.tsx +++ b/src/components/AnnotationModal.tsx @@ -323,7 +323,7 @@ export function AnnotationModal() { const handleDone = useCallback(() => { if (!sourceNodeId) return; const flattenedImage = flattenImage(); - updateNodeData(sourceNodeId, { annotations, outputImage: flattenedImage }); + updateNodeData(sourceNodeId, { annotations, outputImage: flattenedImage, outputImageRef: undefined }); closeModal(); }, [sourceNodeId, annotations, flattenImage, updateNodeData, closeModal]); diff --git a/src/components/ConnectionDropMenu.tsx b/src/components/ConnectionDropMenu.tsx index 5127de91..a8a008d0 100644 --- a/src/components/ConnectionDropMenu.tsx +++ b/src/components/ConnectionDropMenu.tsx @@ -149,6 +149,15 @@ const IMAGE_SOURCE_OPTIONS: MenuOption[] = [ ), }, + { + type: "glbViewer", + label: "3D Viewer", + icon: ( + + + + ), + }, { type: "annotation", label: "Annotate", @@ -296,9 +305,35 @@ const AUDIO_SOURCE_OPTIONS: MenuOption[] = [ }, ]; +// 3D target options (nodes that accept 3D input) +const THREE_D_TARGET_OPTIONS: MenuOption[] = [ + { + type: "glbViewer", + label: "3D Viewer", + icon: ( + + + + ), + }, +]; + +// 3D source options (nodes that produce 3D output) +const THREE_D_SOURCE_OPTIONS: MenuOption[] = [ + { + type: "generate3d", + label: "Generate 3D", + icon: ( + + + + ), + }, +]; + interface ConnectionDropMenuProps { position: { x: number; y: number }; - handleType: "image" | "text" | "video" | "audio" | "easeCurve" | null; + handleType: "image" | "text" | "video" | "audio" | "3d" | "easeCurve" | null; connectionType: "source" | "target"; // source = dragging from output, target = dragging from input onSelect: (selection: { type: NodeType | MenuAction; isAction: boolean }) => void; onClose: () => void; @@ -322,11 +357,13 @@ export function ConnectionDropMenu({ // Dragging from a source handle (output), need nodes with target handles (inputs) if (handleType === "video") return VIDEO_TARGET_OPTIONS; if (handleType === "audio") return AUDIO_TARGET_OPTIONS; + if (handleType === "3d") return THREE_D_TARGET_OPTIONS; return handleType === "image" ? IMAGE_TARGET_OPTIONS : TEXT_TARGET_OPTIONS; } else { // Dragging from a target handle (input), need nodes with source handles (outputs) if (handleType === "video") return VIDEO_SOURCE_OPTIONS; if (handleType === "audio") return AUDIO_SOURCE_OPTIONS; + if (handleType === "3d") return THREE_D_SOURCE_OPTIONS; return handleType === "image" ? IMAGE_SOURCE_OPTIONS : TEXT_SOURCE_OPTIONS; } }, [handleType, connectionType]); diff --git a/src/components/CostIndicator.tsx b/src/components/CostIndicator.tsx index e07e872b..e701d206 100644 --- a/src/components/CostIndicator.tsx +++ b/src/components/CostIndicator.tsx @@ -2,7 +2,7 @@ import { useState, useMemo } from "react"; import { useWorkflowStore } from "@/store/workflowStore"; -import { calculatePredictedCost, formatCost } from "@/utils/costCalculator"; +import { calculatePredictedCost, formatCost, hasNonGeminiProviders } from "@/utils/costCalculator"; import { CostDialog } from "./CostDialog"; export function CostIndicator() { @@ -14,9 +14,10 @@ export function CostIndicator() { return calculatePredictedCost(nodes); }, [nodes]); + const nonGemini = useMemo(() => hasNonGeminiProviders(nodes), [nodes]); const hasAnyNodes = predictedCost.nodeCount > 0; - if (!hasAnyNodes && incurredCost === 0) { + if (nonGemini || (!hasAnyNodes && incurredCost === 0)) { return null; } diff --git a/src/components/FloatingActionBar.tsx b/src/components/FloatingActionBar.tsx index 69958f20..2aa4b05c 100644 --- a/src/components/FloatingActionBar.tsx +++ b/src/components/FloatingActionBar.tsx @@ -136,6 +136,17 @@ function GenerateComboButton() { Video + + )} diff --git a/src/components/Header.tsx b/src/components/Header.tsx index 0029cba1..aeb28f80 100644 --- a/src/components/Header.tsx +++ b/src/components/Header.tsx @@ -4,6 +4,7 @@ import { useState, useRef, useMemo, useCallback } from "react"; import { useWorkflowStore, WorkflowFile } from "@/store/workflowStore"; import { ProjectSetupModal } from "./ProjectSetupModal"; import { CostIndicator } from "./CostIndicator"; +import { KeyboardShortcutsDialog } from "./KeyboardShortcutsDialog"; function CommentsNavigationIcon() { // Subscribe to nodes so we re-render when comments change @@ -69,6 +70,8 @@ export function Header() { loadWorkflow, previousWorkflowSnapshot, revertToSnapshot, + shortcutsDialogOpen, + setShortcutsDialogOpen, } = useWorkflowStore(); const [showProjectModal, setShowProjectModal] = useState(false); @@ -168,6 +171,35 @@ export function Header() { } }, [revertToSnapshot]); + const settingsButtons = ( +
+ +
+ ); + return ( <> - {/* Settings - separated */} - + {settingsButtons} ) : ( <> @@ -337,31 +345,7 @@ export function Header() { - {/* Settings - separated */} - + {settingsButtons} )} @@ -401,6 +385,17 @@ export function Header() { Made by Willie · + + · + setShortcutsDialogOpen(false)} + /> ); } diff --git a/src/components/KeyboardShortcutsDialog.tsx b/src/components/KeyboardShortcutsDialog.tsx new file mode 100644 index 00000000..9fea1570 --- /dev/null +++ b/src/components/KeyboardShortcutsDialog.tsx @@ -0,0 +1,149 @@ +"use client"; + +import { useEffect } from "react"; + +interface ShortcutItem { + keys: string[]; + description: string; +} + +interface ShortcutGroup { + title: string; + shortcuts: ShortcutItem[]; +} + +const isMac = typeof navigator !== "undefined" && /Mac|iPod|iPhone|iPad/.test(navigator.userAgent); +const modKey = isMac ? "⌘" : "Ctrl"; + +const shortcutGroups: ShortcutGroup[] = [ + { + title: "General", + shortcuts: [ + { keys: [`${modKey}`, "Enter"], description: "Run workflow" }, + { keys: [`${modKey}`, "C"], description: "Copy selected nodes" }, + { keys: [`${modKey}`, "V"], description: "Paste nodes / image / text" }, + { keys: ["?"], description: "Show keyboard shortcuts" }, + ], + }, + { + title: "Add Nodes", + shortcuts: [ + { keys: ["Shift", "P"], description: "Add Prompt node" }, + { keys: ["Shift", "I"], description: "Add Image Input node" }, + { keys: ["Shift", "G"], description: "Add Generate Image node" }, + { keys: ["Shift", "V"], description: "Add Generate Video node" }, + { keys: ["Shift", "L"], description: "Add LLM Text node" }, + { keys: ["Shift", "A"], description: "Add Annotation node" }, + ], + }, + { + title: "Layout (select 2+ nodes first)", + shortcuts: [ + { keys: ["V"], description: "Stack selected vertically" }, + { keys: ["H"], description: "Stack selected horizontally" }, + { keys: ["G"], description: "Arrange selected as grid" }, + ], + }, + { + title: "Canvas", + shortcuts: [ + { keys: ["Scroll"], description: "Zoom in / out" }, + { keys: ["Trackpad"], description: "Pan (macOS)" }, + { keys: ["Delete"], description: "Delete selected nodes" }, + ], + }, +]; + +function Kbd({ children }: { children: string }) { + return ( + + {children} + + ); +} + +interface KeyboardShortcutsDialogProps { + isOpen: boolean; + onClose: () => void; +} + +export function KeyboardShortcutsDialog({ isOpen, onClose }: KeyboardShortcutsDialogProps) { + useEffect(() => { + if (!isOpen) return; + + const handleKeyDown = (e: KeyboardEvent) => { + if (e.key === "Escape") { + onClose(); + } + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [isOpen, onClose]); + + if (!isOpen) return null; + + return ( +
+
+ {/* Header */} +
+

+ Keyboard Shortcuts +

+ +
+ + {/* Content */} +
+ {shortcutGroups.map((group) => ( +
+

+ {group.title} +

+
+ {group.shortcuts.map((shortcut, idx) => ( +
+ + {shortcut.description} + +
+ {shortcut.keys.map((key, keyIdx) => ( + + {keyIdx > 0 && ( + + + )} + {key} + + ))} +
+
+ ))} +
+
+ ))} +
+ + {/* Footer */} +
+ +
+
+
+ ); +} + diff --git a/src/components/ProjectSetupModal.tsx b/src/components/ProjectSetupModal.tsx index f234080a..4f12c64a 100644 --- a/src/components/ProjectSetupModal.tsx +++ b/src/components/ProjectSetupModal.tsx @@ -3,6 +3,7 @@ import { useState, useEffect } from "react"; import { generateWorkflowId, useWorkflowStore } from "@/store/workflowStore"; import { ProviderType, ProviderSettings, NodeDefaultsConfig, LLMProvider, LLMModelType } from "@/types"; +import { CanvasNavigationSettings, PanMode, ZoomMode, SelectionMode } from "@/types/canvas"; import { EnvStatusResponse } from "@/app/api/env-status/route"; import { loadNodeDefaults, saveNodeDefaults } from "@/store/utils/localStorage"; import { ProviderModel } from "@/lib/providers/types"; @@ -94,10 +95,12 @@ export function ProjectSetupModal({ toggleProvider, maxConcurrentCalls, setMaxConcurrentCalls, + canvasNavigationSettings, + updateCanvasNavigationSettings, } = useWorkflowStore(); // Tab state - const [activeTab, setActiveTab] = useState<"project" | "providers" | "nodeDefaults">("project"); + const [activeTab, setActiveTab] = useState<"project" | "providers" | "nodeDefaults" | "canvas">("project"); // Project tab state const [name, setName] = useState(""); @@ -132,6 +135,9 @@ export function ProjectSetupModal({ const [showImageModelDialog, setShowImageModelDialog] = useState(false); const [showVideoModelDialog, setShowVideoModelDialog] = useState(false); + // Canvas tab state + const [localCanvasSettings, setLocalCanvasSettings] = useState(canvasNavigationSettings); + // Pre-fill when opening in settings mode useEffect(() => { if (isOpen) { @@ -169,13 +175,16 @@ export function ProjectSetupModal({ setShowImageModelDialog(false); setShowVideoModelDialog(false); + // Sync canvas settings + setLocalCanvasSettings(canvasNavigationSettings); + // Fetch env status fetch("/api/env-status") .then((res) => res.json()) .then((data: EnvStatusResponse) => setEnvStatus(data)) .catch(() => setEnvStatus(null)); } - }, [isOpen, mode, workflowName, saveDirectoryPath, useExternalImageStorage, providerSettings]); + }, [isOpen, mode, workflowName, saveDirectoryPath, useExternalImageStorage, providerSettings, canvasNavigationSettings]); const handleBrowse = async () => { setIsBrowsing(true); @@ -285,11 +294,18 @@ export function ProjectSetupModal({ onClose(); }; + const handleSaveCanvas = () => { + updateCanvasNavigationSettings(localCanvasSettings); + onClose(); + }; + const handleSave = () => { if (activeTab === "project") { handleSaveProject(); } else if (activeTab === "providers") { handleSaveProviders(); + } else if (activeTab === "canvas") { + handleSaveCanvas(); } else { handleSaveNodeDefaults(); } @@ -324,15 +340,16 @@ export function ProjectSetupModal({ return (
-

- {mode === "new" ? "New Project" : "Project Settings"} -

+
+

+ {mode === "new" ? "New Project" : "Project Settings"} +

- {/* Tab Bar */} -
+ {/* Tab Bar */} +
+ +
+ {/* Scrollable tab content area */} +
+ {/* Project Tab Content */} {activeTab === "project" && (
@@ -961,7 +988,117 @@ export function ProjectSetupModal({
)} -
+ {/* Canvas Tab Content */} + {activeTab === "canvas" && ( +
+ {/* Pan Mode */} +
+

Pan Mode

+
+ {([ + { value: "space" as PanMode, label: "Space + Drag", description: "Hold Space and drag to pan (default)" }, + { value: "middleMouse" as PanMode, label: "Middle Mouse", description: "Click and drag with middle mouse button" }, + { value: "always" as PanMode, label: "Always On", description: "Pan without holding any keys" }, + ] as const).map((option) => ( + + ))} +
+
+ + {/* Zoom Mode */} +
+

Zoom Mode

+
+ {([ + { value: "altScroll" as ZoomMode, label: "Alt + Scroll", description: "Hold Alt and scroll to zoom (default)" }, + { value: "ctrlScroll" as ZoomMode, label: "Ctrl + Scroll", description: "Hold Ctrl/Cmd and scroll to zoom" }, + { value: "scroll" as ZoomMode, label: "Scroll", description: "Scroll to zoom without holding any keys" }, + ] as const).map((option) => ( + + ))} +
+
+ + {/* Selection Mode */} +
+

Selection Mode

+
+ {([ + { value: "click" as SelectionMode, label: "Click", description: "Click to select nodes (default)" }, + { value: "altDrag" as SelectionMode, label: "Alt + Drag", description: "Hold Alt and drag to select multiple nodes" }, + { value: "shiftDrag" as SelectionMode, label: "Shift + Drag", description: "Hold Shift and drag to select multiple nodes" }, + ] as const).map((option) => ( + + ))} +
+
+
+ )} + +
+ + {/* Fixed footer */} +
); })} diff --git a/src/components/WorkflowCanvas.tsx b/src/components/WorkflowCanvas.tsx index 8d6d0671..0c8da611 100644 --- a/src/components/WorkflowCanvas.tsx +++ b/src/components/WorkflowCanvas.tsx @@ -19,6 +19,7 @@ import "@xyflow/react/dist/style.css"; import { useWorkflowStore, WorkflowFile } from "@/store/workflowStore"; import { useToast } from "@/components/Toast"; +import dynamic from "next/dynamic"; import { ImageInputNode, AudioInputNode, @@ -27,6 +28,7 @@ import { PromptConstructorNode, GenerateImageNode, GenerateVideoNode, + Generate3DNode, LLMGenerateNode, SplitGridNode, OutputNode, @@ -35,6 +37,9 @@ import { VideoStitchNode, EaseCurveNode, } from "./nodes"; + +// Lazy-load GLBViewerNode to avoid bundling three.js for users who don't use 3D nodes +const GLBViewerNode = dynamic(() => import("./nodes/GLBViewerNode").then(mod => ({ default: mod.GLBViewerNode })), { ssr: false }); import { EditableEdge, ReferenceEdge } from "./edges"; import { ConnectionDropMenu, MenuAction } from "./ConnectionDropMenu"; import { MultiSelectToolbar } from "./MultiSelectToolbar"; @@ -59,6 +64,7 @@ const nodeTypes: NodeTypes = { promptConstructor: PromptConstructorNode, nanoBanana: GenerateImageNode, generateVideo: GenerateVideoNode, + generate3d: Generate3DNode, llmGenerate: LLMGenerateNode, splitGrid: SplitGridNode, output: OutputNode, @@ -66,6 +72,7 @@ const nodeTypes: NodeTypes = { imageCompare: ImageCompareNode, videoStitch: VideoStitchNode, easeCurve: EaseCurveNode, + glbViewer: GLBViewerNode, }; const edgeTypes: EdgeTypes = { @@ -79,10 +86,12 @@ const edgeTypes: EdgeTypes = { // - Video handles can only connect to generateVideo or output nodes // Helper to determine handle type from handle ID // For dynamic handles, we use naming convention: image inputs contain "image", text inputs are "prompt" or "negative_prompt" -const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | "audio" | "easeCurve" | null => { +const getHandleType = (handleId: string | null | undefined): "image" | "text" | "video" | "audio" | "3d" | "easeCurve" | null => { if (!handleId) return null; // EaseCurve handles (must check before other types) if (handleId === "easeCurve") return "easeCurve"; + // 3D handles + if (handleId === "3d") return "3d"; // Standard handles if (handleId === "video") return "video"; if (handleId === "audio" || handleId.startsWith("audio")) return "audio"; @@ -111,6 +120,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] return { inputs: ["image", "text"], outputs: ["image"] }; case "generateVideo": return { inputs: ["image", "text"], outputs: ["video"] }; + case "generate3d": + return { inputs: ["image", "text"], outputs: ["3d"] }; case "llmGenerate": return { inputs: ["text", "image"], outputs: ["text"] }; case "splitGrid": @@ -125,6 +136,8 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] return { inputs: ["video", "audio"], outputs: ["video"] }; case "easeCurve": return { inputs: ["video", "easeCurve"], outputs: ["video", "easeCurve"] }; + case "glbViewer": + return { inputs: ["3d"], outputs: ["image"] }; default: return { inputs: [], outputs: [] }; } @@ -133,7 +146,7 @@ const getNodeHandles = (nodeType: string): { inputs: string[]; outputs: string[] interface ConnectionDropState { position: { x: number; y: number }; flowPosition: { x: number; y: number }; - handleType: "image" | "text" | "video" | "audio" | "easeCurve" | null; + handleType: "image" | "text" | "video" | "audio" | "3d" | "easeCurve" | null; connectionType: "source" | "target"; sourceNodeId: string | null; sourceHandleId: string | null; @@ -210,7 +223,7 @@ const findScrollableAncestor = (target: HTMLElement, deltaX: number, deltaY: num }; export function WorkflowCanvas() { - const { nodes, edges, groups, onNodesChange, onEdgesChange, onConnect, addNode, updateNodeData, loadWorkflow, getNodeById, addToGlobalHistory, setNodeGroupId, executeWorkflow, isModalOpen, showQuickstart, setShowQuickstart, navigationTarget, setNavigationTarget, captureSnapshot, applyEditOperations, setWorkflowMetadata } = + const { nodes, edges, groups, onNodesChange, onEdgesChange, onConnect, addNode, updateNodeData, loadWorkflow, getNodeById, addToGlobalHistory, setNodeGroupId, executeWorkflow, isModalOpen, showQuickstart, setShowQuickstart, navigationTarget, setNavigationTarget, captureSnapshot, applyEditOperations, setWorkflowMetadata, canvasNavigationSettings, setShortcutsDialogOpen } = useWorkflowStore(); const { screenToFlowPosition, getViewport, zoomIn, zoomOut, setViewport, setCenter } = useReactFlow(); const { show: showToast } = useToast(); @@ -324,6 +337,11 @@ export function WorkflowCanvas() { return false; } + // 3D connections: 3d handles can only connect to matching 3d handles + if (sourceType === "3d" || targetType === "3d") { + return sourceType === "3d" && targetType === "3d"; + } + // Audio connections: audio handles can only connect to audio handles if (sourceType === "audio" || targetType === "audio") { return sourceType === "audio" && targetType === "audio"; @@ -448,7 +466,7 @@ export function WorkflowCanvas() { // Helper to find a compatible handle on a node by type const findCompatibleHandle = ( node: Node, - handleType: "image" | "text" | "video" | "audio" | "easeCurve", + handleType: "image" | "text" | "video" | "audio" | "3d" | "easeCurve", needInput: boolean, batchUsed?: Set ): string | null => { @@ -474,8 +492,9 @@ export function WorkflowCanvas() { return null; } } - // Output handle - check for video or image type + // Output handle - check for video, 3d, or image type if (handleType === "video") return "video"; + if (handleType === "3d") return "3d"; return handleType === "image" ? "image" : null; } @@ -824,6 +843,12 @@ export function WorkflowCanvas() { // VideoStitch accepts audio targetHandleId = "audio"; } + } else if (handleType === "3d") { + if (nodeType === "glbViewer") { + targetHandleId = "3d"; + } else if (nodeType === "nanoBanana") { + sourceHandleIdForNewNode = "3d"; + } } else if (handleType === "easeCurve") { if (nodeType === "easeCurve") { targetHandleId = "easeCurve"; @@ -932,8 +957,16 @@ export function WorkflowCanvas() { const scrollableElement = findScrollableAncestor(target, event.deltaX, event.deltaY); if (scrollableElement) return; - // Pinch gesture (ctrlKey) always zooms - if (event.ctrlKey) { + const { zoomMode } = canvasNavigationSettings; + + // Check if zoom should be triggered based on settings + const shouldZoom = + zoomMode === "scroll" || + (zoomMode === "altScroll" && event.altKey) || + (zoomMode === "ctrlScroll" && (event.ctrlKey || event.metaKey)); + + // Pinch gesture (ctrlKey + trackpad) always zooms regardless of settings + if (event.ctrlKey && !event.altKey) { event.preventDefault(); if (event.deltaY < 0) zoomIn(); else zoomOut(); @@ -943,34 +976,46 @@ export function WorkflowCanvas() { // On macOS, differentiate trackpad from mouse if (isMacOS) { if (isMouseWheel(event)) { - // Mouse wheel → zoom - event.preventDefault(); - if (event.deltaY < 0) zoomIn(); - else zoomOut(); + // Mouse wheel → zoom if settings allow + if (shouldZoom) { + event.preventDefault(); + if (event.deltaY < 0) zoomIn(); + else zoomOut(); + } } else { - // Trackpad scroll → pan (also prevent horizontal swipe navigation) - event.preventDefault(); - const viewport = getViewport(); - setViewport({ - x: viewport.x - event.deltaX, - y: viewport.y - event.deltaY, - zoom: viewport.zoom, - }); + // Trackpad scroll + if (shouldZoom) { + // Zoom + event.preventDefault(); + if (event.deltaY < 0) zoomIn(); + else zoomOut(); + } else { + // Pan (also prevent horizontal swipe navigation) + event.preventDefault(); + const viewport = getViewport(); + setViewport({ + x: viewport.x - event.deltaX, + y: viewport.y - event.deltaY, + zoom: viewport.zoom, + }); + } } return; } - // Non-macOS: default zoom behavior - event.preventDefault(); - if (event.deltaY < 0) zoomIn(); - else zoomOut(); + // Non-macOS + if (shouldZoom) { + event.preventDefault(); + if (event.deltaY < 0) zoomIn(); + else zoomOut(); + } }; wrapper.addEventListener('wheel', handleWheelNonPassive, { passive: false }); return () => { wrapper.removeEventListener('wheel', handleWheelNonPassive); }; - }, [isModalOpen, zoomIn, zoomOut, getViewport, setViewport]); + }, [isModalOpen, zoomIn, zoomOut, getViewport, setViewport, canvasNavigationSettings]); // Keyboard shortcuts for copy/paste and stacking selected nodes const handleKeyDown = useCallback((event: KeyboardEvent) => { @@ -982,6 +1027,13 @@ export function WorkflowCanvas() { return; } + // Handle keyboard shortcuts dialog (? key) + if (event.key === "?" && !event.ctrlKey && !event.metaKey) { + event.preventDefault(); + setShortcutsDialogOpen(true); + return; + } + // Handle workflow execution (Ctrl/Cmd + Enter) if ((event.ctrlKey || event.metaKey) && event.key === "Enter") { event.preventDefault(); @@ -1042,6 +1094,7 @@ export function WorkflowCanvas() { promptConstructor: { width: 340, height: 280 }, nanoBanana: { width: 300, height: 300 }, generateVideo: { width: 300, height: 300 }, + generate3d: { width: 300, height: 300 }, llmGenerate: { width: 320, height: 360 }, splitGrid: { width: 300, height: 320 }, output: { width: 320, height: 320 }, @@ -1049,6 +1102,7 @@ export function WorkflowCanvas() { imageCompare: { width: 400, height: 360 }, videoStitch: { width: 400, height: 280 }, easeCurve: { width: 340, height: 480 }, + glbViewer: { width: 360, height: 380 }, }; const dims = defaultDimensions[nodeType]; addNode(nodeType, { x: centerX - dims.width / 2, y: centerY - dims.height / 2 }); @@ -1089,6 +1143,7 @@ export function WorkflowCanvas() { // Update the selected imageInput node with the pasted image updateNodeData(selectedImageInputNode.id, { image: dataUrl, + imageRef: undefined, filename: `pasted-${Date.now()}.png`, dimensions: { width: img.width, height: img.height }, }); @@ -1225,7 +1280,7 @@ export function WorkflowCanvas() { ]); }); } - }, [nodes, onNodesChange, copySelectedNodes, pasteNodes, clearClipboard, clipboard, getViewport, addNode, updateNodeData, executeWorkflow]); + }, [nodes, onNodesChange, copySelectedNodes, pasteNodes, clearClipboard, clipboard, getViewport, addNode, updateNodeData, executeWorkflow, setShortcutsDialogOpen]); useEffect(() => { window.addEventListener("keydown", handleKeyDown); @@ -1546,8 +1601,28 @@ export function WorkflowCanvas() { fitView deleteKeyCode={["Backspace", "Delete"]} multiSelectionKeyCode="Shift" - selectionOnDrag={isMacOS && !isModalOpen} - panOnDrag={!isMacOS && !isModalOpen} + selectionOnDrag={ + canvasNavigationSettings.selectionMode === "altDrag" || canvasNavigationSettings.selectionMode === "shiftDrag" + ? false + : canvasNavigationSettings.panMode === "always" + ? false + : isMacOS && !isModalOpen + } + selectionKeyCode={ + isModalOpen ? null + : canvasNavigationSettings.selectionMode === "altDrag" ? "Alt" + : canvasNavigationSettings.selectionMode === "shiftDrag" ? "Shift" + : "Shift" + } + panOnDrag={ + isModalOpen + ? false + : canvasNavigationSettings.panMode === "always" + ? true + : canvasNavigationSettings.panMode === "middleMouse" + ? [2] + : !isMacOS + } selectNodesOnDrag={false} nodeDragThreshold={5} zoomOnScroll={false} @@ -1555,7 +1630,13 @@ export function WorkflowCanvas() { minZoom={0.1} maxZoom={4} defaultViewport={{ x: 0, y: 0, zoom: 1 }} - panActivationKeyCode={isModalOpen ? null : "Space"} + panActivationKeyCode={ + isModalOpen + ? null + : canvasNavigationSettings.panMode === "space" + ? "Space" + : null + } nodesDraggable={!isModalOpen} nodesConnectable={!isModalOpen} elementsSelectable={!isModalOpen} @@ -1589,6 +1670,8 @@ export function WorkflowCanvas() { return "#22c55e"; case "generateVideo": return "#9333ea"; + case "generate3d": + return "#fb923c"; case "llmGenerate": return "#06b6d4"; case "splitGrid": @@ -1603,6 +1686,8 @@ export function WorkflowCanvas() { return "#f97316"; case "easeCurve": return "#bef264"; // lime-300 (easy-peasy-ease) + case "glbViewer": + return "#38bdf8"; // sky-400 (3D viewport) default: return "#94a3b8"; } diff --git a/src/components/__tests__/AnnotationNode.test.tsx b/src/components/__tests__/AnnotationNode.test.tsx index e8061bc2..bed5fc1c 100644 --- a/src/components/__tests__/AnnotationNode.test.tsx +++ b/src/components/__tests__/AnnotationNode.test.tsx @@ -347,7 +347,9 @@ describe("AnnotationNode", () => { expect(mockUpdateNodeData).toHaveBeenCalledWith("test-annotation-1", { sourceImage: null, + sourceImageRef: undefined, outputImage: null, + outputImageRef: undefined, annotations: [], }); }); @@ -412,7 +414,9 @@ describe("AnnotationNode", () => { await waitFor(() => { expect(mockUpdateNodeData).toHaveBeenCalledWith("test-annotation-1", { sourceImage: "data:image/png;base64,uploadedImage", + sourceImageRef: undefined, outputImage: null, + outputImageRef: undefined, annotations: [], }); }); diff --git a/src/components/__tests__/ConnectionDropMenu.test.tsx b/src/components/__tests__/ConnectionDropMenu.test.tsx index d42267c3..9d47cc39 100644 --- a/src/components/__tests__/ConnectionDropMenu.test.tsx +++ b/src/components/__tests__/ConnectionDropMenu.test.tsx @@ -79,6 +79,16 @@ describe("ConnectionDropMenu", () => { expect(screen.queryByText("Annotate")).not.toBeInTheDocument(); expect(screen.queryByText("LLM Generate")).not.toBeInTheDocument(); }); + + it("should show 3D-accepting nodes when dragging from 3D output", () => { + render(); + + expect(screen.getByText("3D Viewer")).toBeInTheDocument(); + // Should NOT show image/text/video nodes + expect(screen.queryByText("Annotate")).not.toBeInTheDocument(); + expect(screen.queryByText("Generate Image")).not.toBeInTheDocument(); + expect(screen.queryByText("Generate Video")).not.toBeInTheDocument(); + }); }); describe("Node Type Filtering - Target Connection (from input handle)", () => { @@ -110,6 +120,15 @@ describe("ConnectionDropMenu", () => { expect(screen.queryByText("Image Input")).not.toBeInTheDocument(); expect(screen.queryByText("Prompt")).not.toBeInTheDocument(); }); + + it("should show 3D-producing nodes when dragging from 3D input", () => { + render(); + + expect(screen.getByText("Generate 3D")).toBeInTheDocument(); + // Should NOT show other nodes + expect(screen.queryByText("Image Input")).not.toBeInTheDocument(); + expect(screen.queryByText("Generate Image")).not.toBeInTheDocument(); + }); }); describe("Menu Item Click", () => { diff --git a/src/components/__tests__/CostIndicator.test.tsx b/src/components/__tests__/CostIndicator.test.tsx index c1fc165c..84350c43 100644 --- a/src/components/__tests__/CostIndicator.test.tsx +++ b/src/components/__tests__/CostIndicator.test.tsx @@ -441,4 +441,170 @@ describe("CostIndicator", () => { expect(screen.queryByTitle("View cost details")).not.toBeInTheDocument(); }); }); + + describe("Non-Gemini Provider Hiding", () => { + it("should not render when a nanoBanana node has a non-Gemini selectedModel", () => { + const nodes: WorkflowNode[] = [ + { + id: "node-1", + type: "nanoBanana", + position: { x: 0, y: 0 }, + data: { + model: "nano-banana", + resolution: "1K", + selectedModel: { + provider: "fal", + modelId: "fal-ai/flux", + displayName: "Flux", + }, + }, + }, + ]; + + mockUseWorkflowStore.mockImplementation((selector) => { + return selector(createDefaultState({ nodes })); + }); + + render(); + + expect(screen.queryByTitle("View cost details")).not.toBeInTheDocument(); + }); + + it("should not render when a generateVideo node has a non-Gemini selectedModel", () => { + const nodes: WorkflowNode[] = [ + { + id: "node-1", + type: "generateVideo", + position: { x: 0, y: 0 }, + data: { + selectedModel: { + provider: "kie", + modelId: "kling-video", + displayName: "Kling Video", + }, + status: "idle", + }, + }, + ]; + + mockUseWorkflowStore.mockImplementation((selector) => { + return selector(createDefaultState({ nodes })); + }); + + render(); + + expect(screen.queryByTitle("View cost details")).not.toBeInTheDocument(); + }); + + it("should not render when a generate3d node has a non-Gemini selectedModel", () => { + const nodes: WorkflowNode[] = [ + { + id: "node-1", + type: "generate3d", + position: { x: 0, y: 0 }, + data: { + selectedModel: { + provider: "fal", + modelId: "fal-3d", + displayName: "Fal 3D", + }, + status: "idle", + }, + }, + ]; + + mockUseWorkflowStore.mockImplementation((selector) => { + return selector(createDefaultState({ nodes })); + }); + + render(); + + expect(screen.queryByTitle("View cost details")).not.toBeInTheDocument(); + }); + + it("should render when a nanoBanana node has selectedModel with provider gemini", () => { + const nodes: WorkflowNode[] = [ + { + id: "node-1", + type: "nanoBanana", + position: { x: 0, y: 0 }, + data: { + model: "nano-banana", + resolution: "1K", + selectedModel: { + provider: "gemini", + modelId: "nano-banana", + displayName: "Nano Banana", + }, + }, + }, + ]; + + mockUseWorkflowStore.mockImplementation((selector) => { + return selector(createDefaultState({ nodes })); + }); + + render(); + + expect(screen.getByTitle("View cost details")).toBeInTheDocument(); + }); + + it("should render when a nanoBanana node has no selectedModel (legacy Gemini)", () => { + const nodes: WorkflowNode[] = [ + { + id: "node-1", + type: "nanoBanana", + position: { x: 0, y: 0 }, + data: { + model: "nano-banana", + resolution: "1K", + }, + }, + ]; + + mockUseWorkflowStore.mockImplementation((selector) => { + return selector(createDefaultState({ nodes })); + }); + + render(); + + expect(screen.getByTitle("View cost details")).toBeInTheDocument(); + }); + + it("should not render when mix of Gemini and non-Gemini nodes exist", () => { + const nodes: WorkflowNode[] = [ + { + id: "node-1", + type: "nanoBanana", + position: { x: 0, y: 0 }, + data: { + model: "nano-banana", + resolution: "1K", + }, + }, + { + id: "node-2", + type: "nanoBanana", + position: { x: 100, y: 0 }, + data: { + model: "nano-banana", + resolution: "1K", + selectedModel: { + provider: "replicate", + modelId: "some-model", + displayName: "Some Model", + }, + }, + }, + ]; + + mockUseWorkflowStore.mockImplementation((selector) => { + return selector(createDefaultState({ nodes })); + }); + + render(); + + expect(screen.queryByTitle("View cost details")).not.toBeInTheDocument(); + }); + }); }); diff --git a/src/components/__tests__/Generate3DNode.test.tsx b/src/components/__tests__/Generate3DNode.test.tsx new file mode 100644 index 00000000..6d587973 --- /dev/null +++ b/src/components/__tests__/Generate3DNode.test.tsx @@ -0,0 +1,234 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { Generate3DNode } from "@/components/nodes/Generate3DNode"; +import { ReactFlowProvider } from "@xyflow/react"; +import { Generate3DNodeData } from "@/types"; + +// Mock deduplicatedFetch +vi.mock("@/utils/deduplicatedFetch", () => ({ + deduplicatedFetch: (...args: Parameters) => fetch(...args), + clearFetchCache: vi.fn(), +})); + +// Mock the workflow store +const mockUpdateNodeData = vi.fn(); +const mockRegenerateNode = vi.fn(); +const mockUseWorkflowStore = vi.fn(); + +vi.mock("@/store/workflowStore", () => ({ + useWorkflowStore: (selector?: (state: unknown) => unknown) => { + if (selector) { + return mockUseWorkflowStore(selector); + } + return mockUseWorkflowStore((s: unknown) => s); + }, + useProviderApiKeys: () => ({ + replicateApiKey: null, + falApiKey: null, + kieApiKey: null, + wavespeedApiKey: null, + replicateEnabled: false, + kieEnabled: false, + }), +})); + +// Mock useReactFlow +const mockSetNodes = vi.fn(); + +vi.mock("@xyflow/react", async () => { + const actual = await vi.importActual("@xyflow/react"); + return { + ...actual, + useReactFlow: () => ({ + getNodes: vi.fn(() => []), + setNodes: mockSetNodes, + screenToFlowPosition: vi.fn((pos) => pos), + }), + }; +}); + +// Mock Toast +vi.mock("@/components/Toast", () => ({ + useToast: { + getState: () => ({ + show: vi.fn(), + }), + }, +})); + +// Mock createPortal +vi.mock("react-dom", async () => { + const actual = await vi.importActual("react-dom"); + return { + ...actual, + createPortal: (node: React.ReactNode) => node, + }; +}); + +// Mock fetch +const mockFetch = vi.fn(); +global.fetch = mockFetch; + +function TestWrapper({ children }: { children: React.ReactNode }) { + return {children}; +} + +describe("Generate3DNode", () => { + beforeEach(() => { + vi.clearAllMocks(); + mockFetch.mockResolvedValue({ + ok: true, + json: () => Promise.resolve({ models: [], success: true }), + }); + + mockUseWorkflowStore.mockImplementation((selector) => { + const state = { + updateNodeData: mockUpdateNodeData, + regenerateNode: mockRegenerateNode, + isRunning: false, + currentNodeIds: [], + groups: {}, + nodes: [], + recentModels: [], + trackModelUsage: vi.fn(), + incrementModalCount: vi.fn(), + decrementModalCount: vi.fn(), + getNodesWithComments: vi.fn(() => []), + markCommentViewed: vi.fn(), + setNavigationTarget: vi.fn(), + generationsPath: null, + }; + return selector(state); + }); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + const createNodeData = (overrides: Partial = {}): Generate3DNodeData => ({ + inputImages: [], + inputPrompt: null, + output3dUrl: null, + status: "idle", + error: null, + ...overrides, + }); + + const createNodeProps = (data: Partial = {}) => ({ + id: "test-3d-node", + type: "generate3d" as const, + data: createNodeData(data), + selected: false, + }); + + describe("Basic Rendering", () => { + it("should render with default title when no model selected", () => { + render( + + + + ); + + expect(screen.getByText("Select 3D model...")).toBeInTheDocument(); + }); + + it("should render model name when selected", () => { + render( + + + + ); + + expect(screen.getByText("TripoSR")).toBeInTheDocument(); + }); + + it("should render Browse button", () => { + render( + + + + ); + + expect(screen.getByText("Browse")).toBeInTheDocument(); + }); + + it("should render 'Run to generate' when idle", () => { + render( + + + + ); + + expect(screen.getByText("Run to generate")).toBeInTheDocument(); + }); + }); + + describe("Handles", () => { + it("should render image and text input handles", () => { + const { container } = render( + + + + ); + + const imageHandle = container.querySelector('[data-handletype="image"]'); + const textHandle = container.querySelector('[data-handletype="text"]'); + expect(imageHandle).toBeInTheDocument(); + expect(textHandle).toBeInTheDocument(); + }); + + it("should render 3D output handle", () => { + const { container } = render( + + + + ); + + const outputHandle = container.querySelector('[data-handletype="3d"]'); + expect(outputHandle).toBeInTheDocument(); + }); + + it("should render output label as '3D'", () => { + render( + + + + ); + + expect(screen.getByText("3D")).toBeInTheDocument(); + }); + }); + + describe("Output States", () => { + it("should show 3D model generated indicator when output exists", () => { + render( + + + + ); + + expect(screen.getByText("3D Model Generated")).toBeInTheDocument(); + expect(screen.getByText("Connect to 3D Viewer")).toBeInTheDocument(); + }); + + it("should show error message when status is error", () => { + render( + + + + ); + + expect(screen.getByText("Model not found")).toBeInTheDocument(); + }); + }); +}); diff --git a/src/components/__tests__/ImageInputNode.test.tsx b/src/components/__tests__/ImageInputNode.test.tsx index 07e12a1f..4cbb673c 100644 --- a/src/components/__tests__/ImageInputNode.test.tsx +++ b/src/components/__tests__/ImageInputNode.test.tsx @@ -234,6 +234,7 @@ describe("ImageInputNode", () => { await waitFor(() => { expect(mockUpdateNodeData).toHaveBeenCalledWith("test-image-1", { image: "data:image/png;base64,test123", + imageRef: undefined, filename: "test.png", dimensions: { width: 1024, height: 768 }, }); @@ -406,6 +407,7 @@ describe("ImageInputNode", () => { expect(mockUpdateNodeData).toHaveBeenCalledWith("test-image-1", { image: null, + imageRef: undefined, filename: null, dimensions: null, }); diff --git a/src/components/__tests__/ModelSearchDialog.test.tsx b/src/components/__tests__/ModelSearchDialog.test.tsx index f30a7f02..7fbaf9b6 100644 --- a/src/components/__tests__/ModelSearchDialog.test.tsx +++ b/src/components/__tests__/ModelSearchDialog.test.tsx @@ -104,6 +104,14 @@ const sampleModels: ProviderModel[] = [ capabilities: ["text-to-video", "image-to-video"], coverImage: "https://example.com/kling.jpg", }, + { + id: "fal-ai/triposr", + name: "TripoSR", + description: "3D model generation from images", + provider: "fal", + capabilities: ["image-to-3d"], + coverImage: "https://example.com/triposr.jpg", + }, ]; describe("ModelSearchDialog", () => { @@ -412,7 +420,7 @@ describe("ModelSearchDialog", () => { ); await waitFor(() => { - expect(screen.getByText(/3 models? found/)).toBeInTheDocument(); + expect(screen.getByText(/4 models? found/)).toBeInTheDocument(); }); }); }); @@ -475,6 +483,7 @@ describe("ModelSearchDialog", () => { provider: "replicate", modelId: "stability-ai/sdxl", displayName: "SDXL", + capabilities: ["text-to-image"], }, }) ); @@ -506,6 +515,38 @@ describe("ModelSearchDialog", () => { provider: "fal", modelId: "kling-video/v1.6/pro", displayName: "Kling Video Pro", + capabilities: ["text-to-video", "image-to-video"], + }, + }) + ); + }); + + it("should create generate3d node for 3D models", async () => { + const onClose = vi.fn(); + + render( + + + + ); + + await waitFor(() => { + expect(screen.getByText("TripoSR")).toBeInTheDocument(); + }); + + // Click on the TripoSR 3D model card + const modelCard = screen.getByText("TripoSR").closest("button"); + fireEvent.click(modelCard!); + + expect(mockAddNode).toHaveBeenCalledWith( + "generate3d", + expect.any(Object), + expect.objectContaining({ + selectedModel: { + provider: "fal", + modelId: "fal-ai/triposr", + displayName: "TripoSR", + capabilities: ["image-to-3d"], }, }) ); diff --git a/src/components/__tests__/SplitGridSettingsModal.test.tsx b/src/components/__tests__/SplitGridSettingsModal.test.tsx index e3aec33d..83977d84 100644 --- a/src/components/__tests__/SplitGridSettingsModal.test.tsx +++ b/src/components/__tests__/SplitGridSettingsModal.test.tsx @@ -68,8 +68,8 @@ describe("SplitGridSettingsModal", () => { }); }); - describe("Number of Images Selection", () => { - it("should render target count options (4, 6, 8, 9, 10)", () => { + describe("Grid Layout Selection", () => { + it("should render all layout options", () => { render( { /> ); - expect(screen.getByText("4")).toBeInTheDocument(); - expect(screen.getByText("6")).toBeInTheDocument(); - expect(screen.getByText("8")).toBeInTheDocument(); - expect(screen.getByText("9")).toBeInTheDocument(); - expect(screen.getByText("10")).toBeInTheDocument(); + expect(screen.getByText("2x2")).toBeInTheDocument(); + expect(screen.getByText("1x5")).toBeInTheDocument(); + expect(screen.getByText("2x3")).toBeInTheDocument(); + expect(screen.getByText("3x2")).toBeInTheDocument(); + expect(screen.getByText("2x4")).toBeInTheDocument(); + expect(screen.getByText("3x3")).toBeInTheDocument(); + expect(screen.getByText("2x5")).toBeInTheDocument(); }); - it("should highlight selected target count", () => { + it("should highlight selected layout", () => { render( { /> ); - // Default is 6, check its button has the selected styling + // Default is 2x3, check its button has the selected styling const buttons = screen.getAllByRole("button"); - const sixButton = buttons.find(btn => btn.textContent?.includes("6")); - expect(sixButton).toHaveClass("border-blue-500"); + const selectedButton = buttons.find(btn => btn.textContent?.includes("2x3")); + expect(selectedButton).toHaveClass("border-blue-500"); }); - it("should update target count when option is clicked", () => { + it("should update layout when option is clicked", () => { render( { /> ); - // Click on 9 + // Click on 3x3 const buttons = screen.getAllByRole("button"); - const nineButton = buttons.find(btn => btn.textContent?.includes("9")); - fireEvent.click(nineButton!); + const threeByThreeButton = buttons.find(btn => btn.textContent?.includes("3x3")); + fireEvent.click(threeByThreeButton!); // The grid description should update expect(screen.getByText(/3x3 = 9 images/)).toBeInTheDocument(); @@ -127,9 +129,27 @@ describe("SplitGridSettingsModal", () => { /> ); - // Default is 6, which is 2x3 + // Default is 2x3 expect(screen.getByText(/2x3 = 6 images/)).toBeInTheDocument(); }); + + it("should allow selecting 3x2 layout (6 images, portrait orientation)", () => { + render( + + ); + + // Click on 3x2 + const buttons = screen.getAllByRole("button"); + const threeByTwoButton = buttons.find(btn => btn.textContent?.includes("3x2")); + fireEvent.click(threeByTwoButton!); + + // Should show 3x2 = 6 images + expect(screen.getByText(/3x2 = 6 images/)).toBeInTheDocument(); + }); }); describe("Default Prompt", () => { @@ -344,7 +364,7 @@ describe("SplitGridSettingsModal", () => { expect(screen.getByText("Create 6 Generate Sets")).toBeInTheDocument(); }); - it("should update Create button text when target count changes", () => { + it("should update Create button text when layout changes", () => { render( { /> ); - // Click on 9 + // Click on 3x3 const buttons = screen.getAllByRole("button"); - const nineButton = buttons.find(btn => btn.textContent?.includes("9") && !btn.textContent?.includes("Create")); - fireEvent.click(nineButton!); + const threeByThreeButton = buttons.find(btn => btn.textContent?.includes("3x3") && !btn.textContent?.includes("Create")); + fireEvent.click(threeByThreeButton!); expect(screen.getByText("Create 9 Generate Sets")).toBeInTheDocument(); }); @@ -431,9 +451,9 @@ describe("SplitGridSettingsModal", () => { /> ); - // Each target count button should have a grid preview + // Each layout button should have a grid preview const gridPreviews = container.querySelectorAll(".aspect-video"); - expect(gridPreviews.length).toBe(6); // 6 options: 4, 5, 6, 8, 9, 10 + expect(gridPreviews.length).toBe(7); // 7 layout options }); }); @@ -441,6 +461,8 @@ describe("SplitGridSettingsModal", () => { it("should use node data values as initial state", () => { const nodeData = createDefaultNodeData(); nodeData.targetCount = 9; + nodeData.gridRows = 3; + nodeData.gridCols = 3; nodeData.defaultPrompt = "Existing prompt"; nodeData.generateSettings = { aspectRatio: "16:9", diff --git a/src/components/__tests__/WorkflowCanvas.test.tsx b/src/components/__tests__/WorkflowCanvas.test.tsx index 9ced4f4b..f62d1ff9 100644 --- a/src/components/__tests__/WorkflowCanvas.test.tsx +++ b/src/components/__tests__/WorkflowCanvas.test.tsx @@ -138,6 +138,10 @@ const createDefaultState = (overrides = {}) => ({ setNavigationTarget: vi.fn(), getNodesWithComments: vi.fn(() => []), markCommentViewed: vi.fn(), + canvasNavigationSettings: { panMode: "space", zoomMode: "altScroll", selectionMode: "click" }, + captureSnapshot: vi.fn(), + applyEditOperations: vi.fn(() => ({ applied: 0, skipped: [] })), + setWorkflowMetadata: vi.fn(), ...overrides, }); diff --git a/src/components/modals/ModelSearchDialog.tsx b/src/components/modals/ModelSearchDialog.tsx index f6ef7246..89307462 100644 --- a/src/components/modals/ModelSearchDialog.tsx +++ b/src/components/modals/ModelSearchDialog.tsx @@ -89,7 +89,7 @@ function getPaneCenter() { } // Capability filter options -type CapabilityFilter = "all" | "image" | "video"; +type CapabilityFilter = "all" | "image" | "video" | "3d"; // API response type interface ModelsResponse { @@ -200,7 +200,9 @@ export function ModelSearchDialog({ const capabilities = capabilityFilter === "image" ? "text-to-image,image-to-image" - : "text-to-video,image-to-video"; + : capabilityFilter === "video" + ? "text-to-video,image-to-video" + : "text-to-3d,image-to-3d"; params.set("capabilities", capabilities); } if (bypassCache) { @@ -315,14 +317,18 @@ export function ModelSearchDialog({ const isVideoModel = model.capabilities.some( (cap) => cap === "text-to-video" || cap === "image-to-video" ); + const is3DModel = model.capabilities.some( + (cap) => cap === "text-to-3d" || cap === "image-to-3d" + ); - const nodeType = isVideoModel ? "generateVideo" : "nanoBanana"; + const nodeType = isVideoModel ? "generateVideo" : is3DModel ? "generate3d" : "nanoBanana"; addNode(nodeType, position, { selectedModel: { provider: model.provider, modelId: model.id, displayName: model.name, + capabilities: model.capabilities, }, }); @@ -413,9 +419,13 @@ export function ModelSearchDialog({ const isVideo = matchingModel.capabilities.some( (cap) => cap === "text-to-video" || cap === "image-to-video" ); + const is3D = matchingModel.capabilities.some( + (cap) => cap === "text-to-3d" || cap === "image-to-3d" + ); if (capabilityFilter === "image") return isImage; if (capabilityFilter === "video") return isVideo; + if (capabilityFilter === "3d") return is3D; return true; }) .slice(0, 4); // Show max 4 @@ -476,6 +486,14 @@ export function ModelSearchDialog({ color = "bg-pink-500/20 text-pink-300"; label = "img→vid"; break; + case "text-to-3d": + color = "bg-orange-500/20 text-orange-300"; + label = "txt→3d"; + break; + case "image-to-3d": + color = "bg-amber-500/20 text-amber-300"; + label = "img→3d"; + break; } if (label) { @@ -635,6 +653,7 @@ export function ModelSearchDialog({ + {/* Refresh Cache */} diff --git a/src/components/nodes/AnnotationNode.tsx b/src/components/nodes/AnnotationNode.tsx index 0bcc0891..9a35b075 100644 --- a/src/components/nodes/AnnotationNode.tsx +++ b/src/components/nodes/AnnotationNode.tsx @@ -37,7 +37,9 @@ export function AnnotationNode({ id, data, selected }: NodeProps { updateNodeData(id, { sourceImage: null, + sourceImageRef: undefined, outputImage: null, + outputImageRef: undefined, annotations: [], }); }, [id, updateNodeData]); diff --git a/src/components/nodes/BaseNode.tsx b/src/components/nodes/BaseNode.tsx index 467eb8d7..61bfcdf8 100644 --- a/src/components/nodes/BaseNode.tsx +++ b/src/components/nodes/BaseNode.tsx @@ -26,6 +26,7 @@ interface BaseNodeProps { isExecuting?: boolean; hasError?: boolean; className?: string; + contentClassName?: string; minWidth?: number; minHeight?: number; headerAction?: ReactNode; @@ -48,6 +49,7 @@ export function BaseNode({ isExecuting = false, hasError = false, className = "", + contentClassName, minWidth = 180, minHeight = 100, headerAction, @@ -251,14 +253,14 @@ export function BaseNode({ />
-
+
{/* Title Section */}
{titlePrefix} @@ -452,7 +454,7 @@ export function BaseNode({
)}
-
{children}
+
{children}
); diff --git a/src/components/nodes/GLBViewerNode.tsx b/src/components/nodes/GLBViewerNode.tsx new file mode 100644 index 00000000..696e547c --- /dev/null +++ b/src/components/nodes/GLBViewerNode.tsx @@ -0,0 +1,550 @@ +"use client"; + +import { useCallback, useRef, useState, useEffect, Suspense } from "react"; +import { Handle, Position, NodeProps, Node } from "@xyflow/react"; +import { Canvas, useThree, useFrame } from "@react-three/fiber"; +import { OrbitControls } from "@react-three/drei"; +import { BaseNode } from "./BaseNode"; +import { useCommentNavigation } from "@/hooks/useCommentNavigation"; +import { useWorkflowStore } from "@/store/workflowStore"; +import { useToast } from "@/components/Toast"; +import { GLBViewerNodeData } from "@/types"; +import * as THREE from "three"; +import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js"; + +type GLBViewerNodeType = Node; + +/** + * Renders a loaded GLB model using the raw GLTFLoader (avoids drei caching issues with blob URLs). + */ +function Model({ url, onError }: { url: string; onError?: () => void }) { + const groupRef = useRef(null); + const sceneRef = useRef(null); + const [loaded, setLoaded] = useState(false); + const { camera } = useThree(); + + useEffect(() => { + let cancelled = false; + setLoaded(false); + + const loader = new GLTFLoader(); + try { + loader.load( + url, + (gltf) => { + if (cancelled) return; + + const loadedScene = gltf.scene; + + // Compute bounding box and normalize + const box = new THREE.Box3().setFromObject(loadedScene); + const size = box.getSize(new THREE.Vector3()); + const center = box.getCenter(new THREE.Vector3()); + const maxDim = Math.max(size.x, size.y, size.z); + + if (maxDim > 0) { + // Scale to fit in a ~2 unit box + const scale = 2 / maxDim; + loadedScene.scale.setScalar(scale); + + // Re-center the model at origin + loadedScene.position.set( + -center.x * scale, + -center.y * scale, + -center.z * scale + ); + } + + sceneRef.current = loadedScene; + setLoaded(true); + + // Fit camera to model + const dist = 3.5; + camera.position.set(dist, dist * 0.6, dist); + camera.lookAt(0, 0, 0); + }, + undefined, + (error) => { + if (cancelled) return; + console.warn("GLB load failed (file may need re-upload):", error); + onError?.(); + } + ); + } catch { + if (!cancelled) onError?.(); + } + + return () => { + cancelled = true; + // Cleanup previous scene + if (sceneRef.current) { + sceneRef.current.traverse((obj) => { + if (obj instanceof THREE.Mesh) { + try { obj.geometry?.dispose(); } catch (e) { console.warn("GLB geometry dispose failed:", e); } + try { + if (Array.isArray(obj.material)) { + obj.material.forEach((m) => { m.dispose(); }); + } else { + obj.material?.dispose(); + } + } catch (e) { console.warn("GLB material dispose failed:", e); } + } + }); + sceneRef.current = null; + } + }; + }, [url, camera, onError]); + + if (!loaded || !sceneRef.current) return null; + + return ( + + + + ); +} + +/** + * 3D grid floor and subtle environment — hidden during capture via ref. + */ +function SceneEnvironment({ groupRef }: { groupRef: React.RefObject }) { + return ( + + + + ); +} + +/** + * Helper component inside Canvas to expose the gl context for capture. + * Directly hides grid objects (via ref) before rendering the capture frame. + * Uses a ref-based approach so the capture function always has fresh references. + */ +function CaptureHelper({ + captureRef, + envGroupRef, +}: { + captureRef: React.MutableRefObject<(() => string | null) | null>; + envGroupRef: React.RefObject; +}) { + const { gl, scene, camera } = useThree(); + + // Update the capture function every frame so it always has current gl/scene/camera + useFrame(() => { + captureRef.current = () => { + try { + // Directly hide environment objects in the Three.js scene graph (synchronous) + if (envGroupRef.current) { + envGroupRef.current.visible = false; + } + + // Render without the grid + gl.render(scene, camera); + const dataUrl = gl.domElement.toDataURL("image/png"); + + // Restore environment objects + if (envGroupRef.current) { + envGroupRef.current.visible = true; + } + + return dataUrl; + } catch (err) { + console.warn("GLB capture failed:", err); + // Restore environment objects on failure + if (envGroupRef.current) { + envGroupRef.current.visible = true; + } + return null; + } + }; + }); + + return null; +} + +/** + * Auto-rotate the model slowly when not interacting. + */ +function AutoRotate({ enabled }: { enabled: boolean }) { + const { camera } = useThree(); + + useFrame((_, delta) => { + if (!enabled) return; + const angle = delta * 0.3; + const pos = camera.position.clone(); + const cos = Math.cos(angle); + const sin = Math.sin(angle); + camera.position.x = pos.x * cos - pos.z * sin; + camera.position.z = pos.x * sin + pos.z * cos; + camera.lookAt(0, 0, 0); + }); + + return null; +} + +/** + * Loading indicator (wireframe sphere) shown while GLB is parsing. + */ +function LoadingIndicator() { + return ( + + + + + ); +} + +export function GLBViewerNode({ id, data, selected }: NodeProps) { + const nodeData = data as GLBViewerNodeData; + const commentNavigation = useCommentNavigation(id); + const updateNodeData = useWorkflowStore((state) => state.updateNodeData); + const fileInputRef = useRef(null); + const captureRef = useRef<(() => string | null) | null>(null); + const viewportRef = useRef(null); + const [autoRotate, setAutoRotate] = useState(false); + const [isInteracting, setIsInteracting] = useState(false); + const envGroupRef = useRef(null); + + // Auto-resize node when capture image appears/disappears + const prevCapturedRef = useRef(null); + useEffect(() => { + const hadCapture = prevCapturedRef.current != null; + const hasCapture = nodeData.capturedImage != null; + prevCapturedRef.current = nodeData.capturedImage ?? null; + + // Only resize when capture state changes + if (hadCapture === hasCapture) return; + + requestAnimationFrame(() => { + const storeState = useWorkflowStore.getState(); + const thisNode = storeState.nodes.find((n) => n.id === id); + if (!thisNode) return; + + const currentHeight = typeof thisNode.style?.height === "number" + ? thisNode.style.height + : 380; // default node height + + const currentWidth = typeof thisNode.style?.width === "number" + ? thisNode.style.width + : 360; + + // Get the viewport height to size the captured image area proportionally + const viewportH = viewportRef.current?.offsetHeight ?? 200; + // Extra space: controls bar (~30px) + label row (~20px) + captured image (~viewport height) + padding (~20px) + const captureExtraHeight = viewportH + 70; + + const newHeight = hasCapture + ? currentHeight + captureExtraHeight + : Math.max(380, currentHeight - captureExtraHeight); + + useWorkflowStore.setState((state) => ({ + nodes: state.nodes.map((node) => { + if (node.id !== id) return node; + return { + ...node, + style: { ...node.style, width: currentWidth, height: newHeight }, + }; + }), + hasUnsavedChanges: true, + })); + }); + }, [id, nodeData.capturedImage]); + + // Revoke blob URL when it changes or when node is unmounted + const glbUrlRef = useRef(nodeData.glbUrl); + useEffect(() => { + glbUrlRef.current = nodeData.glbUrl; + return () => { + if (glbUrlRef.current) { + URL.revokeObjectURL(glbUrlRef.current); + } + }; + }, [nodeData.glbUrl]); + + // Prevent wheel events from reaching React Flow (stop graph zoom/pan while scrolling to zoom 3D) + useEffect(() => { + const el = viewportRef.current; + if (!el) return; + const stopWheel = (e: WheelEvent) => e.stopPropagation(); + el.addEventListener("wheel", stopWheel, { passive: false }); + return () => el.removeEventListener("wheel", stopWheel); + }, [nodeData.glbUrl]); + + // Shared file processing logic for both click-to-upload and drag-and-drop + const processFile = useCallback( + (file: File) => { + if (!file.name.toLowerCase().endsWith(".glb")) { + useToast.getState().show("Please upload a .GLB file", "warning"); + return; + } + + if (file.size > 100 * 1024 * 1024) { + useToast.getState().show("File too large. Maximum size is 100MB", "warning"); + return; + } + + // Revoke previous URL if exists + if (nodeData.glbUrl) { + URL.revokeObjectURL(nodeData.glbUrl); + } + + const url = URL.createObjectURL(file); + updateNodeData(id, { + glbUrl: url, + filename: file.name, + capturedImage: null, + }); + }, + [id, nodeData.glbUrl, updateNodeData] + ); + + // If the blob URL becomes stale (e.g. after page reload), clear it so user can re-upload + const handleLoadError = useCallback(() => { + updateNodeData(id, { glbUrl: null, filename: null, capturedImage: null }); + }, [id, updateNodeData]); + + const handleFileChange = useCallback( + (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + processFile(file); + }, + [processFile] + ); + + const handleDrop = useCallback( + (e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + + const file = e.dataTransfer.files?.[0]; + if (!file) return; + processFile(file); + }, + [processFile] + ); + + const handleDragOver = useCallback((e: React.DragEvent) => { + e.preventDefault(); + e.stopPropagation(); + }, []); + + const handleRemove = useCallback(() => { + if (nodeData.glbUrl) { + URL.revokeObjectURL(nodeData.glbUrl); + } + updateNodeData(id, { + glbUrl: null, + filename: null, + capturedImage: null, + }); + }, [id, nodeData.glbUrl, updateNodeData]); + + const handleCapture = useCallback(() => { + if (captureRef.current) { + const base64 = captureRef.current(); + if (base64) { + updateNodeData(id, { capturedImage: base64 }); + } else { + useToast.getState().show("Failed to capture 3D view", "error"); + } + } + }, [id, updateNodeData]); + + return ( + updateNodeData(id, { customTitle: title || undefined })} + onCommentChange={(comment) => updateNodeData(id, { comment: comment || undefined })} + selected={selected} + commentNavigation={commentNavigation ?? undefined} + contentClassName={nodeData.glbUrl ? "flex-1 min-h-0 overflow-hidden flex flex-col" : undefined} + > + + + {nodeData.glbUrl ? ( +
+ {/* 3D Viewport — fills node edge-to-edge */} +
setIsInteracting(true)} + onPointerUp={() => setIsInteracting(false)} + > + { + gl.setClearColor(new THREE.Color("#1a1a1a")); + gl.toneMapping = THREE.ACESFilmicToneMapping; + gl.toneMappingExposure = 1.2; + }} + > + {/* Lighting */} + + + + + + {/* Environment grid (hidden during capture via ref) */} + + + {/* Model */} + }> + + + + {/* Controls */} + + + + + + {/* Controls bar — overlaid on viewport */} +
+
+ + {nodeData.filename} + + +
+ +
+ + +
+
+
+ + {/* Captured image preview */} + {nodeData.capturedImage && ( +
+
+ +
+ Captured + + +
+ Captured 3D render +
+ )} +
+ ) : ( +
fileInputRef.current?.click()} + onDrop={handleDrop} + onDragOver={handleDragOver} + className="w-full flex-1 min-h-[150px] border border-dashed border-neutral-600 rounded flex flex-col items-center justify-center cursor-pointer hover:border-neutral-500 hover:bg-neutral-700/50 transition-colors" + > + + + + + Drop .GLB or click + +
+ )} + + {/* 3D input handle - accepts generated 3D models */} + +
+ 3D +
+ + {/* Output handle - image (captured viewport) */} + +
+ Image +
+ + ); +} diff --git a/src/components/nodes/Generate3DNode.tsx b/src/components/nodes/Generate3DNode.tsx new file mode 100644 index 00000000..403877f4 --- /dev/null +++ b/src/components/nodes/Generate3DNode.tsx @@ -0,0 +1,443 @@ +"use client"; + +import React, { useCallback, useState, useEffect, useMemo, useRef } from "react"; +import { Handle, Position, NodeProps, Node, useReactFlow } from "@xyflow/react"; +import { BaseNode } from "./BaseNode"; +import { useCommentNavigation } from "@/hooks/useCommentNavigation"; +import { ModelParameters } from "./ModelParameters"; +import { useWorkflowStore, useProviderApiKeys } from "@/store/workflowStore"; +import { Generate3DNodeData, ProviderType, SelectedModel, ModelInputDef } from "@/types"; +import { ProviderModel, ModelCapability } from "@/lib/providers/types"; +import { ModelSearchDialog } from "@/components/modals/ModelSearchDialog"; +import { useToast } from "@/components/Toast"; + +// Provider badge component +function ProviderBadge({ provider }: { provider: ProviderType }) { + const providerName = provider === "gemini" ? "Gemini" : provider === "replicate" ? "Replicate" : provider === "kie" ? "Kie.ai" : provider === "wavespeed" ? "WaveSpeed" : "fal.ai"; + + return ( + + {provider === "replicate" ? ( + + + + + + ) : provider === "wavespeed" ? ( + + + + + + ) : ( + + + + )} + + ); +} + +// 3D generation capabilities +const THREE_D_CAPABILITIES: ModelCapability[] = ["text-to-3d", "image-to-3d"]; + +type Generate3DNodeType = Node; + +export function Generate3DNode({ id, data, selected }: NodeProps) { + const nodeData = data; + const commentNavigation = useCommentNavigation(id); + const updateNodeData = useWorkflowStore((state) => state.updateNodeData); + const { replicateApiKey, falApiKey, kieApiKey } = useProviderApiKeys(); + const [isBrowseDialogOpen, setIsBrowseDialogOpen] = useState(false); + + // Get the current selected provider (default to fal since most 3D models are there) + const currentProvider: ProviderType = nodeData.selectedModel?.provider || "fal"; + + const handleParametersChange = useCallback( + (parameters: Record) => { + updateNodeData(id, { parameters }); + }, + [id, updateNodeData] + ); + + // Handle inputs loaded from schema + const handleInputsLoaded = useCallback( + (inputs: ModelInputDef[]) => { + updateNodeData(id, { inputSchema: inputs }); + }, + [id, updateNodeData] + ); + + // Handle parameters expand/collapse - resize node height + const { setNodes } = useReactFlow(); + const handleParametersExpandChange = useCallback( + (expanded: boolean, parameterCount: number) => { + const parameterHeight = expanded ? Math.max(parameterCount * 28 + 16, 60) : 0; + const baseHeight = 300; + const newHeight = baseHeight + parameterHeight; + + setNodes((nodes) => + nodes.map((node) => + node.id === id + ? { ...node, style: { ...node.style, height: newHeight } } + : node + ) + ); + }, + [id, setNodes] + ); + + const regenerateNode = useWorkflowStore((state) => state.regenerateNode); + const isRunning = useWorkflowStore((state) => state.isRunning); + + const handleRegenerate = useCallback(() => { + regenerateNode(id); + }, [id, regenerateNode]); + + // Handle model selection from browse dialog + const handleBrowseModelSelect = useCallback((model: ProviderModel) => { + const newSelectedModel: SelectedModel = { + provider: model.provider, + modelId: model.id, + displayName: model.name, + }; + updateNodeData(id, { selectedModel: newSelectedModel, parameters: {} }); + setIsBrowseDialogOpen(false); + }, [id, updateNodeData]); + + // Dynamic title based on selected model + const displayTitle = useMemo(() => { + if (nodeData.selectedModel?.displayName && nodeData.selectedModel.modelId) { + return nodeData.selectedModel.displayName; + } + return "Select 3D model..."; + }, [nodeData.selectedModel?.displayName, nodeData.selectedModel?.modelId]); + + // Provider badge as title prefix + const titlePrefix = useMemo(() => ( + + ), [currentProvider]); + + // Header action element - browse button + const headerAction = useMemo(() => ( + + ), []); + + // Track previous status to detect error transitions + const prevStatusRef = useRef(nodeData.status); + + // Show toast when error occurs + useEffect(() => { + if (nodeData.status === "error" && prevStatusRef.current !== "error" && nodeData.error) { + useToast.getState().show("3D generation failed", "error", true, nodeData.error); + } + prevStatusRef.current = nodeData.status; + }, [nodeData.status, nodeData.error]); + + const handleClear3D = useCallback(() => { + updateNodeData(id, { output3dUrl: null, status: "idle", error: null }); + }, [id, updateNodeData]); + + return ( + <> + updateNodeData(id, { customTitle: title || undefined })} + onCommentChange={(comment) => updateNodeData(id, { comment: comment || undefined })} + onRun={handleRegenerate} + selected={selected} + isExecuting={isRunning} + hasError={nodeData.status === "error"} + headerAction={headerAction} + titlePrefix={titlePrefix} + commentNavigation={commentNavigation ?? undefined} + > + {/* Dynamic input handles based on model schema */} + {nodeData.inputSchema && nodeData.inputSchema.length > 0 ? ( + (() => { + const imageInputs = nodeData.inputSchema!.filter(i => i.type === "image"); + const textInputs = nodeData.inputSchema!.filter(i => i.type === "text"); + + const hasImageInput = imageInputs.length > 0; + const hasTextInput = textInputs.length > 0; + + const handles: Array<{ + id: string; + type: "image" | "text"; + label: string; + schemaName: string | null; + description: string | null; + isPlaceholder: boolean; + }> = []; + + if (hasImageInput) { + imageInputs.forEach((input, index) => { + handles.push({ + id: `image-${index}`, + type: "image", + label: input.label, + schemaName: input.name, + description: input.description || null, + isPlaceholder: false, + }); + }); + } else { + handles.push({ + id: "image", + type: "image", + label: "Image", + schemaName: null, + description: "Not used by this model", + isPlaceholder: true, + }); + } + + if (hasTextInput) { + textInputs.forEach((input, index) => { + handles.push({ + id: `text-${index}`, + type: "text", + label: input.label, + schemaName: input.name, + description: input.description || null, + isPlaceholder: false, + }); + }); + } else { + handles.push({ + id: "text", + type: "text", + label: "Prompt", + schemaName: null, + description: "Not used by this model", + isPlaceholder: true, + }); + } + + const imageHandles = handles.filter(h => h.type === "image"); + const textHandles = handles.filter(h => h.type === "text"); + const totalSlots = imageHandles.length + textHandles.length + 1; + + const renderedHandles = handles.map((handle) => { + const isImage = handle.type === "image"; + const typeIndex = isImage + ? imageHandles.findIndex(h => h.id === handle.id) + : textHandles.findIndex(h => h.id === handle.id); + const adjustedIndex = isImage ? typeIndex : imageHandles.length + 1 + typeIndex; + const topPercent = ((adjustedIndex + 1) / (totalSlots + 1)) * 100; + + return ( + + +
+ {handle.label} +
+ + ); + }); + + return ( + <> + {renderedHandles} + {hasImageInput && ( + + )} + {hasTextInput && ( + + )} + + ); + })() + ) : ( + // Default handles when no schema + <> + +
+ Image +
+ +
+ Prompt +
+ + )} + + {/* 3D output handle */} + + {/* Output label */} +
+ 3D +
+ +
+ {/* Preview area */} + {nodeData.output3dUrl ? ( +
+ + + + 3D Model Generated + Connect to 3D Viewer + {/* Loading overlay for re-generation */} + {nodeData.status === "loading" && ( +
+ + + + +
+ )} + {/* Error overlay */} + {nodeData.status === "error" && ( +
+ + + + Generation failed + See toast for details +
+ )} +
+ +
+
+ ) : ( +
+ {nodeData.status === "loading" ? ( + + + + + ) : nodeData.status === "error" ? ( + + {nodeData.error || "Failed"} + + ) : ( + + Run to generate + + )} +
+ )} + + {/* Model-specific parameters */} + {nodeData.selectedModel?.modelId && ( + + )} +
+ + + {/* Model browser dialog */} + {isBrowseDialogOpen && ( + setIsBrowseDialogOpen(false)} + onModelSelected={handleBrowseModelSelect} + initialCapabilityFilter="3d" + /> + )} + + ); +} diff --git a/src/components/nodes/GenerateImageNode.tsx b/src/components/nodes/GenerateImageNode.tsx index dbf397b9..686afe93 100644 --- a/src/components/nodes/GenerateImageNode.tsx +++ b/src/components/nodes/GenerateImageNode.tsx @@ -210,6 +210,7 @@ export function GenerateImageNode({ id, data, selected }: NodeProps Prompt
- {/* Image output */} + {/* Output handle */}