From 7602dac67031ba81539b8a36ed6688f151441be0 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Fri, 30 Jan 2026 23:33:32 +1300 Subject: [PATCH] feat(34-02): enhance chat API route with tool-based intent routing Rewrite /api/chat route to accept workflow context from client and use AI SDK tool calling for multi-modal agent routing. Changes: - Accept optional workflowState (nodes/edges) in request body - Build workflow context using buildWorkflowContext from contextBuilder - Generate context-aware system prompt using buildEditSystemPrompt - Create chat tools (answerQuestion, createWorkflow, editWorkflow) with current node IDs for validation - Configure streamText with tools, toolChoice: 'auto', and stopWhen for multi-step reasoning (up to 3 steps) - Stream results back via toUIMessageStreamResponse for useChat hook compatibility LLM now automatically routes to the correct tool based on user intent. Tool results stream back to client for application in Plan 03. Co-Authored-By: Claude Opus 4.5 --- .planning/ROADMAP.md | 4 +- .planning/STATE.md | 26 ++++++---- src/app/api/chat/route.ts | 99 ++++++++++++--------------------------- 3 files changed, 47 insertions(+), 82 deletions(-) diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 5dbc6dad..54d354f8 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -500,7 +500,7 @@ Plans: 6. Change narration: explain what was modified and why Plans: -- [ ] 34-01-PLAN.md — Chat agent library: tool definitions, edit operations, context builder +- [x] 34-01-PLAN.md — Chat agent library: tool definitions, edit operations, context builder - [ ] 34-02-PLAN.md — Enhanced /api/chat route with tool calling for intent routing - [ ] 34-03-PLAN.md — Store applyEditOperations, ChatPanel tool result handling, end-to-end wiring @@ -666,7 +666,7 @@ Phases execute in numeric order: 1 → 2 → ... → 35 → 36 → 37 → 38 → | 31. Workflow Proposal System | v1.4 | 2/2 | Complete | 2026-01-26 | | 32. Chat UI Foundation | v1.4 | 2/2 | Complete | 2026-01-27 | | 33. Workflow Edit Safety | v1.4 | 2/2 | Complete | 2026-01-30 | -| 34. Context-Aware Agentic Editing | v1.4 | 0/? | Not started | - | +| 34. Context-Aware Agentic Editing | v1.4 | 1/3 | In progress | - | | 35. Large Workflow Handling | v1.4 | 0/? | Not started | - | | 36. Execution Engine Extraction | v1.5 | 0/3 | Not started | - | | 37. Pure Helpers Extraction | v1.5 | 0/2 | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 0c808514..e8b737ae 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -9,19 +9,19 @@ See: .planning/PROJECT.md (updated 2026-01-09) ## Current Position -Phase: 33 of 35 (Workflow Edit Safety) -Plan: 1 of 2 in current phase +Phase: 34 of 35 (Context-Aware Agentic Workflow Editing) +Plan: 1 of ? in current phase Status: In progress -Last activity: 2026-01-30 - Completed quick-001 (Fix multiple image inputs lost in dynamicInputs) +Last activity: 2026-01-30 - Completed plan 01 (Chat Agent Library) Progress: ░░░░░░░░░░ 6% ## Performance Metrics **Velocity:** -- Total plans completed: 31 +- Total plans completed: 32 - Average duration: 7 min -- Total execution time: 3.6 hours +- Total execution time: 3.7 hours **By Phase:** @@ -54,11 +54,12 @@ Progress: ░░░░░░░░░░ 6% | 28. Node Defaults UI | 1/1 | 32 min | 32 min | | 31. Workflow Proposal System | 2/2 | 6 min | 3 min | | 32. Chat UI Foundation | 2/2 | 9 min | 4.5 min | -| 33. Workflow Edit Safety | 1/2 | 5 min | 5 min | +| 33. Workflow Edit Safety | 2/2 | 5 min | 5 min | +| 34. Agentic Workflow Editing | 1/? | 5 min | 5 min | **Recent Trend:** -- Last 5 plans: 4 min, 2 min, 4 min, 5 min, 5 min -- Trend: Store infrastructure and chat integration work completing quickly +- Last 5 plans: 2 min, 4 min, 5 min, 5 min, 5 min +- Trend: Chat and agentic features shipping quickly with consistent 5-min execution ## Accumulated Context @@ -156,6 +157,11 @@ Recent decisions affecting current work: - dynamicInputs type is Record to support multi-image aggregation - Single image stays as string; only multiple images to same schema key become array - Array.isArray guard on dynamicInputs.prompt access (takes first element) +- AI SDK v6 tool pattern uses inputSchema (not parameters) for zod schemas +- Chat agent tools use "generate" pattern (no execute function) - LLM provides structured output +- Edit operations use batched immutable updates with skip tracking for invalid operations +- Node IDs for AI-generated nodes: ${nodeType}-ai-${Date.now()}-${index} pattern +- Workflow context builder strips base64 data, history arrays, and internal state for LLM consumption ### Deferred Issues @@ -191,6 +197,6 @@ Recent decisions affecting current work: ## Session Continuity Last session: 2026-01-30 -Stopped at: Completed quick-001 (Fix multiple image inputs lost in dynamicInputs) +Stopped at: Completed Phase 34 Plan 01 (Chat Agent Library) Resume file: None -Next action: Execute plan 33-02 (UI integration - Revert AI Changes button) +Next action: Execute Phase 34 Plan 02 (Enhance chat API route with tool calling) diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index af72fd89..ba63fe4b 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -1,77 +1,18 @@ -import { streamText, convertToModelMessages, UIMessage } from 'ai'; +import { streamText, convertToModelMessages, UIMessage, stepCountIs } from 'ai'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; +import { createChatTools, buildEditSystemPrompt } from '@/lib/chat/tools'; +import { buildWorkflowContext } from '@/lib/chat/contextBuilder'; +import { WorkflowNode } from '@/types'; +import { WorkflowEdge } from '@/types/workflow'; export const maxDuration = 60; // 1 minute timeout -// System prompt with Node Banana domain expertise -const SYSTEM_PROMPT = `You are a workflow expert for Node Banana, a visual node-based AI image generation tool. Be concise and direct — short bullet points, no fluff. Use the same language the user sees in the UI. Never expose internal property names, JSON structure, or code. - -## Node Types - -### Image Input -Upload or load source images. Connects its **image** output to other nodes. - -### Prompt -A text box where users write generation instructions. Connects its **text** output to Generate or LLM nodes. - -### Generate Image (nanoBanana) -AI image generation. Requires both an **image** connection AND a **text** connection. -- **Model dropdown**: Choose "Nano Banana" (fast) or "Nano Banana Pro" (high quality). Can also use Replicate or fal.ai models via the model browser. -- **Aspect Ratio dropdown**: 1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9, 21:9 -- **Resolution dropdown** (Nano Banana Pro only): 1K, 2K, or 4K — this is a dropdown on the node, NOT something you put in the prompt -- **Google Search checkbox** (Nano Banana Pro only): enables grounding with web search -- Can accept **multiple image inputs** from different Image Input nodes -- External provider models (Replicate, fal.ai) show additional parameter controls like seed, steps, guidance - -### Generate Video -AI video generation. Takes image + text inputs, outputs video. Only available with Replicate or fal.ai models (not Gemini). - -### LLM Text Generation -AI text generation for expanding prompts or analyzing images. -- **Provider dropdown**: Google or OpenAI -- **Model dropdown**: Gemini 3 Flash, Gemini 2.5 Flash, Gemini 3.0 Pro (Google) / GPT-4.1 Mini, GPT-4.1 Nano (OpenAI) -- **Parameters** (collapsible): Temperature slider (0-2), Max Tokens slider (256-16384) -- Takes **text** input (required), optional **image** input - -### Split Grid -Splits one image into a grid for parallel generation. Click "Configure" to open settings: -- **Number of Images**: Choose 4, 6, 8, 9, or 10 (shows grid preview) -- **Default Prompt**: Applied to all generated images (each can be edited individually after) -- Automatically creates child Image Input + Prompt + Generate nodes for each grid cell - -### Annotation -Draw or mark up images using a canvas editor (Konva). Takes an image in, outputs the annotated image. - -### Output -Displays the final generated image or video. Connect any image or video output here to see results. - -## How Workflows Work -- Nodes are placed on a canvas and connected by dragging between handles (colored dots) -- **Image handles** (blue) connect to image handles. **Text handles** (green) connect to text handles. -- One Image Input can fan out to many Generate nodes — just draw multiple connections -- Each node can be renamed by editing its title -- Nodes can be visually grouped with colored boxes for organization -- Workflows run left-to-right: input → processing → output - -## Common Questions & Correct Answers -- "How do I change resolution?" → Use the **Resolution dropdown** on the Generate node (not the prompt). Only available with Nano Banana Pro. -- "How do I change aspect ratio?" → Use the **Aspect Ratio dropdown** on the Generate node. -- "How do I switch models?" → Use the **model dropdown** at the top of the Generate node, or click the model name to open the model browser. -- "How do I get multiple variations?" → Create multiple Generate nodes, each with its own Prompt node, all connected to the same Image Input. -- "How do I upscale?" → Change the Resolution dropdown from 2K to 4K on the Generate node. - -## Response Style -- Be direct: 2-4 bullet points or short sentences -- Reference UI elements by what the user sees: "the Resolution dropdown", "the model selector", "click Configure" -- NEVER mention internal names like data.resolution, aspectRatio, targetCount, selectedModel, etc. -- NEVER output JSON, code snippets, or node data structures -- Suggest actual prompt text in quotes when relevant -- Ask one clarifying question at a time if goal is unclear -- When they're ready, mention the "Build Workflow" button`; - export async function POST(request: Request) { try { - const { messages } = await request.json() as { messages: UIMessage[] }; + const { messages, workflowState } = await request.json() as { + messages: UIMessage[]; + workflowState?: { nodes: WorkflowNode[]; edges: WorkflowEdge[] }; + }; // Get API key from environment const apiKey = process.env.GEMINI_API_KEY; @@ -79,17 +20,35 @@ export async function POST(request: Request) { return new Response('GEMINI_API_KEY not configured', { status: 500 }); } + // Build workflow context from client-provided state + const context = buildWorkflowContext( + workflowState?.nodes || [], + workflowState?.edges || [] + ); + + // Build context-aware system prompt + const systemPrompt = buildEditSystemPrompt(context); + + // Extract node IDs for tool validation + const nodeIds = (workflowState?.nodes || []).map(n => n.id); + + // Create chat tools with current workflow context + const tools = createChatTools(nodeIds); + // Create Google provider with API key const google = createGoogleGenerativeAI({ apiKey }); // Convert UI messages to model messages format const modelMessages = await convertToModelMessages(messages); - // Create streaming response using Vercel AI SDK + // Create streaming response with tool calling const result = streamText({ model: google('gemini-2.5-flash'), - system: SYSTEM_PROMPT, + system: systemPrompt, messages: modelMessages, + tools: tools, + toolChoice: 'auto', // Let LLM decide which tool to use + stopWhen: stepCountIs(3), // Allow multi-step reasoning for complex requests }); // Return the UI message stream response for useChat compatibility