diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 5dbc6dad..54d354f8 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -500,7 +500,7 @@ Plans: 6. Change narration: explain what was modified and why Plans: -- [ ] 34-01-PLAN.md — Chat agent library: tool definitions, edit operations, context builder +- [x] 34-01-PLAN.md — Chat agent library: tool definitions, edit operations, context builder - [ ] 34-02-PLAN.md — Enhanced /api/chat route with tool calling for intent routing - [ ] 34-03-PLAN.md — Store applyEditOperations, ChatPanel tool result handling, end-to-end wiring @@ -666,7 +666,7 @@ Phases execute in numeric order: 1 → 2 → ... → 35 → 36 → 37 → 38 → | 31. Workflow Proposal System | v1.4 | 2/2 | Complete | 2026-01-26 | | 32. Chat UI Foundation | v1.4 | 2/2 | Complete | 2026-01-27 | | 33. Workflow Edit Safety | v1.4 | 2/2 | Complete | 2026-01-30 | -| 34. Context-Aware Agentic Editing | v1.4 | 0/? | Not started | - | +| 34. Context-Aware Agentic Editing | v1.4 | 1/3 | In progress | - | | 35. Large Workflow Handling | v1.4 | 0/? | Not started | - | | 36. Execution Engine Extraction | v1.5 | 0/3 | Not started | - | | 37. Pure Helpers Extraction | v1.5 | 0/2 | Not started | - | diff --git a/.planning/STATE.md b/.planning/STATE.md index 0c808514..e8b737ae 100644 --- a/.planning/STATE.md +++ b/.planning/STATE.md @@ -9,19 +9,19 @@ See: .planning/PROJECT.md (updated 2026-01-09) ## Current Position -Phase: 33 of 35 (Workflow Edit Safety) -Plan: 1 of 2 in current phase +Phase: 34 of 35 (Context-Aware Agentic Workflow Editing) +Plan: 1 of ? in current phase Status: In progress -Last activity: 2026-01-30 - Completed quick-001 (Fix multiple image inputs lost in dynamicInputs) +Last activity: 2026-01-30 - Completed plan 01 (Chat Agent Library) Progress: ░░░░░░░░░░ 6% ## Performance Metrics **Velocity:** -- Total plans completed: 31 +- Total plans completed: 32 - Average duration: 7 min -- Total execution time: 3.6 hours +- Total execution time: 3.7 hours **By Phase:** @@ -54,11 +54,12 @@ Progress: ░░░░░░░░░░ 6% | 28. Node Defaults UI | 1/1 | 32 min | 32 min | | 31. Workflow Proposal System | 2/2 | 6 min | 3 min | | 32. Chat UI Foundation | 2/2 | 9 min | 4.5 min | -| 33. Workflow Edit Safety | 1/2 | 5 min | 5 min | +| 33. Workflow Edit Safety | 2/2 | 5 min | 5 min | +| 34. Agentic Workflow Editing | 1/? | 5 min | 5 min | **Recent Trend:** -- Last 5 plans: 4 min, 2 min, 4 min, 5 min, 5 min -- Trend: Store infrastructure and chat integration work completing quickly +- Last 5 plans: 2 min, 4 min, 5 min, 5 min, 5 min +- Trend: Chat and agentic features shipping quickly with consistent 5-min execution ## Accumulated Context @@ -156,6 +157,11 @@ Recent decisions affecting current work: - dynamicInputs type is Record to support multi-image aggregation - Single image stays as string; only multiple images to same schema key become array - Array.isArray guard on dynamicInputs.prompt access (takes first element) +- AI SDK v6 tool pattern uses inputSchema (not parameters) for zod schemas +- Chat agent tools use "generate" pattern (no execute function) - LLM provides structured output +- Edit operations use batched immutable updates with skip tracking for invalid operations +- Node IDs for AI-generated nodes: ${nodeType}-ai-${Date.now()}-${index} pattern +- Workflow context builder strips base64 data, history arrays, and internal state for LLM consumption ### Deferred Issues @@ -191,6 +197,6 @@ Recent decisions affecting current work: ## Session Continuity Last session: 2026-01-30 -Stopped at: Completed quick-001 (Fix multiple image inputs lost in dynamicInputs) +Stopped at: Completed Phase 34 Plan 01 (Chat Agent Library) Resume file: None -Next action: Execute plan 33-02 (UI integration - Revert AI Changes button) +Next action: Execute Phase 34 Plan 02 (Enhance chat API route with tool calling) diff --git a/src/app/api/chat/route.ts b/src/app/api/chat/route.ts index af72fd89..ba63fe4b 100644 --- a/src/app/api/chat/route.ts +++ b/src/app/api/chat/route.ts @@ -1,77 +1,18 @@ -import { streamText, convertToModelMessages, UIMessage } from 'ai'; +import { streamText, convertToModelMessages, UIMessage, stepCountIs } from 'ai'; import { createGoogleGenerativeAI } from '@ai-sdk/google'; +import { createChatTools, buildEditSystemPrompt } from '@/lib/chat/tools'; +import { buildWorkflowContext } from '@/lib/chat/contextBuilder'; +import { WorkflowNode } from '@/types'; +import { WorkflowEdge } from '@/types/workflow'; export const maxDuration = 60; // 1 minute timeout -// System prompt with Node Banana domain expertise -const SYSTEM_PROMPT = `You are a workflow expert for Node Banana, a visual node-based AI image generation tool. Be concise and direct — short bullet points, no fluff. Use the same language the user sees in the UI. Never expose internal property names, JSON structure, or code. - -## Node Types - -### Image Input -Upload or load source images. Connects its **image** output to other nodes. - -### Prompt -A text box where users write generation instructions. Connects its **text** output to Generate or LLM nodes. - -### Generate Image (nanoBanana) -AI image generation. Requires both an **image** connection AND a **text** connection. -- **Model dropdown**: Choose "Nano Banana" (fast) or "Nano Banana Pro" (high quality). Can also use Replicate or fal.ai models via the model browser. -- **Aspect Ratio dropdown**: 1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9, 21:9 -- **Resolution dropdown** (Nano Banana Pro only): 1K, 2K, or 4K — this is a dropdown on the node, NOT something you put in the prompt -- **Google Search checkbox** (Nano Banana Pro only): enables grounding with web search -- Can accept **multiple image inputs** from different Image Input nodes -- External provider models (Replicate, fal.ai) show additional parameter controls like seed, steps, guidance - -### Generate Video -AI video generation. Takes image + text inputs, outputs video. Only available with Replicate or fal.ai models (not Gemini). - -### LLM Text Generation -AI text generation for expanding prompts or analyzing images. -- **Provider dropdown**: Google or OpenAI -- **Model dropdown**: Gemini 3 Flash, Gemini 2.5 Flash, Gemini 3.0 Pro (Google) / GPT-4.1 Mini, GPT-4.1 Nano (OpenAI) -- **Parameters** (collapsible): Temperature slider (0-2), Max Tokens slider (256-16384) -- Takes **text** input (required), optional **image** input - -### Split Grid -Splits one image into a grid for parallel generation. Click "Configure" to open settings: -- **Number of Images**: Choose 4, 6, 8, 9, or 10 (shows grid preview) -- **Default Prompt**: Applied to all generated images (each can be edited individually after) -- Automatically creates child Image Input + Prompt + Generate nodes for each grid cell - -### Annotation -Draw or mark up images using a canvas editor (Konva). Takes an image in, outputs the annotated image. - -### Output -Displays the final generated image or video. Connect any image or video output here to see results. - -## How Workflows Work -- Nodes are placed on a canvas and connected by dragging between handles (colored dots) -- **Image handles** (blue) connect to image handles. **Text handles** (green) connect to text handles. -- One Image Input can fan out to many Generate nodes — just draw multiple connections -- Each node can be renamed by editing its title -- Nodes can be visually grouped with colored boxes for organization -- Workflows run left-to-right: input → processing → output - -## Common Questions & Correct Answers -- "How do I change resolution?" → Use the **Resolution dropdown** on the Generate node (not the prompt). Only available with Nano Banana Pro. -- "How do I change aspect ratio?" → Use the **Aspect Ratio dropdown** on the Generate node. -- "How do I switch models?" → Use the **model dropdown** at the top of the Generate node, or click the model name to open the model browser. -- "How do I get multiple variations?" → Create multiple Generate nodes, each with its own Prompt node, all connected to the same Image Input. -- "How do I upscale?" → Change the Resolution dropdown from 2K to 4K on the Generate node. - -## Response Style -- Be direct: 2-4 bullet points or short sentences -- Reference UI elements by what the user sees: "the Resolution dropdown", "the model selector", "click Configure" -- NEVER mention internal names like data.resolution, aspectRatio, targetCount, selectedModel, etc. -- NEVER output JSON, code snippets, or node data structures -- Suggest actual prompt text in quotes when relevant -- Ask one clarifying question at a time if goal is unclear -- When they're ready, mention the "Build Workflow" button`; - export async function POST(request: Request) { try { - const { messages } = await request.json() as { messages: UIMessage[] }; + const { messages, workflowState } = await request.json() as { + messages: UIMessage[]; + workflowState?: { nodes: WorkflowNode[]; edges: WorkflowEdge[] }; + }; // Get API key from environment const apiKey = process.env.GEMINI_API_KEY; @@ -79,17 +20,35 @@ export async function POST(request: Request) { return new Response('GEMINI_API_KEY not configured', { status: 500 }); } + // Build workflow context from client-provided state + const context = buildWorkflowContext( + workflowState?.nodes || [], + workflowState?.edges || [] + ); + + // Build context-aware system prompt + const systemPrompt = buildEditSystemPrompt(context); + + // Extract node IDs for tool validation + const nodeIds = (workflowState?.nodes || []).map(n => n.id); + + // Create chat tools with current workflow context + const tools = createChatTools(nodeIds); + // Create Google provider with API key const google = createGoogleGenerativeAI({ apiKey }); // Convert UI messages to model messages format const modelMessages = await convertToModelMessages(messages); - // Create streaming response using Vercel AI SDK + // Create streaming response with tool calling const result = streamText({ model: google('gemini-2.5-flash'), - system: SYSTEM_PROMPT, + system: systemPrompt, messages: modelMessages, + tools: tools, + toolChoice: 'auto', // Let LLM decide which tool to use + stopWhen: stepCountIs(3), // Allow multi-step reasoning for complex requests }); // Return the UI message stream response for useChat compatibility