diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 5fe519ed..5dbc6dad 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -480,19 +480,29 @@ Plans: **Plans**: 2 plans Plans: -- [ ] 33-01-PLAN.md — Store snapshot state, capture/revert/clear actions, manual change tracking -- [ ] 33-02-PLAN.md — Wire snapshot capture in AI flow, Revert AI Changes button in Header +- [x] 33-01-PLAN.md — Store snapshot state, capture/revert/clear actions, manual change tracking +- [x] 33-02-PLAN.md — Wire snapshot capture in AI flow, Revert AI Changes button in Header -#### Phase 34: Agentic Workflow Editing +#### Phase 34: Context-Aware Agentic Workflow Editing -**Goal**: Agent interprets edit requests and modifies workflow JSON with change narration +**Goal**: Make the chat agent context-aware and multi-modal — routing user messages by intent to different behaviors: answering app usage questions, creating new workflows from scratch, or making targeted edits to the current workflow with full project context awareness and change narration **Depends on**: Phase 33 -**Research**: Likely (system prompts for structured edits) -**Research topics**: System prompts for JSON file updates, structured output for workflow modifications, change explanation patterns -**Plans**: TBD +**Research**: Likely (intent classification, context-aware prompting, structured edits) +**Research topics**: Intent classification approaches (keyword vs LLM-based), system prompt design for context-aware editing, workflow JSON mutation strategies, project context injection, change explanation patterns +**Plans**: 3 plans + +**Features:** +1. Intent detection: classify user messages as help/question, new workflow creation, or workflow editing +2. Help mode: answer questions about how to use the app without modifying anything +3. Create mode: build new workflows from scratch (existing behavior) +4. Edit mode: interpret edit requests against the current workflow, make targeted JSON modifications +5. Project context awareness: agent understands current nodes, connections, models, and parameters +6. Change narration: explain what was modified and why Plans: -- [ ] 34-01: TBD (run /gsd:plan-phase 34 to break down) +- [ ] 34-01-PLAN.md — Chat agent library: tool definitions, edit operations, context builder +- [ ] 34-02-PLAN.md — Enhanced /api/chat route with tool calling for intent routing +- [ ] 34-03-PLAN.md — Store applyEditOperations, ChatPanel tool result handling, end-to-end wiring #### Phase 35: Large Workflow Handling @@ -655,8 +665,8 @@ Phases execute in numeric order: 1 → 2 → ... → 35 → 36 → 37 → 38 → | 30. Small Fixes | v1.3 | 0/0 | Deferred | - | | 31. Workflow Proposal System | v1.4 | 2/2 | Complete | 2026-01-26 | | 32. Chat UI Foundation | v1.4 | 2/2 | Complete | 2026-01-27 | -| 33. Workflow Edit Safety | v1.4 | 0/? | Not started | - | -| 34. Agentic Workflow Editing | v1.4 | 0/? | Not started | - | +| 33. Workflow Edit Safety | v1.4 | 2/2 | Complete | 2026-01-30 | +| 34. Context-Aware Agentic Editing | v1.4 | 0/? | Not started | - | | 35. Large Workflow Handling | v1.4 | 0/? | Not started | - | | 36. Execution Engine Extraction | v1.5 | 0/3 | Not started | - | | 37. Pure Helpers Extraction | v1.5 | 0/2 | Not started | - | diff --git a/src/lib/chat/contextBuilder.ts b/src/lib/chat/contextBuilder.ts new file mode 100644 index 00000000..fabe3d90 --- /dev/null +++ b/src/lib/chat/contextBuilder.ts @@ -0,0 +1,126 @@ +import { WorkflowNode } from "@/types"; +import { WorkflowEdge } from "@/types/workflow"; +import type { + NanoBananaNodeData, + GenerateVideoNodeData, +} from "@/types"; + +/** + * Lightweight workflow context for LLM consumption. + * Strips all base64 image data, history arrays, and internal state. + */ +export interface WorkflowContext { + nodeCount: number; + nodes: Array<{ + id: string; + type: string; + title: string; + model?: string; + }>; + connections: Array<{ + from: string; + to: string; + type: string; + }>; + isEmpty: boolean; +} + +/** + * Builds a lightweight workflow context from nodes and edges. + * Omits all base64 image data, history arrays, and internal state. + * + * @param nodes - Current workflow nodes + * @param edges - Current workflow edges + * @returns Lightweight workflow context suitable for LLM injection + */ +export function buildWorkflowContext( + nodes: WorkflowNode[], + edges: WorkflowEdge[] +): WorkflowContext { + const isEmpty = nodes.length === 0; + + // Map nodes to lightweight summaries + const nodeSummaries = nodes.map((node) => { + // Extract title: use customTitle if available, otherwise generate from type + const title = + (node.data as { customTitle?: string }).customTitle || + generateNodeTitle(node.type); + + // For nanoBanana and generateVideo nodes, include model name + let model: string | undefined; + if (node.type === "nanoBanana" || node.type === "generateVideo") { + const nodeData = node.data as NanoBananaNodeData | GenerateVideoNodeData; + model = nodeData.selectedModel?.displayName; + } + + return { + id: node.id, + type: node.type, + title, + ...(model && { model }), + }; + }); + + // Map edges to connection summaries + const connections = edges.map((edge) => ({ + from: edge.source, + to: edge.target, + type: edge.sourceHandle || "unknown", + })); + + return { + nodeCount: nodes.length, + nodes: nodeSummaries, + connections, + isEmpty, + }; +} + +/** + * Formats workflow context as a readable string for injection into LLM system prompt. + * + * @param context - Workflow context + * @returns Formatted string suitable for system prompt + */ +export function formatContextForPrompt(context: WorkflowContext): string { + if (context.isEmpty) { + return "The canvas is currently empty."; + } + + const lines: string[] = []; + + // List nodes + lines.push(`Current workflow has ${context.nodeCount} node(s):`); + for (const node of context.nodes) { + const modelInfo = node.model ? ` (${node.model})` : ""; + lines.push(` - ${node.id}: ${node.title}${modelInfo}`); + } + + // List connections + if (context.connections.length > 0) { + lines.push(""); + lines.push("Connections:"); + for (const conn of context.connections) { + lines.push(` - ${conn.from} → ${conn.to} (${conn.type})`); + } + } + + return lines.join("\n"); +} + +/** + * Generates a human-readable title from a node type. + */ +function generateNodeTitle(type: string): string { + const titles: Record = { + imageInput: "Image Input", + annotation: "Annotation", + prompt: "Prompt", + nanoBanana: "Generate Image", + generateVideo: "Generate Video", + llmGenerate: "LLM Generate", + splitGrid: "Split Grid", + output: "Output", + }; + return titles[type] || type; +} diff --git a/src/lib/chat/editOperations.ts b/src/lib/chat/editOperations.ts new file mode 100644 index 00000000..38722285 --- /dev/null +++ b/src/lib/chat/editOperations.ts @@ -0,0 +1,215 @@ +import { NodeType, WorkflowNode, WorkflowNodeData } from "@/types"; +import { WorkflowEdge } from "@/types/workflow"; +import { createDefaultNodeData } from "@/store/utils/nodeDefaults"; +import { defaultNodeDimensions } from "@/store/utils/nodeDefaults"; + +/** + * Edit operation types for workflow modifications. + * Each operation represents a single atomic change to the workflow. + */ +export type EditOperation = + | { + type: "addNode"; + nodeType: NodeType; + position?: { x: number; y: number }; + data?: Record; + } + | { type: "removeNode"; nodeId: string } + | { type: "updateNode"; nodeId: string; data: Record } + | { + type: "addEdge"; + source: string; + target: string; + sourceHandle?: string; + targetHandle?: string; + } + | { type: "removeEdge"; edgeId: string }; + +/** + * Result of applying edit operations to the workflow. + */ +export interface ApplyEditResult { + nodes: WorkflowNode[]; + edges: WorkflowEdge[]; + applied: number; + skipped: string[]; +} + +/** + * Applies a batch of edit operations to the current workflow state. + * Uses immutable updates (single pass, not individual setState calls). + * Invalid operations are skipped with reasons tracked. + * + * @param operations - List of edit operations to apply + * @param storeState - Current workflow state (nodes and edges) + * @returns Updated nodes, edges, count of applied operations, and skipped operations with reasons + */ +export function applyEditOperations( + operations: EditOperation[], + storeState: { nodes: WorkflowNode[]; edges: WorkflowEdge[] } +): ApplyEditResult { + let nodes = [...storeState.nodes]; + let edges = [...storeState.edges]; + const skipped: string[] = []; + let applied = 0; + + for (const [index, operation] of operations.entries()) { + switch (operation.type) { + case "addNode": { + // Generate unique ID with timestamp and index + const nodeId = `${operation.nodeType}-ai-${Date.now()}-${index}`; + + // Get default position and data + const position = operation.position ?? { x: 200, y: 200 }; + const defaultData = createDefaultNodeData(operation.nodeType); + const dimensions = defaultNodeDimensions[operation.nodeType]; + + // Merge provided data with defaults + const nodeData = { + ...defaultData, + ...operation.data, + } as WorkflowNodeData; + + // Create new node + const newNode: WorkflowNode = { + id: nodeId, + type: operation.nodeType, + position, + data: nodeData, + measured: dimensions, + }; + + nodes.push(newNode); + applied++; + break; + } + + case "removeNode": { + const nodeExists = nodes.find((n) => n.id === operation.nodeId); + if (!nodeExists) { + skipped.push( + `removeNode: node "${operation.nodeId}" not found` + ); + break; + } + + // Remove node and its connected edges + nodes = nodes.filter((n) => n.id !== operation.nodeId); + edges = edges.filter( + (e) => e.source !== operation.nodeId && e.target !== operation.nodeId + ); + applied++; + break; + } + + case "updateNode": { + const nodeIndex = nodes.findIndex((n) => n.id === operation.nodeId); + if (nodeIndex === -1) { + skipped.push( + `updateNode: node "${operation.nodeId}" not found` + ); + break; + } + + // Update node data immutably + nodes = nodes.map((n) => + n.id === operation.nodeId + ? { + ...n, + data: { + ...n.data, + ...operation.data, + } as WorkflowNodeData, + } + : n + ); + applied++; + break; + } + + case "addEdge": { + // Validate source and target nodes exist + const sourceExists = nodes.find((n) => n.id === operation.source); + const targetExists = nodes.find((n) => n.id === operation.target); + + if (!sourceExists) { + skipped.push( + `addEdge: source node "${operation.source}" not found` + ); + break; + } + if (!targetExists) { + skipped.push( + `addEdge: target node "${operation.target}" not found` + ); + break; + } + + // Generate edge ID + const handleSuffix = operation.sourceHandle + ? `-${operation.sourceHandle}` + : ""; + const edgeId = `edge-ai-${operation.source}-${operation.target}${handleSuffix}`; + + // Create new edge + const newEdge: WorkflowEdge = { + id: edgeId, + source: operation.source, + target: operation.target, + sourceHandle: operation.sourceHandle, + targetHandle: operation.targetHandle, + }; + + edges.push(newEdge); + applied++; + break; + } + + case "removeEdge": { + const edgeExists = edges.find((e) => e.id === operation.edgeId); + if (!edgeExists) { + skipped.push( + `removeEdge: edge "${operation.edgeId}" not found` + ); + break; + } + + edges = edges.filter((e) => e.id !== operation.edgeId); + applied++; + break; + } + } + } + + return { + nodes, + edges, + applied, + skipped, + }; +} + +/** + * Generates a human-readable summary of what operations were applied. + * + * @param operations - List of edit operations + * @returns Human-readable summary string + */ +export function narrateOperations(operations: EditOperation[]): string { + const narratives = operations.map((op) => { + switch (op.type) { + case "addNode": + return `Added a ${op.nodeType} node`; + case "removeNode": + return `Removed node ${op.nodeId}`; + case "updateNode": + return `Updated ${op.nodeId} settings`; + case "addEdge": + return `Connected ${op.source} to ${op.target}`; + case "removeEdge": + return `Removed connection ${op.edgeId}`; + } + }); + + return narratives.join("\n"); +}