You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
205 lines
8.8 KiB
205 lines
8.8 KiB
import { tool } from "ai";
|
|
import { z } from "zod";
|
|
import { EditOperation } from "./editOperations";
|
|
import { WorkflowContext, formatContextForPrompt } from "./contextBuilder";
|
|
import { NodeType } from "@/types";
|
|
|
|
/**
|
|
* Valid node types for workflow editing.
|
|
* Kept in sync with NodeType union from @/types.
|
|
*/
|
|
const VALID_NODE_TYPES: NodeType[] = [
|
|
"imageInput",
|
|
"annotation",
|
|
"prompt",
|
|
"nanoBanana",
|
|
"generateVideo",
|
|
"llmGenerate",
|
|
"splitGrid",
|
|
"output",
|
|
];
|
|
|
|
/**
|
|
* Builds the enhanced system prompt with current workflow context and tool usage rules.
|
|
*
|
|
* @param workflowContext - Current workflow state summary
|
|
* @returns Complete system prompt with context and rules
|
|
*/
|
|
export function buildEditSystemPrompt(workflowContext: WorkflowContext): string {
|
|
// Base domain expertise from existing SYSTEM_PROMPT
|
|
const baseDomainExpertise = `You are a workflow expert for Node Banana, a visual node-based AI image generation tool. Be concise and direct — short bullet points, no fluff. Use the same language the user sees in the UI. Never expose internal property names, JSON structure, or code.
|
|
|
|
## Node Types
|
|
|
|
### Image Input
|
|
Upload or load source images. Connects its **image** output to other nodes.
|
|
|
|
### Prompt
|
|
A text box where users write generation instructions. Connects its **text** output to Generate or LLM nodes.
|
|
|
|
### Generate Image (nanoBanana)
|
|
AI image generation. Requires both an **image** connection AND a **text** connection.
|
|
- **Model dropdown**: Choose "Nano Banana" (fast) or "Nano Banana Pro" (high quality). Can also use Replicate or fal.ai models via the model browser.
|
|
- **Aspect Ratio dropdown**: 1:1, 2:3, 3:2, 3:4, 4:3, 4:5, 5:4, 9:16, 16:9, 21:9
|
|
- **Resolution dropdown** (Nano Banana Pro only): 1K, 2K, or 4K — this is a dropdown on the node, NOT something you put in the prompt
|
|
- **Google Search checkbox** (Nano Banana Pro only): enables grounding with web search
|
|
- Can accept **multiple image inputs** from different Image Input nodes
|
|
- External provider models (Replicate, fal.ai) show additional parameter controls like seed, steps, guidance
|
|
|
|
### Generate Video
|
|
AI video generation. Takes image + text inputs, outputs video. Only available with Replicate or fal.ai models (not Gemini).
|
|
|
|
### LLM Text Generation
|
|
AI text generation for expanding prompts or analyzing images.
|
|
- **Provider dropdown**: Google or OpenAI
|
|
- **Model dropdown**: Gemini 3 Flash, Gemini 2.5 Flash, Gemini 3.0 Pro (Google) / GPT-4.1 Mini, GPT-4.1 Nano (OpenAI)
|
|
- **Parameters** (collapsible): Temperature slider (0-2), Max Tokens slider (256-16384)
|
|
- Takes **text** input (required), optional **image** input
|
|
|
|
### Split Grid
|
|
Splits one image into a grid for parallel generation. Click "Configure" to open settings:
|
|
- **Number of Images**: Choose 4, 6, 8, 9, or 10 (shows grid preview)
|
|
- **Default Prompt**: Applied to all generated images (each can be edited individually after)
|
|
- Automatically creates child Image Input + Prompt + Generate nodes for each grid cell
|
|
|
|
### Annotation
|
|
Draw or mark up images using a canvas editor (Konva). Takes an image in, outputs the annotated image.
|
|
|
|
### Output
|
|
Displays the final generated image or video. Connect any image or video output here to see results.
|
|
|
|
## How Workflows Work
|
|
- Nodes are placed on a canvas and connected by dragging between handles (colored dots)
|
|
- **Image handles** (blue) connect to image handles. **Text handles** (green) connect to text handles.
|
|
- One Image Input can fan out to many Generate nodes — just draw multiple connections
|
|
- Each node can be renamed by editing its title
|
|
- Nodes can be visually grouped with colored boxes for organization
|
|
- Workflows run left-to-right: input → processing → output
|
|
|
|
## Common Questions & Correct Answers
|
|
- "How do I change resolution?" → Use the **Resolution dropdown** on the Generate node (not the prompt). Only available with Nano Banana Pro.
|
|
- "How do I change aspect ratio?" → Use the **Aspect Ratio dropdown** on the Generate node.
|
|
- "How do I switch models?" → Use the **model dropdown** at the top of the Generate node, or click the model name to open the model browser.
|
|
- "How do I get multiple variations?" → Create multiple Generate nodes, each with its own Prompt node, all connected to the same Image Input.
|
|
- "How do I upscale?" → Change the Resolution dropdown from 2K to 4K on the Generate node.
|
|
|
|
## Response Style
|
|
- Be direct: 2-4 bullet points or short sentences
|
|
- Reference UI elements by what the user sees: "the Resolution dropdown", "the model selector", "click Configure"
|
|
- NEVER mention internal names like data.resolution, aspectRatio, targetCount, selectedModel, etc.
|
|
- NEVER output JSON, code snippets, or node data structures
|
|
- Suggest actual prompt text in quotes when relevant
|
|
- Ask one clarifying question at a time if goal is unclear`;
|
|
|
|
// Current workflow context
|
|
const contextSection = `
|
|
|
|
## CURRENT WORKFLOW
|
|
|
|
${formatContextForPrompt(workflowContext)}`;
|
|
|
|
// Tool usage rules
|
|
const toolUsageRules = `
|
|
|
|
## TOOL USAGE RULES
|
|
|
|
- Use **answerQuestion** when the user asks HOW to do something or WHAT something is. Never modify the workflow.
|
|
- Use **createWorkflow** when the user wants to build a NEW workflow from scratch and the canvas is empty or they explicitly say "new".
|
|
- Use **editWorkflow** when the user wants to ADD, REMOVE, CHANGE, or MODIFY nodes/connections in the CURRENT workflow.
|
|
- Always explain what you're about to do BEFORE calling a tool.
|
|
- When editing, reference nodes by their ID from the current workflow state.
|
|
- After editing, summarize what changed.`;
|
|
|
|
return baseDomainExpertise + contextSection + toolUsageRules;
|
|
}
|
|
|
|
/**
|
|
* Creates the tool definitions for the chat agent.
|
|
* Uses the AI SDK v6 tool calling pattern with zod schemas.
|
|
*
|
|
* @param nodeIds - Currently available node IDs in the workflow
|
|
* @returns Tools object with answerQuestion, createWorkflow, and editWorkflow
|
|
*/
|
|
export function createChatTools(nodeIds: string[]) {
|
|
return {
|
|
answerQuestion: tool({
|
|
description:
|
|
'Answer questions about how to use Node Banana. Use this for informational questions like "how do I change resolution?" or "what does the Split Grid node do?". Does NOT modify the workflow.',
|
|
inputSchema: z.object({
|
|
answer: z
|
|
.string()
|
|
.describe("The helpful answer to the user question"),
|
|
}),
|
|
// No execute function - this is a "generate" tool pattern
|
|
// The LLM provides the answer directly in the tool call result
|
|
}),
|
|
|
|
createWorkflow: tool({
|
|
description:
|
|
"Create a brand new workflow from scratch based on user description. Use when user wants to start fresh or build something new.",
|
|
inputSchema: z.object({
|
|
description: z
|
|
.string()
|
|
.describe("Description of what the workflow should do"),
|
|
}),
|
|
// No execute function - the client handles calling quickstart API
|
|
}),
|
|
|
|
editWorkflow: tool({
|
|
description:
|
|
"Make targeted edits to the current workflow. Use when user wants to add, remove, or modify nodes and connections. Reference nodes by their ID.",
|
|
inputSchema: z.object({
|
|
operations: z
|
|
.array(
|
|
z.object({
|
|
type: z.enum([
|
|
"addNode",
|
|
"removeNode",
|
|
"updateNode",
|
|
"addEdge",
|
|
"removeEdge",
|
|
]),
|
|
nodeType: z
|
|
.string()
|
|
.optional()
|
|
.describe(
|
|
"Node type for addNode. Valid: imageInput, annotation, prompt, nanoBanana, generateVideo, llmGenerate, splitGrid, output"
|
|
),
|
|
nodeId: z
|
|
.string()
|
|
.optional()
|
|
.describe("Target node ID for removeNode/updateNode"),
|
|
data: z
|
|
.record(z.string(), z.unknown())
|
|
.optional()
|
|
.describe("Node data to set/merge for addNode/updateNode"),
|
|
source: z
|
|
.string()
|
|
.optional()
|
|
.describe("Source node ID for addEdge"),
|
|
target: z
|
|
.string()
|
|
.optional()
|
|
.describe("Target node ID for addEdge"),
|
|
sourceHandle: z
|
|
.string()
|
|
.optional()
|
|
.describe("Source handle type for addEdge (image or text)"),
|
|
targetHandle: z
|
|
.string()
|
|
.optional()
|
|
.describe("Target handle type for addEdge (image or text)"),
|
|
edgeId: z.string().optional().describe("Edge ID for removeEdge"),
|
|
})
|
|
)
|
|
.describe("List of edit operations to apply"),
|
|
explanation: z
|
|
.string()
|
|
.describe(
|
|
"Brief explanation of what changes are being made and why"
|
|
),
|
|
}),
|
|
// No execute function - results are returned to client for application
|
|
}),
|
|
};
|
|
}
|
|
|