From 70c05c6f7e1545bae260e492e7706117a2397aee Mon Sep 17 00:00:00 2001 From: yuanzoo Date: Sat, 16 May 2026 22:43:01 +0800 Subject: [PATCH] Optimize NewApiWG image parameter flow --- prd-image-workflow.md | 22 +++- src/app/api/generate/__tests__/route.test.ts | 19 +++- .../providers/__tests__/newapiwg.test.ts | 58 +++++++++- src/app/api/generate/providers/newapiwg.ts | 103 ++++++++++++------ src/app/api/generate/route.ts | 4 +- .../__tests__/ModelParameters.test.tsx | 43 ++++++++ src/components/nodes/GenerateImageNode.tsx | 11 ++ src/components/nodes/ModelParameters.tsx | 57 +++++++--- 8 files changed, 260 insertions(+), 57 deletions(-) diff --git a/prd-image-workflow.md b/prd-image-workflow.md index a3cadc29..317ee42d 100644 --- a/prd-image-workflow.md +++ b/prd-image-workflow.md @@ -1,8 +1,8 @@ # Product Requirements Document ## Node-Based Image Annotation & Generation Workflow -**Version:** 1.0 -**Last Updated:** December 2025 +**Version:** 1.0 +**Last Updated:** May 2026 **Status:** Draft --- @@ -146,10 +146,11 @@ Output Node → [image input] | NB-03 | "Generate" button for manual trigger (when not in workflow run) | Should | | NB-04 | Display generation status (idle/loading/complete/error) | Must | | NB-05 | Show thumbnail of generated result | Must | -| NB-06 | Output: generated image as base64 | Must | +| NB-06 | Output: generated image as base64 or provider URL for immediate preview | Must | | NB-07 | Display error messages from API failures | Must | | NB-08 | Aspect ratio selector (1:1, 16:9, 9:16, 4:3, 3:4) | Should | | NB-09 | Resolution selector (1K, 2K) | Should | +| NB-10 | For provider-hosted image URLs, show the URL immediately and persist to local storage asynchronously | Must | **UI:** - Two input handles (image, prompt) on left @@ -259,6 +260,21 @@ Output Node → [image input] **Description:** When user clicks "Run Workflow," execute all nodes in dependency order. +#### 3.5.1 Provider URL-First Image Persistence + +**Context:** Some NewApiWG image models, such as `gpt-image-2-all`, return a provider-hosted image URL in the upstream response. Synchronously downloading that URL before updating the node makes the gateway appear fast while the PopiArt node remains in loading state. + +**Product Strategy:** Prefer fast visual feedback without abandoning local-first persistence. + +| ID | Requirement | Priority | +|----|-------------|----------| +| WF-URL-01 | Display provider-hosted image URLs immediately when the generation API returns them | Must | +| WF-URL-02 | Download and save provider-hosted image URLs to the workflow `generations` folder asynchronously | Must | +| WF-URL-03 | Treat the saved local generation as the durable project artifact after persistence completes | Must | +| WF-URL-04 | For chained generation, pass provider URLs directly to models that accept URL image inputs | Should | +| WF-URL-05 | For models that require inline image data, convert provider URLs only at the next provider boundary instead of blocking initial preview | Should | +| WF-URL-06 | If local persistence fails, keep the provider URL visible and surface save errors through existing logging without failing the completed generation | Should | + | ID | Requirement | Priority | |----|-------------|----------| | WE-01 | Validate workflow before execution (all required inputs connected) | Must | diff --git a/src/app/api/generate/__tests__/route.test.ts b/src/app/api/generate/__tests__/route.test.ts index c73a282b..4a288549 100644 --- a/src/app/api/generate/__tests__/route.test.ts +++ b/src/app/api/generate/__tests__/route.test.ts @@ -1009,21 +1009,22 @@ describe("/api/generate route", () => { ], }), }); - mockFetch.mockResolvedValueOnce({ - ok: true, - headers: new Headers({ "content-type": "image/png" }), - arrayBuffer: () => Promise.resolve(new Uint8Array(Buffer.from("generated-image")).buffer), - }); const request = createMockPostRequest( { prompt: "A cafe scene", + aspectRatio: "9:16", + resolution: "2K", selectedModel: { provider: "newapiwg", modelId: "gpt-image-2-all", displayName: "gpt-image-2-all", capabilities: ["text-to-image", "image-to-image"], }, + parameters: { + quality: "auto", + size: "1024x1024", + }, }, { "X-NewApiWG-Key": "test-newapiwg-key" } ); @@ -1033,8 +1034,14 @@ describe("/api/generate route", () => { expect(response.status).toBe(200); expect(data.success).toBe(true); - expect(data.image).toMatch(/^data:image\/png;base64,/); + expect(data.image).toBe("https://cdn.example.com/generated-image.png"); expect(data.contentType).toBe("image"); + expect(mockFetch).toHaveBeenCalledTimes(1); + const upstreamBody = JSON.parse(mockFetch.mock.calls[0][1].body as string); + expect(upstreamBody.quality).toBe("auto"); + expect(upstreamBody.size).toBe("1024x1536"); + expect(upstreamBody.aspectRatio).toBeUndefined(); + expect(upstreamBody.resolution).toBeUndefined(); }); it("should pass top-level aspect ratio and resolution to native NewApiWG image models", async () => { diff --git a/src/app/api/generate/providers/__tests__/newapiwg.test.ts b/src/app/api/generate/providers/__tests__/newapiwg.test.ts index 46b29d55..69d6606d 100644 --- a/src/app/api/generate/providers/__tests__/newapiwg.test.ts +++ b/src/app/api/generate/providers/__tests__/newapiwg.test.ts @@ -268,6 +268,30 @@ describe("NewApiWG generation payloads", () => { }); }); + it("converts HTTP reference images only when Gemini native generation needs inline data", async () => { + const result = await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ + model: { + id: "apiyi_nano_banana_2", + name: "apiyi_nano_banana_2", + description: null, + provider: "newapiwg", + capabilities: ["text-to-image", "image-to-image"], + }, + prompt: "draw a banana", + images: ["https://cdn.example.com/seedream.png"], + })); + + expect(result.success).toBe(true); + expect(capturedBody!.contents).toEqual([ + { + parts: [ + { text: "draw a banana" }, + { inlineData: { mimeType: "image/png", data: "YWJj" } }, + ], + }, + ]); + }); + it("accepts Gemini native fileData URLs without image file extensions", async () => { vi.stubGlobal("fetch", vi.fn(async (url: string | URL | Request, init?: RequestInit) => { capturedBody = init?.body ? JSON.parse(init.body as string) : null; @@ -369,6 +393,7 @@ describe("NewApiWG generation payloads", () => { }); it("extracts markdown image URLs from gpt-image-2-all chat responses", async () => { + const fetchMock = vi.mocked(fetch); const result = await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ model: { id: "gpt-image-2-all", @@ -395,6 +420,37 @@ describe("NewApiWG generation payloads", () => { { type: "image_url", image_url: { url: "data:image/png;base64,input" } }, ], }]); + expect(fetchMock.mock.calls.some(([url]) => + String(url).includes("r2cdn.copilotbase.com") + )).toBe(false); + }); + + it("lets composer aspect ratio override stale OpenAI-compatible size parameters", async () => { + await generateWithNewApiWG("test-key", "https://newapi.example/v1", makeInput({ + model: { + id: "gpt-image-2-all", + name: "gpt-image-2-all", + description: null, + provider: "newapiwg", + capabilities: ["text-to-image", "image-to-image"], + }, + prompt: "draw a portrait", + parameters: { + quality: "auto", + size: "1024x1024", + aspectRatio: "9:16", + aspect_ratio: "9:16", + resolution: "2K", + imageSize: "2K", + }, + })); + + expect(capturedBody!.quality).toBe("auto"); + expect(capturedBody!.size).toBe("1024x1536"); + expect(capturedBody!.aspectRatio).toBeUndefined(); + expect(capturedBody!.aspect_ratio).toBeUndefined(); + expect(capturedBody!.resolution).toBeUndefined(); + expect(capturedBody!.imageSize).toBeUndefined(); }); it("routes Seedream image models through images generations", async () => { @@ -413,7 +469,7 @@ describe("NewApiWG generation payloads", () => { expect(result.success).toBe(true); expect(result.outputs?.[0]).toEqual({ type: "image", - data: "data:image/png;base64,YWJj", + data: "", url: "https://cdn.example.com/seedream.png", }); expect(capturedBody!.model).toBe("seedream-5-0-260128"); diff --git a/src/app/api/generate/providers/newapiwg.ts b/src/app/api/generate/providers/newapiwg.ts index b7495736..c5551920 100644 --- a/src/app/api/generate/providers/newapiwg.ts +++ b/src/app/api/generate/providers/newapiwg.ts @@ -288,14 +288,6 @@ async function fetchNewApiWGJsonWithRetry( throw new Error("unreachable"); } -async function urlToDataUrl(url: string): Promise { - const response = await fetch(url); - if (!response.ok) throw new Error(`Failed to fetch generated media: ${response.status}`); - const contentType = response.headers.get("content-type") || "application/octet-stream"; - const buffer = Buffer.from(await response.arrayBuffer()); - return `data:${contentType};base64,${buffer.toString("base64")}`; -} - function getNewApiWGTaskId(data: unknown): string | null { if (!data || typeof data !== "object") return null; const record = data as Record; @@ -499,25 +491,27 @@ function ceilToMultiple(value: number, multiple: number): number { } function resolveNewApiWGImageSize(parameters: Record | undefined): string { - const explicitSize = getStringParameter(parameters, "size"); - if (explicitSize) return explicitSize; - const requestedSize = getStringParameter(parameters, "imageSize") ?? getStringParameter(parameters, "resolution"); - if (!requestedSize) return "1024x1024"; - if (/^\d+x\d+$/i.test(requestedSize)) return requestedSize; - - const targetPixels = NEWAPIWG_IMAGE_RESOLUTION_PIXELS[requestedSize]; - if (!targetPixels) return "1024x1024"; + if (requestedSize) { + if (/^\d+x\d+$/i.test(requestedSize)) return requestedSize; + + const targetPixels = NEWAPIWG_IMAGE_RESOLUTION_PIXELS[requestedSize]; + if (targetPixels) { + const ratio = parseAspectRatio( + getStringParameter(parameters, "aspectRatio") ?? + getStringParameter(parameters, "aspect_ratio") + ); + const width = ceilToMultiple(Math.sqrt(targetPixels * ratio.width / ratio.height), 8); + const height = ceilToMultiple(Math.sqrt(targetPixels * ratio.height / ratio.width), 8); + return `${width}x${height}`; + } + } - const ratio = parseAspectRatio( - getStringParameter(parameters, "aspectRatio") ?? - getStringParameter(parameters, "aspect_ratio") - ); - const width = ceilToMultiple(Math.sqrt(targetPixels * ratio.width / ratio.height), 8); - const height = ceilToMultiple(Math.sqrt(targetPixels * ratio.height / ratio.width), 8); - return `${width}x${height}`; + const explicitSize = getStringParameter(parameters, "size"); + if (explicitSize) return explicitSize; + return "1024x1024"; } function isImagenGenerationModel(modelId: string): boolean { @@ -528,6 +522,33 @@ function isOpenAIChatImageModel(modelId: string): boolean { return /(^|[_\-/])sora[_\-/]?image$/i.test(modelId) || modelId === "gpt-image-2-all"; } +function resolveOpenAIChatImageSize(parameters: Record | undefined): string | null { + const aspectRatio = + getStringParameter(parameters, "aspectRatio") ?? + getStringParameter(parameters, "aspect_ratio"); + if (aspectRatio) { + const ratio = parseAspectRatio(aspectRatio); + if (ratio.width < ratio.height) return "1024x1536"; + if (ratio.width > ratio.height) return "1536x1024"; + return "1024x1024"; + } + + return getStringParameter(parameters, "size"); +} + +function normalizeOpenAIChatImageParameters( + parameters: Record | undefined +): Record { + const normalized = { ...(parameters || {}) }; + const size = resolveOpenAIChatImageSize(parameters); + if (size) normalized.size = size; + delete normalized.aspectRatio; + delete normalized.aspect_ratio; + delete normalized.resolution; + delete normalized.imageSize; + return normalized; +} + function isGeminiNativeImageModel(modelId: string): boolean { return /gemini-.*image-preview/i.test(modelId) || /nano[_\-/]?banana/i.test(modelId); } @@ -556,6 +577,10 @@ function getImageUrlCandidate(value: unknown): string | null { return null; } +function isHttpUrl(value: string): boolean { + return value.startsWith("http://") || value.startsWith("https://"); +} + function extractImageFromText(text: string): string | null { const dataUrl = text.match(/data:image\/[a-z0-9.+-]+;base64,[A-Za-z0-9+/=]+/i)?.[0]; if (dataUrl) return dataUrl; @@ -651,7 +676,7 @@ async function generateImageViaNewApiWGChat( messages: [{ role: "user", content }], stream: false, ...dynamicInputsWithoutPrompt(input), - ...(input.parameters || {}), + ...normalizeOpenAIChatImageParameters(input.parameters), }), }, input.model.name, "NewApiWG chat image API error"); @@ -674,11 +699,7 @@ async function generateImageViaNewApiWGChat( } if (image.startsWith("http")) { - try { - return { success: true, outputs: [{ type: "image", data: await urlToDataUrl(image), url: image }] }; - } catch { - return { success: true, outputs: [{ type: "image", data: "", url: image }] }; - } + return { success: true, outputs: [{ type: "image", data: "", url: image }] }; } return { success: true, outputs: [{ type: "image", data: image }] }; @@ -690,6 +711,20 @@ function dataUrlToInlineData(image: string): { mimeType: string; data: string } return { mimeType: match[1], data: match[2] }; } +async function imageUrlToInlineData(image: string): Promise<{ mimeType: string; data: string } | null> { + if (!isHttpUrl(image)) return null; + const response = await fetch(image); + if (!response.ok) throw new Error(`Failed to fetch reference image: ${response.status}`); + const contentType = response.headers.get("content-type"); + const mimeType = contentType?.startsWith("image/") ? contentType : "image/png"; + const buffer = Buffer.from(await response.arrayBuffer()); + return { mimeType, data: buffer.toString("base64") }; +} + +async function imageToInlineData(image: string): Promise<{ mimeType: string; data: string } | null> { + return dataUrlToInlineData(image) ?? imageUrlToInlineData(image); +} + function extractGeminiNativeImage(value: unknown): string | null { const direct = extractNewApiWGImage(value); if (direct) return direct; @@ -739,7 +774,7 @@ async function generateImageViaNewApiWGGemini( ): Promise { const parts: Array<{ text: string } | { inlineData: { mimeType: string; data: string } }> = [{ text: prompt }]; for (const image of input.images || []) { - const inlineData = dataUrlToInlineData(image); + const inlineData = await imageToInlineData(image); if (inlineData) parts.push({ inlineData }); } @@ -908,8 +943,12 @@ export async function generateWithNewApiWG( } if (typeof url === "string" && url.length > 0) { - const dataUrl = url.startsWith("data:") ? url : await urlToDataUrl(url); - return { success: true, outputs: [{ type: isVideo ? "video" : "image", data: dataUrl, url }] }; + if (url.startsWith("data:")) { + return { success: true, outputs: [{ type: isVideo ? "video" : "image", data: url }] }; + } + if (isHttpUrl(url)) { + return { success: true, outputs: [{ type: isVideo ? "video" : "image", data: "", url }] }; + } } return { success: false, error: `No ${isVideo ? "video" : "image"} returned from NewApiWG` }; diff --git a/src/app/api/generate/route.ts b/src/app/api/generate/route.ts index dd56b31c..93ab2da2 100644 --- a/src/app/api/generate/route.ts +++ b/src/app/api/generate/route.ts @@ -89,11 +89,11 @@ function mergeImageGenerationParameters( resolution: string | undefined ): Record | undefined { const merged = { ...(parameters || {}) }; - if (aspectRatio && merged.aspectRatio === undefined && merged.aspect_ratio === undefined) { + if (aspectRatio) { merged.aspectRatio = aspectRatio; merged.aspect_ratio = aspectRatio; } - if (resolution && merged.resolution === undefined && merged.imageSize === undefined) { + if (resolution) { merged.resolution = resolution; merged.imageSize = resolution; } diff --git a/src/components/__tests__/ModelParameters.test.tsx b/src/components/__tests__/ModelParameters.test.tsx index 7f2676ea..8664b1b1 100644 --- a/src/components/__tests__/ModelParameters.test.tsx +++ b/src/components/__tests__/ModelParameters.test.tsx @@ -217,6 +217,49 @@ describe("ModelParameters", () => { }); describe("Parameter Count Display", () => { + it("should hide composer-priority parameters and keep visible defaults", async () => { + const onParametersChange = vi.fn(); + (global.fetch as ReturnType).mockResolvedValueOnce({ + ok: true, + json: () => + Promise.resolve({ + parameters: [ + createMockParameter({ + name: "quality", + type: "string", + enum: ["auto", "high"], + default: "auto", + }), + createMockParameter({ + name: "size", + type: "string", + enum: ["1024x1024", "1024x1536"], + default: "1024x1024", + }), + ], + }), + }); + + render( + + ); + + await waitFor(() => { + expect(screen.getByText("Quality")).toBeInTheDocument(); + }); + + expect(screen.queryByText("Size")).not.toBeInTheDocument(); + expect(onParametersChange).toHaveBeenCalledWith({}); + await waitFor(() => { + expect(onParametersChange).toHaveBeenCalledWith({ quality: "auto" }); + }); + }); + it("should render all parameters that have values", async () => { (global.fetch as ReturnType).mockResolvedValueOnce({ ok: true, diff --git a/src/components/nodes/GenerateImageNode.tsx b/src/components/nodes/GenerateImageNode.tsx index 58a62e4c..68ca7cf9 100644 --- a/src/components/nodes/GenerateImageNode.tsx +++ b/src/components/nodes/GenerateImageNode.tsx @@ -51,6 +51,16 @@ const EXTENDED_ASPECT_RATIOS: AspectRatio[] = ["1:1", "1:4", "1:8", "2:3", "3:2" // Resolutions per model (nano-banana-pro: 1K-4K, nano-banana-2: 512-4K) const RESOLUTIONS_PRO: Resolution[] = ["1K", "2K", "4K"]; const RESOLUTIONS_NB2: Resolution[] = ["512", "1K", "2K", "4K"]; +const COMPOSER_PRIORITY_IMAGE_PARAMETERS = [ + "size", + "image_size", + "output_size", + "dimensions", + "aspectRatio", + "aspect_ratio", + "resolution", + "imageSize", +]; // Hardcoded Gemini image models (always available) const GEMINI_IMAGE_MODELS: { value: ModelType; label: string }[] = [ @@ -708,6 +718,7 @@ export function GenerateImageNode({ id, data, selected }: NodeProps )} diff --git a/src/components/nodes/ModelParameters.tsx b/src/components/nodes/ModelParameters.tsx index 33772627..0ae47fd9 100644 --- a/src/components/nodes/ModelParameters.tsx +++ b/src/components/nodes/ModelParameters.tsx @@ -62,6 +62,7 @@ interface ModelParametersProps { onParametersChange: (parameters: Record) => void; onExpandChange?: (expanded: boolean, parameterCount: number) => void; onInputsLoaded?: (inputs: ModelInputDef[]) => void; + hiddenParameterNames?: string[]; } /** @@ -76,6 +77,7 @@ function ModelParametersInner({ onParametersChange, onExpandChange, onInputsLoaded, + hiddenParameterNames, }: ModelParametersProps) { const { t } = useI18n(); const [schema, setSchema] = useState([]); @@ -153,28 +155,57 @@ function ModelParametersInner({ fetchSchema(); }, [modelId, provider, replicateApiKey, falApiKey, kieApiKey, wavespeedApiKey, onInputsLoaded]); - // Pre-populate schema defaults into parameters + const hiddenParameterSet = useMemo( + () => new Set(hiddenParameterNames ?? []), + [hiddenParameterNames] + ); + const visibleSchema = useMemo( + () => schema.filter((param) => !hiddenParameterSet.has(param.name)), + [hiddenParameterSet, schema] + ); + + // Remove parameters controlled by a higher-priority surface, such as the bottom composer. useEffect(() => { - if (schema.length === 0) return; + if (hiddenParameterSet.size === 0) return; + const nextParameters = { ...parameters }; + let changed = false; + for (const name of hiddenParameterSet) { + if (Object.prototype.hasOwnProperty.call(nextParameters, name)) { + delete nextParameters[name]; + changed = true; + } + } + if (changed) { + onParametersChange(nextParameters); + } + }, [hiddenParameterSet, onParametersChange, parameters]); + + // Pre-populate visible schema defaults into parameters + useEffect(() => { + if (visibleSchema.length === 0) return; + const baseParameters = { ...parameters }; + for (const name of hiddenParameterSet) { + delete baseParameters[name]; + } const defaults: Record = {}; let hasNewDefaults = false; - for (const param of schema) { - if (param.default !== undefined && parameters[param.name] === undefined) { + for (const param of visibleSchema) { + if (param.default !== undefined && baseParameters[param.name] === undefined) { defaults[param.name] = param.default; hasNewDefaults = true; } } if (hasNewDefaults) { - onParametersChange({ ...parameters, ...defaults }); + onParametersChange({ ...baseParameters, ...defaults }); } - }, [schema, parameters, onParametersChange]); + }, [visibleSchema, parameters, onParametersChange]); // Notify parent to resize node when schema loads useEffect(() => { - if (schema.length > 0 && onExpandChange) { - onExpandChange(true, schema.length); + if (visibleSchema.length > 0 && onExpandChange) { + onExpandChange(true, visibleSchema.length); } - }, [schema, onExpandChange]); + }, [visibleSchema, onExpandChange]); const handleParameterChange = useCallback( (name: string, value: unknown) => { @@ -194,7 +225,7 @@ function ModelParametersInner({ ); const sortedSchema = useMemo(() => { - return [...schema].sort((a, b) => { + return [...visibleSchema].sort((a, b) => { // Sort order: dropdowns first, then numbers, then strings, then checkboxes last const typeOrder = (p: ModelParameter) => { if (p.enum && p.enum.length > 0) return 0; // dropdowns first @@ -204,7 +235,7 @@ function ModelParametersInner({ }; return typeOrder(a) - typeOrder(b); }); - }, [schema]); + }, [visibleSchema]); const useGrid = sortedSchema.length > 4; const gridRef = useRef(null); @@ -240,7 +271,7 @@ function ModelParametersInner({ } // Don't render if no schema available and not loading - if (!isLoading && schema.length === 0 && !error) { + if (!isLoading && visibleSchema.length === 0 && !error) { return null; } @@ -250,7 +281,7 @@ function ModelParametersInner({ {error} ) : isLoading ? ( {t("node.loadingParameters")} - ) : schema.length === 0 ? ( + ) : visibleSchema.length === 0 ? ( {t("node.noParameters")} ) : (