From dcd21746d2691d54c05b28098a50265ea70245f4 Mon Sep 17 00:00:00 2001 From: jiajia Date: Thu, 16 Apr 2026 18:46:53 +0800 Subject: [PATCH] Keep Gemini image edits role-aware on the server path The server now resolves source, identity, and style artifacts separately, emits role-labeled Gemini parts, and logs the effective request shape so deployed debugging can confirm which references reached upstream. Constraint: The test deployment must stay on gemini-3.1-flash-image-preview while gaining better multi-image editing semantics Rejected: Change the default model to gemini-3-pro-image-preview | test-server parity required keeping flash as default Confidence: high Scope-risk: moderate Reversibility: clean Directive: Preserve the source/identity/style logging until the upstream gateway exposes equivalent structured traces Tested: go test ./internal/server Tested: image img2img against http://101.42.99.35:18080/v1 (job_5af212df2c27) Not-tested: Non-Gemini multi-image providers still fall back to the first reference only --- internal/server/newapi.go | 41 ++++++- internal/server/newapi_image_test.go | 79 ++++++++++++- internal/server/server.go | 165 ++++++++++++++++++++++----- 3 files changed, 247 insertions(+), 38 deletions(-) diff --git a/internal/server/newapi.go b/internal/server/newapi.go index 249397d..473cba9 100644 --- a/internal/server/newapi.go +++ b/internal/server/newapi.go @@ -12,6 +12,7 @@ import ( _ "image/jpeg" _ "image/png" "io" + "log" "mime/multipart" "net/http" "net/textproto" @@ -138,6 +139,7 @@ type upstreamModel struct { } type imageEditReference struct { + Role string Filename string ContentType string Content []byte @@ -479,18 +481,21 @@ func (c *newAPIClient) generateImageRefs(ctx context.Context, token, modelID str return decodeImageGenerationResponse(resp.StatusCode, respBody, modelID) } -func (c *newAPIClient) generateEditedImageRefs(ctx context.Context, token, modelID string, input map[string]any, ref imageEditReference) ([]resultRef, map[string]any, error) { +func (c *newAPIClient) generateEditedImageRefs(ctx context.Context, token, modelID string, input map[string]any, refs []imageEditReference) ([]resultRef, map[string]any, error) { if !c.enabled() { return nil, nil, errors.New("PopiNewAPI base URL is not configured") } + if len(refs) == 0 { + return nil, nil, errors.New("reference image content is required") + } if useGeminiImageGenerateContent(modelID) { - return c.generateGeminiImageRefs(ctx, token, modelID, input, []imageEditReference{ref}) + return c.generateGeminiImageRefs(ctx, token, modelID, input, refs) } if useSeedreamImageGenerations(modelID) { - return c.generateSeedreamImageRefs(ctx, token, modelID, input, []imageEditReference{ref}) + return c.generateSeedreamImageRefs(ctx, token, modelID, input, refs) } if useMiniMaxImageGenerations(modelID) { - return c.generateMiniMaxImageRefs(ctx, token, modelID, input, []imageEditReference{ref}) + return c.generateMiniMaxImageRefs(ctx, token, modelID, input, refs) } token = c.authorizedImageEditToken(token) if token == "" { @@ -501,6 +506,7 @@ func (c *newAPIClient) generateEditedImageRefs(ctx context.Context, token, model if prompt == "" { return nil, nil, errors.New("prompt is required") } + ref := refs[0] if len(ref.Content) == 0 { return nil, nil, errors.New("reference image content is required") } @@ -735,10 +741,13 @@ func (c *newAPIClient) generateGeminiImageRefs(ctx context.Context, token, model parts := make([]map[string]any, 0, 1+len(refs)) parts = append(parts, map[string]any{"text": prompt}) - for _, ref := range refs { + for index, ref := range refs { if len(ref.Content) == 0 { return nil, nil, errors.New("reference image content is required") } + if label := geminiReferenceLabel(ref, index); label != "" { + parts = append(parts, map[string]any{"text": label}) + } contentType := strings.TrimSpace(ref.ContentType) if contentType == "" { contentType = http.DetectContentType(ref.Content) @@ -770,6 +779,13 @@ func (c *newAPIClient) generateGeminiImageRefs(ctx context.Context, token, model if len(imageConfig) > 0 { generationConfig["imageConfig"] = imageConfig } + log.Printf( + "popiartServer: Gemini image request model=%s parts=%d refs=%d image_config=%v", + modelID, + len(parts), + len(refs), + imageConfig, + ) body, err := json.Marshal(payload) if err != nil { @@ -899,6 +915,21 @@ func (c *newAPIClient) submitImageToVideoTask(ctx context.Context, token, modelI return taskID, nil } +func geminiReferenceLabel(ref imageEditReference, index int) string { + switch strings.ToLower(strings.TrimSpace(ref.Role)) { + case "source": + return fmt.Sprintf("Image %d is the source scene. Keep the scene layout, camera framing, main action, and spatial relationships from this image.", index+1) + case "identity": + return fmt.Sprintf("Image %d is the identity reference. Keep the character face, hair, accessories, and recognizability from this image.", index+1) + case "style": + return fmt.Sprintf("Image %d is the style reference. Apply only the visual style, palette, texture, and illustration treatment from this image. Do not change the character identity because of this image.", index+1) + case "reference": + return fmt.Sprintf("Image %d is an additional reference. Use it only for supporting details explicitly requested in the prompt.", index+1) + default: + return "" + } +} + func (c *newAPIClient) submitImageToVideoTaskByURL(ctx context.Context, token, modelID string, input map[string]any, ref imageEditReference, prompt string) (string, error) { payload := map[string]any{ "model": modelID, diff --git a/internal/server/newapi_image_test.go b/internal/server/newapi_image_test.go index f857b7b..57b2e11 100644 --- a/internal/server/newapi_image_test.go +++ b/internal/server/newapi_image_test.go @@ -52,11 +52,11 @@ func TestGenerateGeminiImageRefsUsesGenerateContentEndpoint(t *testing.T) { refs, usage, err := client.generateEditedImageRefs(context.Background(), "sk-test", "gemini-3-pro-image-preview", map[string]any{ "prompt": "edit this image", "size": "1024x1536", - }, imageEditReference{ + }, []imageEditReference{{ Filename: "source.png", ContentType: "image/png", Content: refBytes, - }) + }}) if err != nil { t.Fatalf("generateEditedImageRefs: %v", err) } @@ -185,12 +185,12 @@ func TestGenerateEditedImageRefsUsesSeedreamImagesGenerationsEndpoint(t *testing refs, usage, err := client.generateEditedImageRefs(context.Background(), "sk-test", "seedream-4-5-251128", map[string]any{ "prompt": "edit this image into a dusk scene", "size": "2K", - }, imageEditReference{ + }, []imageEditReference{{ Filename: "source.jpg", ContentType: "image/jpeg", Content: []byte("binary-image"), URL: "https://example.com/reference.jpg", - }) + }}) if err != nil { t.Fatalf("generateEditedImageRefs: %v", err) } @@ -347,9 +347,9 @@ func TestGenerateMiniMaxEditedImageRefsSendsStandardImageField(t *testing.T) { "prompt": "turn it into watercolor", "size": "832x1248", "response_format": "url", - }, imageEditReference{ + }, []imageEditReference{{ URL: "https://example.com/reference.jpg", - }) + }}) if err != nil { t.Fatalf("generateEditedImageRefs: %v", err) } @@ -360,3 +360,70 @@ func TestGenerateMiniMaxEditedImageRefsSendsStandardImageField(t *testing.T) { t.Fatalf("expected size passthrough for minimax mapping, got %#v", gotBody["size"]) } } + +func TestGenerateGeminiImageRefsAnnotatesMultiImageRoles(t *testing.T) { + const png1x1 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO7Z7xkAAAAASUVORK5CYII=" + + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode request body: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "candidates": [{ + "content": { + "parts": [{ + "inlineData": { + "mimeType": "image/png", + "data": "` + png1x1 + `" + } + }] + } + }] + }`)) + })) + defer srv.Close() + + refBytes, err := base64.StdEncoding.DecodeString(png1x1) + if err != nil { + t.Fatalf("decode fixture: %v", err) + } + client := &newAPIClient{ + baseURL: strings.TrimRight(srv.URL, "/"), + httpClient: srv.Client(), + } + + _, _, err = client.generateGeminiImageRefs(context.Background(), "sk-test", "gemini-3-pro-image-preview", map[string]any{ + "prompt": "Replace the person in Image 1 with the character from Image 2 and apply the style from Image 3.", + }, []imageEditReference{ + {Role: "source", Filename: "source.png", ContentType: "image/png", Content: refBytes}, + {Role: "identity", Filename: "identity.png", ContentType: "image/png", Content: refBytes}, + {Role: "style", Filename: "style.png", ContentType: "image/png", Content: refBytes}, + }) + if err != nil { + t.Fatalf("generateGeminiImageRefs: %v", err) + } + + contents, ok := gotBody["contents"].([]any) + if !ok || len(contents) != 1 { + t.Fatalf("unexpected contents: %#v", gotBody["contents"]) + } + content, ok := contents[0].(map[string]any) + if !ok { + t.Fatalf("unexpected content entry: %#v", contents[0]) + } + parts, ok := content["parts"].([]any) + if !ok || len(parts) != 7 { + t.Fatalf("expected prompt plus three labeled images, got %#v", content["parts"]) + } + if got := parts[1].(map[string]any)["text"]; got != "Image 1 is the source scene. Keep the scene layout, camera framing, main action, and spatial relationships from this image." { + t.Fatalf("unexpected source label: %#v", got) + } + if got := parts[3].(map[string]any)["text"]; got != "Image 2 is the identity reference. Keep the character face, hair, accessories, and recognizability from this image." { + t.Fatalf("unexpected identity label: %#v", got) + } + if got := parts[5].(map[string]any)["text"]; got != "Image 3 is the style reference. Apply only the visual style, palette, texture, and illustration treatment from this image. Do not change the character identity because of this image." { + t.Fatalf("unexpected style label: %#v", got) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index deff402..508cc96 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -1159,7 +1159,7 @@ func (s *Server) executeImageToImageJob(record *job) { defer cancel() input := buildImageToImageInput(record) - ref, err := s.resolveImageToImageReference(ctx, record, input) + imageRefs, err := s.resolveImageToImageReferences(ctx, record, input) if err != nil { if repoErr := s.store.failJob(record.JobID, "REFERENCE_IMAGE_RESOLUTION_FAILED", err.Error(), map[string]any{ "skill_id": record.SkillID, @@ -1169,15 +1169,23 @@ func (s *Server) executeImageToImageJob(record *job) { } return } + log.Printf( + "popiartServer: image job %s using model=%s refs=%s preserve_composition=%t strength=%v", + record.JobID, + modelID, + summarizeImageEditReferenceRoles(imageRefs), + boolValue(input["preserve_composition"]), + input["strength"], + ) var ( - refs []resultRef - usage map[string]any + resultRefs []resultRef + usage map[string]any ) if useMiniMaxImageGenerations(modelID) { - refs, usage, err = s.newapi.generateMiniMaxImageRefs(ctx, record.UpstreamKey, modelID, input, []imageEditReference{ref}) + resultRefs, usage, err = s.newapi.generateMiniMaxImageRefs(ctx, record.UpstreamKey, modelID, input, imageRefs) } else { - refs, usage, err = s.newapi.generateEditedImageRefs(ctx, record.UpstreamKey, modelID, input, ref) + resultRefs, usage, err = s.newapi.generateEditedImageRefs(ctx, record.UpstreamKey, modelID, input, imageRefs) } if err != nil { if repoErr := s.store.failJob(record.JobID, "MODEL_REQUEST_FAILED", err.Error(), map[string]any{ @@ -1189,7 +1197,7 @@ func (s *Server) executeImageToImageJob(record *job) { return } - refs, err = s.persistResultRefs(ctx, record, refs) + resultRefs, err = s.persistResultRefs(ctx, record, resultRefs) if err != nil { if repoErr := s.store.failJob(record.JobID, "MEDIA_PERSIST_FAILED", "failed to persist generated media", map[string]any{ "details": err.Error(), @@ -1201,7 +1209,7 @@ func (s *Server) executeImageToImageJob(record *job) { return } - if err := s.store.completeJobWithResults(record.JobID, refs, usage); err != nil { + if err := s.store.completeJobWithResults(record.JobID, resultRefs, usage); err != nil { log.Printf("popiartServer: complete job %s failed: %v", record.JobID, err) if failErr := s.store.failJob(record.JobID, "RESULT_REF_PERSIST_FAILED", "failed to persist job result refs", map[string]any{ "details": err.Error(), @@ -1394,33 +1402,45 @@ func inferRouteKeyForModel(modelID string, input map[string]any) string { return "image.text2image" } -func (s *Server) resolveImageToImageReference(ctx context.Context, record *job, input map[string]any) (imageEditReference, error) { +func (s *Server) resolveImageToImageReferences(ctx context.Context, record *job, input map[string]any) ([]imageEditReference, error) { if record == nil { - return imageEditReference{}, fmt.Errorf("job record is required") + return nil, fmt.Errorf("job record is required") } if artifactID := strings.TrimSpace(stringValue(input["source_artifact_id"])); artifactID != "" { - item, ref, exists, err := s.store.artifactRef(record.UserID, artifactID) + sourceRef, err := s.resolveArtifactImageReference(ctx, record, artifactID, "source") if err != nil { - return imageEditReference{}, err + return nil, err } - if !exists { - return imageEditReference{}, fmt.Errorf("source artifact not found: %s", artifactID) + refs := []imageEditReference{sourceRef} + seen := map[string]struct{}{artifactID: {}} + var appendArtifactRefs = func(role string, values []string) error { + for _, value := range values { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, exists := seen[value]; exists { + continue + } + ref, err := s.resolveArtifactImageReference(ctx, record, value, role) + if err != nil { + return err + } + seen[value] = struct{}{} + refs = append(refs, ref) + } + return nil } - contentType, _, reader, err := s.newapi.openResultRef(ctx, record.UpstreamKey, ref) - if err != nil { - return imageEditReference{}, err + if err := appendArtifactRefs("identity", stringSliceValue(input["identity_reference_artifact_ids"])); err != nil { + return nil, err } - defer reader.Close() - content, err := io.ReadAll(reader) - if err != nil { - return imageEditReference{}, err + if err := appendArtifactRefs("style", stringSliceValue(input["style_reference_artifact_ids"])); err != nil { + return nil, err + } + if err := appendArtifactRefs("reference", stringSliceValue(input["reference_artifact_ids"])); err != nil { + return nil, err } - return imageEditReference{ - Filename: item.Filename, - ContentType: defaultString(contentType, item.ContentType), - Content: content, - URL: item.URL, - }, nil + return refs, nil } refURL := strings.TrimSpace(stringValue( @@ -1432,9 +1452,25 @@ func (s *Server) resolveImageToImageReference(ctx context.Context, record *job, refURL = defaultAliceReferenceURL } if refURL == "" { + return nil, fmt.Errorf("reference image is required") + } + ref, err := s.downloadReferenceImage(ctx, refURL) + if err != nil { + return nil, err + } + ref.Role = "source" + return []imageEditReference{ref}, nil +} + +func (s *Server) resolveImageToImageReference(ctx context.Context, record *job, input map[string]any) (imageEditReference, error) { + refs, err := s.resolveImageToImageReferences(ctx, record, input) + if err != nil { + return imageEditReference{}, err + } + if len(refs) == 0 { return imageEditReference{}, fmt.Errorf("reference image is required") } - return s.downloadReferenceImage(ctx, refURL) + return refs[0], nil } func (s *Server) resolveVideoReferences(ctx context.Context, record *job, input map[string]any) ([]imageEditReference, error) { @@ -1651,6 +1687,32 @@ func buildAliceShowcasePrompt(input map[string]any, scenePrompt string) string { return strings.Join(parts, ", ") } +func (s *Server) resolveArtifactImageReference(ctx context.Context, record *job, artifactID, role string) (imageEditReference, error) { + item, ref, exists, err := s.store.artifactRef(record.UserID, artifactID) + if err != nil { + return imageEditReference{}, err + } + if !exists { + return imageEditReference{}, fmt.Errorf("%s artifact not found: %s", defaultString(role, "reference"), artifactID) + } + contentType, _, reader, err := s.newapi.openResultRef(ctx, record.UpstreamKey, ref) + if err != nil { + return imageEditReference{}, err + } + defer reader.Close() + content, err := io.ReadAll(reader) + if err != nil { + return imageEditReference{}, err + } + return imageEditReference{ + Role: role, + Filename: item.Filename, + ContentType: defaultString(contentType, item.ContentType), + Content: content, + URL: item.URL, + }, nil +} + func buildGenericImageToImagePrompt(input map[string]any, prompt string) string { parts := make([]string, 0, 6) if prompt != "" { @@ -1668,6 +1730,21 @@ func buildGenericImageToImagePrompt(input map[string]any, prompt string) string return strings.Join(parts, ", ") } +func summarizeImageEditReferenceRoles(refs []imageEditReference) string { + if len(refs) == 0 { + return "none" + } + parts := make([]string, 0, len(refs)) + for _, ref := range refs { + role := strings.TrimSpace(ref.Role) + if role == "" { + role = "reference" + } + parts = append(parts, role) + } + return strings.Join(parts, ",") +} + func stringValue(values ...any) string { for _, value := range values { if text, ok := value.(string); ok && strings.TrimSpace(text) != "" { @@ -1677,6 +1754,40 @@ func stringValue(values ...any) string { return "" } +func boolValue(value any) bool { + switch typed := value.(type) { + case bool: + return typed + default: + return false + } +} + +func stringSliceValue(value any) []string { + switch typed := value.(type) { + case []string: + return cleanedStringValues(typed) + case []any: + items := make([]string, 0, len(typed)) + for _, item := range typed { + items = append(items, strings.TrimSpace(fmt.Sprint(item))) + } + return cleanedStringValues(items) + default: + return nil + } +} + +func cleanedStringValues(values []string) []string { + items := make([]string, 0, len(values)) + for _, value := range values { + if trimmed := strings.TrimSpace(value); trimmed != "" { + items = append(items, trimmed) + } + } + return items +} + func buildVideoResultRef(baseURL, taskID, modelID string, task *videoTaskResult) resultRef { urlValue := strings.TrimSpace(task.URL) if urlValue == "" {