"use client"; import { useCallback, useEffect, useMemo, useRef, useState, type DragEvent as ReactDragEvent, type ReactNode, } from "react"; import { useReactFlow } from "@xyflow/react"; import { Modal } from "antd"; import { useI18n } from "@/i18n"; import { useWorkflowStore } from "@/store/workflowStore"; import { useModelStore, type PopiModelDetail } from "@/store/modelStore"; import { useGenerationPreferenceStore } from "@/store/generationPreferenceStore"; import { getNodeOutputMediaType } from "@/utils/nodeHandles"; import { getGenerationNodeSizeForAspectRatio } from "@/utils/nodeDimensions"; import { deriveComposerContext, deriveComposerContextForNodeId, type ComposerCapability, type ComposerContext, } from "@/utils/composerNodeDescriptor"; import { buildInitialGenerationNodeData, buildSubmittableGenerationConfig, filterTaskRoutingParameters, readAudioGenerationConfig, readImageGenerationConfig, readVideoGenerationConfig, type GenerationNodeConfig, type PopiserverTaskPayloadMedia, } from "@/utils/generationConfig"; import { buildComposerReferenceMaterials, getReferenceMaterialKey, orderReferenceMaterials, readReferenceSubjectList, syncPromptReferenceMaterials, type ComposerReferenceInput, type ComposerReferenceMaterial, type ComposerReferenceMaterialType, } from "@/utils/composerReferenceSubjects"; import { buildPopiserverPriceQuoteRequest, getQuotedPointAmount, type PopiserverPriceQuoteResponse, } from "@/utils/composerPricing"; import { buildComposerPricingContext, buildFinalComposerPrompt, } from "@/utils/composerPricingContext"; import { selectedModelCapabilities } from "@/utils/selectedModel"; import { ComposerPromptMaterials, type ComposerConnectedTextCard, } from "@/components/composer/ComposerPromptMaterials"; import { ComposerModelConfig } from "@/components/composer/ComposerModelConfig"; import { ComposerProcessBody } from "@/components/composer/ComposerProcessBody"; import type { ComposerMentionReferenceItem } from "@/components/composer/ComposerMentionInput"; import type { EaseCurveNodeData, GenerateAudioNodeData, GenerateVideoNodeData, ImageInputNodeData, LLMGenerateNodeData, LLMModelType, NanoBananaNodeData, NodeType, SelectedModel, SmartTextNodeData, VideoStitchNodeData, WorkflowEdge, WorkflowNode, WorkflowNodeData, } from "@/types"; import type { ModelParameter } from "@/lib/providers/types"; import type { ConnectedInputs } from "@/store/utils/connectedInputs"; type ComposerDraft = GenerationNodeConfig; export type ComposerGenerationConfig = Omit; type ComposerConfig = ComposerGenerationConfig; type ComposerPreferenceOptions = { nodeSize?: { width: number; height: number } | null; }; type PricingInputsSnapshot = { config: ComposerConfig; prompt: string; activeModelDetail: PopiModelDetail | undefined; modelSchema: ModelParameter[]; extensionFields?: unknown; connectedInputs: ConnectedInputs | null; connectedVideoDurationSeconds: number; popiserverPricingMedia: PopiserverTaskPayloadMedia; isHidden: boolean; }; const COMPOSER_WIDTH = 630; const PROMPT_LINE_HEIGHT_PX = 24; const PROMPT_MIN_HEIGHT_PX = PROMPT_LINE_HEIGHT_PX * 2; const PROMPT_MAX_HEIGHT_PX = PROMPT_LINE_HEIGHT_PX * 4; const EMPTY_EXTRA_TASK_PARAMS: Record = {}; const EMPTY_SELECTED_MODEL: SelectedModel = { provider: "popiserver", modelId: "", displayName: "", capabilities: [], }; const DEFAULT_PROMPT_TEXT_MODEL: SelectedModel = { provider: "popiserver", modelId: "", displayName: "", capabilities: ["text-to-text"], }; function getComposerModelPopupContainer(): HTMLElement { return document.body; } function finiteNumberFromUnknown(value: unknown): number | null { if (typeof value === "number" && Number.isFinite(value)) return value; if (typeof value === "string") { const parsed = Number.parseFloat(value.trim()); return Number.isFinite(parsed) ? parsed : null; } return null; } function getConnectedTextItems(inputs: { text: string | null; textItems: string[] } | null | undefined): string[] { if (!inputs) return []; const items = inputs.textItems.length > 0 ? inputs.textItems : inputs.text ? [inputs.text] : []; return items.map((item) => item.trim()).filter(Boolean); } function hasAnyCapability(model: SelectedModel, capabilities: string[]): boolean { return Boolean(selectedModelCapabilities(model).some((capability) => capabilities.includes(capability))); } function modelSupportsCapability(model: SelectedModel, capability: ComposerCapability): boolean { if (capability === "all") return true; if (capability === "image") return hasAnyCapability(model, ["text-to-image", "image-to-image"]); if (capability === "video") return hasAnyCapability(model, ["text-to-video", "image-to-video", "video-to-video", "audio-to-video"]); if (capability === "3d") return hasAnyCapability(model, ["text-to-3d", "image-to-3d"]); if (capability === "llm") return hasAnyCapability(model, ["text-to-text"]); return hasAnyCapability(model, ["text-to-audio"]); } function storedModelFitsCapability(model: SelectedModel, capability: ComposerCapability): boolean { if (capability === "all") return true; if (selectedModelCapabilities(model).length === 0) return true; return modelSupportsCapability(model, capability); } function inferCapabilityFromModel(model: SelectedModel): Exclude | null { if (modelSupportsCapability(model, "video")) return "video"; if (modelSupportsCapability(model, "3d")) return "3d"; if (modelSupportsCapability(model, "audio")) return "audio"; if (modelSupportsCapability(model, "image")) return "image"; if (modelSupportsCapability(model, "llm")) return "llm"; return null; } function inferNodeTypeFromModel(model: SelectedModel): NodeType | null { const capability = inferCapabilityFromModel(model); if (capability === "video") return "smartVideo"; if (capability === "3d") return "generate3d"; if (capability === "audio") return "smartAudio"; if (capability === "image") return "smartImage"; if (capability === "llm") return "smartText"; return null; } function preferenceTypeForCapability(capability: ComposerCapability): "image" | "video" | "audio" | null { if (capability === "image") return "image"; if (capability === "video") return "video"; if (capability === "audio") return "audio"; return null; } function formatPointAmount(amount: number): string { if (Number.isInteger(amount)) return String(amount); return amount.toFixed(2).replace(/\.?0+$/, ""); } function createEmptyDraft(): ComposerDraft { return { prompt: "", selectedModel: EMPTY_SELECTED_MODEL, inputImages: [], batchSize: 1, audioVoiceId: undefined, referenceSubjectList: [], parameters: {}, extraTaskParams: {}, }; } function configFromDraft(draft: ComposerDraft): ComposerConfig { const { prompt: _prompt, ...config } = draft; return config; } function buildComposerGenerationConfig(config: ComposerConfig, prompt: string): GenerationNodeConfig { return { ...config, prompt, }; } function referenceSubjectsToMaterials(subjects: ComposerConfig["referenceSubjectList"]): ComposerReferenceMaterial[] { return readReferenceSubjectList(subjects).map((subject, index) => ({ id: index + 1, type: subject.type, url: subject.url, name: subject.name.startsWith("@") ? subject.name : `@${subject.name}`, sourceNodeId: subject.id, ...(subject.thumbnailUrl ? { thumbnailUrl: subject.thumbnailUrl } : {}), ...(subject.duration ? { duration: subject.duration } : {}), })); } function createPromptDraft(node: WorkflowNode): ComposerDraft { const data = node.data as { prompt?: unknown }; return { ...createEmptyDraft(), prompt: typeof data.prompt === "string" ? data.prompt : "", selectedModel: DEFAULT_PROMPT_TEXT_MODEL, }; } function createSelectedModelFromLlmData(data: LLMGenerateNodeData): SelectedModel { if (data.selectedModel?.provider === "popiserver") return data.selectedModel; const model = data.model || ""; return { provider: "popiserver", modelId: model, displayName: model, capabilities: ["text-to-text"], }; } function readDraftFromContext(context: ComposerContext): ComposerDraft { if (context.mode === "empty-create") return createEmptyDraft(); if (context.mode === "unsupported-node" && context.node?.type === "prompt") return createPromptDraft(context.node); if (!context.node) return createEmptyDraft(); if (context.node.type === "smartImage" || context.node.type === "nanoBanana") { return readImageGenerationConfig(context.node.data as NanoBananaNodeData, EMPTY_SELECTED_MODEL); } if (context.node.type === "smartVideo" || context.node.type === "generateVideo") { return readVideoGenerationConfig(context.node.data as GenerateVideoNodeData, EMPTY_SELECTED_MODEL); } if (context.node.type === "smartAudio" || context.node.type === "generateAudio") { return readAudioGenerationConfig(context.node.data as GenerateAudioNodeData, EMPTY_SELECTED_MODEL); } if (context.node.type === "llmGenerate" || context.node.type === "smartText") { const data = context.node.data as LLMGenerateNodeData; return { ...createEmptyDraft(), prompt: data.inputPrompt ?? "", selectedModel: createSelectedModelFromLlmData(data), parameters: {}, }; } return createEmptyDraft(); } function contextIdentity(context: ComposerContext): string { return `${context.mode}:${context.node?.id ?? "none"}:${context.nodeType ?? "none"}:${context.selectedCount}`; } function getConcreteGenerationNodeType(nodeType: NodeType | null): NodeType | null { if (nodeType === "smartImage") return "nanoBanana"; if (nodeType === "smartVideo") return "generateVideo"; if (nodeType === "smartAudio") return "generateAudio"; return null; } function buildInitialDataForNode(nodeType: NodeType, draft: ComposerDraft): Partial { if ( nodeType === "smartImage" || nodeType === "smartVideo" || nodeType === "smartAudio" || nodeType === "nanoBanana" || nodeType === "generateVideo" || nodeType === "generate3d" || nodeType === "generateAudio" ) { return buildInitialGenerationNodeData(nodeType, draft); } if (nodeType === "llmGenerate" || nodeType === "smartText") { return { ...(nodeType === "smartText" ? { manualLocked: false, manualText: "" } : {}), inputPrompt: draft.prompt, inputImages: [], inputVideos: [], outputText: null, provider: "popiserver", model: draft.selectedModel.modelId as LLMModelType, selectedModel: { ...draft.selectedModel, capabilities: draft.selectedModel.capabilities ?? ["text-to-text"], }, status: "idle", error: null, } as Partial; } return {}; } function readWorkflowNodeSize(node: WorkflowNode | null | undefined): { width: number; height: number } | null { if (!node) return null; const width = typeof node.measured?.width === "number" ? node.measured.width : typeof node.style?.width === "number" ? node.style.width : typeof node.width === "number" ? node.width : null; const height = typeof node.measured?.height === "number" ? node.measured.height : typeof node.style?.height === "number" ? node.style.height : typeof node.height === "number" ? node.height : null; return width && height && Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0 ? { width, height } : null; } function buildComposerPersistParameters(node: WorkflowNode, config: ComposerConfig): ComposerConfig["parameters"] { void node; return filterTaskRoutingParameters(config.parameters); } function buildComposerConfigPersistPatch(node: WorkflowNode, config: GenerationNodeConfig): Partial { const nodeType = node.type; const parameters = buildComposerPersistParameters(node, config); if (nodeType === "smartImage" || nodeType === "nanoBanana") { return { inputPrompt: config.prompt.trim(), selectedModel: config.selectedModel, batchSize: config.batchSize, parameters, extraTaskParams: config.extraTaskParams ?? {}, } as Partial; } if (nodeType === "smartVideo" || nodeType === "generateVideo") { return { inputPrompt: config.prompt.trim(), selectedModel: config.selectedModel, parameters, extraTaskParams: config.extraTaskParams ?? {}, subType: config.subType, } as Partial; } if (nodeType === "smartAudio" || nodeType === "generateAudio") { return { inputPrompt: config.prompt.trim(), selectedModel: config.selectedModel, audioVoiceId: config.audioVoiceId, parameters, extraTaskParams: config.extraTaskParams ?? {}, } as Partial; } if (nodeType === "llmGenerate" || nodeType === "smartText") { return { inputPrompt: config.prompt, selectedModel: config.selectedModel, model: config.selectedModel.modelId as LLMModelType, } as Partial; } return {}; } function getConnectedEdgesByType( targetNodeId: string, nodes: WorkflowNode[], edges: WorkflowEdge[], type: ComposerReferenceMaterialType ): Array<{ id: string; sourceNodeId: string }> { return edges .filter((edge) => { if (edge.target !== targetNodeId) return false; const sourceNode = nodes.find((node) => node.id === edge.source); return sourceNode ? getNodeOutputMediaType(sourceNode) === type : false; }) .map((edge) => ({ id: edge.id, sourceNodeId: edge.source })); } function getConnectedEdgeIdsByType( targetNodeId: string, nodes: WorkflowNode[], edges: WorkflowEdge[], type: ComposerReferenceMaterialType | "text" ): string[] { return edges .filter((edge) => { if (edge.target !== targetNodeId) return false; const sourceNode = nodes.find((node) => node.id === edge.source); return sourceNode ? getNodeOutputMediaType(sourceNode) === type : false; }) .map((edge) => edge.id); } function getReferenceImageThumbnailUrl(sourceNode: WorkflowNode | undefined): string | undefined { const previewImg = (sourceNode?.data as { previewImg?: unknown } | undefined)?.previewImg; return typeof previewImg === "string" && previewImg.length > 0 ? previewImg : undefined; } function getReferenceVideoDurationSeconds(videos: ComposerReferenceInput[]): number { const seenUrls = new Set(); const totalDuration = videos.reduce((sum, video) => { if (seenUrls.has(video.url)) return sum; seenUrls.add(video.url); const duration = finiteNumberFromUnknown(video.duration); return duration !== null && duration > 0 ? sum + duration : sum; }, 0); return Math.ceil(totalDuration); } function uniqueReferenceVideoInputs(videos: ComposerReferenceInput[]): ComposerReferenceInput[] { const seenUrls = new Set(); return videos.filter((video) => { if (seenUrls.has(video.url)) return false; seenUrls.add(video.url); return true; }); } function readComposerAspectRatio(config: ComposerConfig): string | undefined { const value = config.parameters?.ratio ?? config.parameters?.aspectRatio ?? config.parameters?.aspect_ratio; return typeof value === "string" && value.length > 0 ? value : undefined; } function IconButton({ label, children, onClick }: { label: string; children: ReactNode; onClick?: () => void }) { return ( ); } function ComposerFrame({ children }: { children: ReactNode }) { return (
{children}
); } interface GenerationComposerProps { targetNodeId: string; embedded?: boolean; isHidden?: boolean; onHiddenPersisted?: () => void; } interface ComposerEditorProps { context: ComposerContext; contextKey: string; initialDraft: ComposerDraft; nodes: WorkflowNode[]; edges: WorkflowEdge[]; dimmedNodeIds: Set; isCurrentNodeRunning: boolean; embedded: boolean; isHidden: boolean; onRun: (context: ComposerContext, prompt: string, config: ComposerConfig) => Promise; onCancelRunning: () => void; onRemoveEdge: (edgeId: string) => void; onApplyNodeSize: (nodeId: string, aspectRatio: unknown) => void; onModelNeedsDetail: (modelId: string) => Promise; onPersistConfig: (context: ComposerContext, config: GenerationNodeConfig) => void; onHiddenPersisted?: () => void; } export function GenerationComposer({ targetNodeId, embedded = false, isHidden = false, onHiddenPersisted }: GenerationComposerProps) { const { t } = useI18n(); const nodes = useWorkflowStore((state) => state.nodes); const edges = useWorkflowStore((state) => state.edges); const dimmedNodeIds = useWorkflowStore((state) => state.dimmedNodeIds); const runningNodeIds = useWorkflowStore((state) => state.runningNodeIds); const addNode = useWorkflowStore((state) => state.addNode); const convertNodeType = useWorkflowStore((state) => state.convertNodeType); const updateNodeData = useWorkflowStore((state) => state.updateNodeData); const regenerateNode = useWorkflowStore((state) => state.regenerateNode); const cancelNodeTask = useWorkflowStore((state) => state.cancelNodeTask); const removeEdge = useWorkflowStore((state) => state.removeEdge); const selectSingleNode = useWorkflowStore((state) => state.selectSingleNode); const onNodesChange = useWorkflowStore((state) => state.onNodesChange); const fetchModelDetail = useModelStore((state) => state.fetchModelDetail); const updateImagePreference = useGenerationPreferenceStore((state) => state.updateImagePreference); const updateVideoPreference = useGenerationPreferenceStore((state) => state.updateVideoPreference); const updateAudioPreference = useGenerationPreferenceStore((state) => state.updateAudioPreference); const { screenToFlowPosition } = useReactFlow(); const context = useMemo( () => { const selectedNodes = nodes.filter((node) => node.selected); if (!isHidden && selectedNodes.length > 1) { return deriveComposerContext(nodes, edges); } const targetNode = nodes.find((node) => node.id === targetNodeId); const selectedNode = selectedNodes[0]; const effectiveTargetNodeId = !isHidden && selectedNodes.length === 1 && selectedNode && selectedNode.id !== targetNodeId ? selectedNode.id : targetNode?.id; return effectiveTargetNodeId ? deriveComposerContextForNodeId(nodes, edges, effectiveTargetNodeId) : deriveComposerContextForNodeId( nodes, edges, selectedNode?.id ?? targetNodeId ); }, [edges, isHidden, nodes, targetNodeId] ); const contextKey = contextIdentity(context); const initialConfig = useMemo(() => readDraftFromContext(context), [contextKey, context]); const activeModelId = initialConfig.selectedModel.modelId; const activeComposerNodeId = (context.mode === "node-edit" || context.mode === "process-node" || context.mode === "unsupported-node") ? context.node?.id ?? null : null; const isCurrentNodeRunning = activeComposerNodeId ? runningNodeIds.has(activeComposerNodeId) : false; useEffect(() => { if (isHidden || !embedded || !activeModelId) return; void fetchModelDetail(activeModelId); }, [activeModelId, embedded, fetchModelDetail, isHidden]); const applyNodeSize = useCallback((nodeId: string, aspectRatioValue: unknown) => { const nextSize = getGenerationNodeSizeForAspectRatio(aspectRatioValue); if (!nextSize) return; onNodesChange([{ id: nodeId, type: "dimensions", dimensions: nextSize, setAttributes: true }]); }, [onNodesChange]); const buildTargetPosition = useCallback((size?: { width: number; height: number }) => { const width = size?.width ?? 300; const height = size?.height ?? 280; return screenToFlowPosition({ x: window.innerWidth / 2 - width / 2, y: window.innerHeight / 2 - height / 2, }); }, [screenToFlowPosition]); const rememberGenerationPreference = useCallback(( capability: ComposerCapability, nextConfig: ComposerConfig, options: ComposerPreferenceOptions = {} ) => { const preferenceType = preferenceTypeForCapability(capability); const patch = { ...nextConfig, model: nextConfig.selectedModel, ...(options.nodeSize ? { nodeSize: options.nodeSize } : {}), }; if (preferenceType === "image") { updateImagePreference(patch); return; } if (preferenceType === "video") { updateVideoPreference(patch); return; } if (preferenceType === "audio") { updateAudioPreference(patch); } }, [updateAudioPreference, updateImagePreference, updateVideoPreference]); const runComposer = useCallback(async (runContext: ComposerContext, prompt: string, config: ComposerConfig) => { const generationConfig = buildComposerGenerationConfig(config, prompt); let submitNode = runContext.node; if (runContext.mode === "node-edit" && runContext.node) { submitNode = useWorkflowStore.getState().nodes.find((node) => node.id === runContext.node?.id) ?? runContext.node; } const connectedInputs = (runContext.mode === "node-edit" || runContext.mode === "process-node") && submitNode ? useWorkflowStore.getState().getConnectedInputs(submitNode.id) : null; const finalPrompt = buildFinalComposerPrompt(prompt, connectedInputs); const submitNodeType = runContext.mode === "empty-create" ? inferNodeTypeFromModel(config.selectedModel) : runContext.nodeType; const submitModelDetail = generationConfig.selectedModel.modelId ? await fetchModelDetail(generationConfig.selectedModel.modelId) : null; const submitConfig = buildSubmittableGenerationConfig({ ...generationConfig, prompt: finalPrompt, }, submitModelDetail); const submitCapability = runContext.mode === "empty-create" ? inferCapabilityFromModel(submitConfig.selectedModel) : runContext.capability; const submitAspectRatio = readComposerAspectRatio(submitConfig); const submitNodeSize = getGenerationNodeSizeForAspectRatio(submitAspectRatio) ?? readWorkflowNodeSize(submitNode); if (submitCapability) rememberGenerationPreference(submitCapability, submitConfig, { nodeSize: submitNodeSize }); if (runContext.mode === "unsupported-node" && runContext.node?.type === "imageInput") { const imageData = runContext.node.data as ImageInputNodeData; if (!imageData.image) return; const selectedModel = modelSupportsCapability(submitConfig.selectedModel, "image") ? submitConfig.selectedModel : EMPTY_SELECTED_MODEL; if (!selectedModel.modelId) return; const nextNodeId = addNode( "smartImage", buildTargetPosition(submitNodeSize ?? undefined), buildInitialDataForNode("smartImage", { ...submitConfig, selectedModel, inputImages: [imageData.image], }), submitNodeSize ?? undefined ); selectSingleNode(nextNodeId); await regenerateNode(nextNodeId); return; } if (runContext.mode === "empty-create") { if (!submitNodeType) return; const nextNodeId = addNode( submitNodeType, buildTargetPosition(submitNodeSize ?? undefined), buildInitialDataForNode(submitNodeType, submitConfig), submitNodeSize ?? undefined ); selectSingleNode(nextNodeId); await regenerateNode(nextNodeId); return; } if (runContext.mode === "node-edit" && submitNode) { const concreteNodeType = getConcreteGenerationNodeType(submitNode.type); if (concreteNodeType) { convertNodeType(submitNode.id, concreteNodeType, { ...buildInitialDataForNode(concreteNodeType, submitConfig), inputPrompt: prompt.trim(), referenceSubjectList: generationConfig.referenceSubjectList, }); applyNodeSize(submitNode.id, submitAspectRatio); await regenerateNode(submitNode.id); return; } updateNodeData(submitNode.id, { ...buildInitialDataForNode(submitNode.type, submitConfig), inputPrompt: prompt.trim(), referenceSubjectList: generationConfig.referenceSubjectList, }); await regenerateNode(submitNode.id); return; } if ( runContext.mode === "process-node" && (runContext.node?.type === "videoStitch" || runContext.node?.type === "easeCurve") ) { await regenerateNode(runContext.node.id); } }, [ addNode, applyNodeSize, buildTargetPosition, convertNodeType, fetchModelDetail, rememberGenerationPreference, regenerateNode, selectSingleNode, updateNodeData, ]); const persistComposerConfig = useCallback((persistContext: ComposerContext, config: GenerationNodeConfig) => { const currentNode = persistContext.node ? useWorkflowStore.getState().nodes.find((node) => node.id === persistContext.node?.id) ?? persistContext.node : null; rememberGenerationPreference(persistContext.capability, config, { nodeSize: readWorkflowNodeSize(currentNode) }); if (!currentNode || persistContext.mode !== "node-edit") return; const patch = buildComposerConfigPersistPatch(currentNode, config); if (Object.keys(patch).length === 0) return; updateNodeData(currentNode.id, patch); }, [rememberGenerationPreference, updateNodeData]); const cancelRunning = useCallback(() => { if (!activeComposerNodeId) return; Modal.confirm({ title: t("composer.cancelTaskTitle"), content: t("composer.cancelTaskConfirm"), okText: t("composer.cancelTaskOk"), cancelText: t("composer.cancelTaskContinue"), okButtonProps: { danger: true }, onOk: () => cancelNodeTask(activeComposerNodeId), }); }, [activeComposerNodeId, cancelNodeTask, t]); if (context.mode === "multi-select" || (context.mode === "unsupported-node" && !context.node)) { return null; } return ( fetchModelDetail(modelId)} onPersistConfig={persistComposerConfig} onHiddenPersisted={onHiddenPersisted} /> ); } function ComposerEditor({ context, contextKey, initialDraft, nodes, edges, dimmedNodeIds, isCurrentNodeRunning, embedded, isHidden, onRun, onCancelRunning, onRemoveEdge, onApplyNodeSize, onModelNeedsDetail, onPersistConfig, onHiddenPersisted, }: ComposerEditorProps) { void dimmedNodeIds; void embedded; const { t } = useI18n(); const promptInputRef = useRef(null); const referenceAudioPreviewRef = useRef(null); const referenceMaterialsRef = useRef([]); const [config, setConfig] = useState(() => configFromDraft(initialDraft)); const [prompt, setPrompt] = useState(initialDraft.prompt); const [isComposerCollapsed, setIsComposerCollapsed] = useState(false); const [playingReferenceAudioKey, setPlayingReferenceAudioKey] = useState(null); const [priceState, setPriceState] = useState<{ amount: number | null; isLoading: boolean }>({ amount: null, isLoading: false, }); const priceRequestIdRef = useRef(0); const suppressAutoPriceRefreshRef = useRef(false); const modelChangeCompletionTimerRef = useRef(null); const priceRefreshTimerRef = useRef(null); const pricingInputsRef = useRef({ config, prompt, activeModelDetail: undefined, modelSchema: [], extensionFields: undefined, connectedInputs: null, connectedVideoDurationSeconds: 0, popiserverPricingMedia: {}, isHidden, }); const reorderNodeReferenceSubjectList = useWorkflowStore((state) => state.reorderNodeReferenceSubjectList); const updateNodeExtraTaskParams = useWorkflowStore((state) => state.updateNodeExtraTaskParams); const updateNodeData = useWorkflowStore((state) => state.updateNodeData); const buildGenerationConfigForContext = useCallback(( targetContext: ComposerContext, nextConfig: ComposerConfig = config, nextPrompt: string = prompt ) => { if (targetContext.mode !== "node-edit" || !targetContext.node) { return buildComposerGenerationConfig(nextConfig, nextPrompt); } const latestNode = useWorkflowStore.getState().nodes.find((node) => node.id === targetContext.node?.id); const latestSubjects = latestNode ? readReferenceSubjectList((latestNode.data as { referenceSubjectList?: unknown }).referenceSubjectList) : readReferenceSubjectList((targetContext.node.data as { referenceSubjectList?: unknown }).referenceSubjectList); return buildComposerGenerationConfig({ ...nextConfig, referenceSubjectList: latestSubjects, }, nextPrompt); }, [config, prompt]); const persistCurrentComposerConfig = useCallback(() => { onPersistConfig(context, buildGenerationConfigForContext(context)); }, [buildGenerationConfigForContext, context, onPersistConfig]); const connectedInputs = useMemo(() => { if ((context.mode !== "node-edit" && context.mode !== "process-node") || !context.node) return null; return useWorkflowStore.getState().getConnectedInputs(context.node.id); }, [context, nodes, edges]); const connectedTextItems = useMemo(() => getConnectedTextItems(connectedInputs), [connectedInputs]); const connectedTextCards = useMemo(() => { if (!context.node) return connectedTextItems.map((text) => ({ text })); const textEdgeIds = getConnectedEdgeIdsByType(context.node.id, nodes, edges, "text"); return connectedTextItems.map((text, index) => ({ text, ...(textEdgeIds[index] ? { edgeId: textEdgeIds[index] } : {}), })); }, [connectedTextItems, context.node, edges, nodes]); const currentModelId = config.selectedModel.modelId; const activeModelDetail = useModelStore((state) => currentModelId ? state.detailsById[currentModelId] : undefined); const modelSchema = activeModelDetail?.parameters ?? []; const extensionFields = activeModelDetail?.extensionFields; const activeCapability = context.mode === "empty-create" ? inferCapabilityFromModel(config.selectedModel) ?? "image" : context.capability; const isProcessNode = context.mode === "process-node" && (context.node?.type === "videoStitch" || context.node?.type === "easeCurve"); const modelIsValid = context.mode === "empty-create" ? Boolean(inferNodeTypeFromModel(config.selectedModel)) : storedModelFitsCapability(config.selectedModel, context.capability); const canChooseModel = context.mode === "empty-create" || context.mode === "node-edit"; const connectedReferenceImages = context.mode === "node-edit" ? connectedInputs?.images ?? [] : []; const connectedReferenceAudio = context.mode === "node-edit" ? connectedInputs?.audio ?? [] : []; const referenceImages = connectedReferenceImages.length > 0 ? connectedReferenceImages : config.inputImages; const estimateReferenceVideoInputs = useMemo(() => { const videoItems = context.mode === "node-edit" ? connectedInputs?.videoItems ?? [] : []; const videoEdges = context.mode === "node-edit" && context.node ? getConnectedEdgesByType(context.node.id, nodes, edges, "video") : []; return uniqueReferenceVideoInputs(videoItems.map((video, index) => ({ url: video.url, ...(video.duration && video.duration > 0 ? { duration: video.duration } : {}), ...(videoEdges[index]?.id ? { edgeId: videoEdges[index].id } : {}), ...(videoEdges[index]?.sourceNodeId ? { sourceNodeId: videoEdges[index].sourceNodeId } : {}), }))); }, [connectedInputs?.videoItems, context.mode, context.node, edges, nodes]); const referenceImageInputs = useMemo(() => { const imageEdges = context.mode === "node-edit" && context.node && connectedReferenceImages.length > 0 ? getConnectedEdgesByType(context.node.id, nodes, edges, "image") : []; return referenceImages.map((url, index) => { const edge = imageEdges[index]; const sourceNodeId = edge?.sourceNodeId; const thumbnailUrl = sourceNodeId ? getReferenceImageThumbnailUrl(nodes.find((node) => node.id === sourceNodeId)) : undefined; return { url, ...(thumbnailUrl ? { thumbnailUrl } : {}), ...(edge?.id ? { edgeId: edge.id } : {}), ...(sourceNodeId ? { sourceNodeId } : {}), }; }); }, [connectedReferenceImages.length, context.mode, context.node, edges, nodes, referenceImages]); const referenceAudioInputs = useMemo(() => { const audioEdges = context.mode === "node-edit" && context.node ? getConnectedEdgesByType(context.node.id, nodes, edges, "audio") : []; return connectedReferenceAudio.map((url, index) => ({ url, ...(audioEdges[index]?.id ? { edgeId: audioEdges[index].id } : {}), ...(audioEdges[index]?.sourceNodeId ? { sourceNodeId: audioEdges[index].sourceNodeId } : {}), })); }, [connectedReferenceAudio, context.mode, context.node, edges, nodes]); const baseReferenceMaterials = useMemo(() => buildComposerReferenceMaterials( referenceImageInputs, estimateReferenceVideoInputs, referenceAudioInputs, ), [estimateReferenceVideoInputs, referenceAudioInputs, referenceImageInputs]); const liveReferenceSubjectOrder = useMemo(() => { if (context.mode !== "node-edit" || !context.node) return []; const nodeData = context.node.data as { referenceSubjectList?: unknown }; return readReferenceSubjectList(nodeData.referenceSubjectList).map((item) => item.id); }, [context.mode, context.node]); const liveReferenceSubjectOrderKey = useMemo( () => liveReferenceSubjectOrder.join("|"), [liveReferenceSubjectOrder] ); const referenceMaterials = useMemo( () => orderReferenceMaterials(baseReferenceMaterials, liveReferenceSubjectOrder), [baseReferenceMaterials, liveReferenceSubjectOrderKey] ); const referenceMaterialsKey = useMemo( () => referenceMaterials .map((material) => `${getReferenceMaterialKey(material)}:${material.type}:${material.url}:${material.name}:${material.thumbnailUrl ?? ""}:${material.duration ?? ""}`) .join("|"), [referenceMaterials] ); useEffect(() => { const previousMaterials = referenceMaterialsRef.current.length > 0 ? referenceMaterialsRef.current : referenceSubjectsToMaterials(config.referenceSubjectList); referenceMaterialsRef.current = referenceMaterials; const nextPrompt = syncPromptReferenceMaterials(prompt, previousMaterials, referenceMaterials); if (nextPrompt === prompt) return; setPrompt(nextPrompt); }, [config.referenceSubjectList, prompt, referenceMaterialsKey, referenceMaterials]); const connectedVideoDurationSeconds = useMemo( () => getReferenceVideoDurationSeconds(estimateReferenceVideoInputs), [estimateReferenceVideoInputs] ); const popiserverPricingMedia = useMemo(() => ({ images: referenceImages, videos: estimateReferenceVideoInputs.map((input) => input.url), voices: referenceAudioInputs.map((input) => input.url), }), [estimateReferenceVideoInputs, referenceAudioInputs, referenceImages]); pricingInputsRef.current = { config, prompt, activeModelDetail, modelSchema, extensionFields, connectedInputs, connectedVideoDurationSeconds, popiserverPricingMedia, isHidden, }; const refreshPrice = useCallback(async ( nextConfig: ComposerConfig, nextPrompt = prompt, detailOverride?: PopiModelDetail | null, inputs: PricingInputsSnapshot = pricingInputsRef.current ) => { const requestId = priceRequestIdRef.current + 1; priceRequestIdRef.current = requestId; if (!nextConfig.selectedModel.modelId) { setPriceState({ amount: null, isLoading: false }); return; } setPriceState({ amount: null, isLoading: true }); const detail = detailOverride !== undefined ? detailOverride : await onModelNeedsDetail(nextConfig.selectedModel.modelId); if (priceRequestIdRef.current !== requestId) return; const generationConfig = buildComposerGenerationConfig(nextConfig, nextPrompt); const normalizedConfig = buildSubmittableGenerationConfig( generationConfig, detail ?? inputs.activeModelDetail ?? { parameters: inputs.modelSchema, extensionFields: inputs.extensionFields } ); const quoteRequest = buildPopiserverPriceQuoteRequest(buildComposerPricingContext({ config: normalizedConfig, prompt: nextPrompt, capability: context.capability, connectedInputs: inputs.connectedInputs, modelDetail: detail, media: inputs.popiserverPricingMedia, inputVideoDurationSeconds: inputs.connectedVideoDurationSeconds, })); if (!quoteRequest) { setPriceState({ amount: null, isLoading: false }); return; } try { const response = await fetch("/api/points/quote", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(quoteRequest), }); const data = response.ok ? await response.json() as PopiserverPriceQuoteResponse : null; const amount = data ? getQuotedPointAmount(data) : null; if (priceRequestIdRef.current === requestId) { setPriceState({ amount, isLoading: false }); } } catch { if (priceRequestIdRef.current === requestId) setPriceState({ amount: null, isLoading: false }); } }, [ context.capability, context.mode, context.node, onModelNeedsDetail, prompt, ]); const handleConfigChange = useCallback((config: ComposerConfig) => { setConfig(config); onPersistConfig(context, buildGenerationConfigForContext(context, config, prompt)); }, [buildGenerationConfigForContext, context, onPersistConfig, prompt]); const handleExtraTaskParamsChange = useCallback((values: Record) => { if (context.node) updateNodeExtraTaskParams(context.node.id, values); const nextConfig = { ...config, extraTaskParams: values, }; setConfig(nextConfig); }, [config, context.node, updateNodeExtraTaskParams]); const schedulePriceRefresh = useCallback(() => { if (priceRefreshTimerRef.current !== null) { window.clearTimeout(priceRefreshTimerRef.current); } priceRefreshTimerRef.current = window.setTimeout(() => { priceRefreshTimerRef.current = null; const latest = pricingInputsRef.current; if (latest.isHidden) return; if (suppressAutoPriceRefreshRef.current) return; if (!latest.config.selectedModel.modelId) return; void refreshPrice(latest.config, latest.prompt, latest.activeModelDetail, latest); }, 200); }, [refreshPrice]); const clearScheduledPriceRefresh = useCallback(() => { if (priceRefreshTimerRef.current === null) return; window.clearTimeout(priceRefreshTimerRef.current); priceRefreshTimerRef.current = null; }, []); useEffect(() => { if (isHidden) { clearScheduledPriceRefresh(); return; } if (suppressAutoPriceRefreshRef.current) return; if (!config.selectedModel.modelId) return; schedulePriceRefresh(); }, [ clearScheduledPriceRefresh, config.audioVoiceId, config.extraTaskParams, config.batchSize, config.parameters, config.referenceSubjectList, config.selectedModel.modelId, connectedVideoDurationSeconds, isHidden, popiserverPricingMedia, prompt, referenceMaterialsKey, schedulePriceRefresh, ]); const hasPromptForSubmit = prompt.trim().length > 0 || connectedTextItems.length > 0; const submitNodeType = context.mode === "empty-create" ? inferNodeTypeFromModel(config.selectedModel) : context.nodeType; const hasVideoSubTypes = activeCapability === "video" && Array.isArray(config.selectedModel.subTypes) && config.selectedModel.subTypes.length > 0; const hasReference = referenceMaterials.length > 0; const hasRequiredPromptOrReference = submitNodeType === "smartAudio" || submitNodeType === "generateAudio" || submitNodeType === "llmGenerate" || submitNodeType === "smartText" ? hasPromptForSubmit : hasPromptForSubmit || hasReference; const videoStitchData = context.node?.type === "videoStitch" ? context.node.data as VideoStitchNodeData : null; const easeCurveData = context.node?.type === "easeCurve" ? context.node.data as EaseCurveNodeData : null; const canRunVideoStitch = Boolean(videoStitchData && !isCurrentNodeRunning && videoStitchData.encoderSupported !== false && (connectedInputs?.videos.length ?? 0) >= 2); const canRunEaseCurve = Boolean(easeCurveData && !isCurrentNodeRunning && easeCurveData.encoderSupported !== false && (connectedInputs?.videos.length ?? 0) >= 1); const canSubmit = canRunVideoStitch || canRunEaseCurve || (!isCurrentNodeRunning && (context.mode === "empty-create" || context.mode === "node-edit") && hasRequiredPromptOrReference && modelIsValid) || (context.mode === "unsupported-node" && context.node?.type === "imageInput" && modelSupportsCapability(config.selectedModel, "image")); const canClickSubmit = isCurrentNodeRunning || canSubmit; const isPriceLoading = priceState.isLoading; const dynamicEstimatedPointAmount = priceState.amount; const modelPointCostLabel = dynamicEstimatedPointAmount !== null ? t("composer.pointCost", { count: formatPointAmount(dynamicEstimatedPointAmount) }) : null; const submitButtonLabel = isCurrentNodeRunning ? "Cancel task" : t("composer.generate"); const priceLabel = modelPointCostLabel ?? (isPriceLoading ? "..." : null); const hasComposerMediaHeader = isProcessNode || connectedTextItems.length > 0 || referenceMaterials.length > 0 || hasVideoSubTypes; const nodeData = context.node?.data as (NanoBananaNodeData | GenerateVideoNodeData | GenerateAudioNodeData | VideoStitchNodeData | EaseCurveNodeData | LLMGenerateNodeData | SmartTextNodeData | undefined); const errorMessage = nodeData?.status === "error" ? nodeData.error : null; const stopReferenceAudioPreview = useCallback(() => { const audio = referenceAudioPreviewRef.current; if (audio) { audio.pause(); audio.currentTime = 0; audio.onended = null; referenceAudioPreviewRef.current = null; } setPlayingReferenceAudioKey(null); }, []); const previousContextRef = useRef<{ key: string; context: ComposerContext } | null>({ key: contextKey, context, }); useEffect(() => { if (isHidden) return; const previous = previousContextRef.current; if (!previous) { previousContextRef.current = { key: contextKey, context }; return; } if (previous.key === contextKey) { previousContextRef.current = { key: contextKey, context }; return; } onPersistConfig(previous.context, buildGenerationConfigForContext(previous.context)); setConfig(configFromDraft(initialDraft)); setPrompt(initialDraft.prompt); referenceMaterialsRef.current = []; previousContextRef.current = { key: contextKey, context }; // The current local draft belongs to the previous composer identity. // eslint-disable-next-line react-hooks/exhaustive-deps }, [contextKey]); useEffect(() => { if (isHidden) { persistCurrentComposerConfig(); stopReferenceAudioPreview(); onHiddenPersisted?.(); return; } setConfig(configFromDraft(initialDraft)); setPrompt(initialDraft.prompt); referenceMaterialsRef.current = []; previousContextRef.current = { key: contextKey, context }; // Hidden composer state is discarded; reopening hydrates from the store. // eslint-disable-next-line react-hooks/exhaustive-deps }, [isHidden]); useEffect(() => () => stopReferenceAudioPreview(), [stopReferenceAudioPreview]); const handleReferenceAudioPreviewToggle = useCallback((material: ComposerReferenceMaterial) => { const materialKey = getReferenceMaterialKey(material); if (playingReferenceAudioKey === materialKey) { stopReferenceAudioPreview(); return; } stopReferenceAudioPreview(); const audio = new Audio(material.url); audio.onended = () => { if (referenceAudioPreviewRef.current === audio) { referenceAudioPreviewRef.current = null; setPlayingReferenceAudioKey(null); } }; referenceAudioPreviewRef.current = audio; void audio.play() .then(() => { if (referenceAudioPreviewRef.current === audio) setPlayingReferenceAudioKey(materialKey); }) .catch(() => { if (referenceAudioPreviewRef.current === audio) { referenceAudioPreviewRef.current = null; setPlayingReferenceAudioKey(null); } }); }, [playingReferenceAudioKey, stopReferenceAudioPreview]); const handleReferenceMaterialDrop = useCallback((event: ReactDragEvent, targetMaterial: ComposerReferenceMaterial) => { const sourceKey = event.dataTransfer.getData("application/x-popiart-reference-material"); const sourceMaterial = referenceMaterials.find((material) => getReferenceMaterialKey(material) === sourceKey); if (!sourceMaterial || getReferenceMaterialKey(sourceMaterial) === getReferenceMaterialKey(targetMaterial)) return; const nextMaterials = [...referenceMaterials]; const sourceIndex = nextMaterials.findIndex((material) => getReferenceMaterialKey(material) === getReferenceMaterialKey(sourceMaterial)); const targetIndex = nextMaterials.findIndex((material) => getReferenceMaterialKey(material) === getReferenceMaterialKey(targetMaterial)); if (sourceIndex < 0 || targetIndex < 0) return; const [removed] = nextMaterials.splice(sourceIndex, 1); nextMaterials.splice(targetIndex, 0, removed); const orderedNextMaterials = orderReferenceMaterials(nextMaterials, nextMaterials.map(getReferenceMaterialKey)); referenceMaterialsRef.current = orderedNextMaterials; const nextPrompt = syncPromptReferenceMaterials(prompt, referenceMaterials, orderedNextMaterials); if (nextPrompt !== prompt) { setPrompt(nextPrompt); } if (context.mode === "node-edit" && context.node) { reorderNodeReferenceSubjectList(context.node.id, sourceKey, getReferenceMaterialKey(targetMaterial)); } }, [context.mode, context.node, prompt, referenceMaterials, reorderNodeReferenceSubjectList]); const handleModelChangeStart = useCallback(() => { suppressAutoPriceRefreshRef.current = true; }, []); const handleModelChangeComplete = useCallback((nextConfig: ComposerConfig, detail: PopiModelDetail | null) => { if (modelChangeCompletionTimerRef.current !== null) { window.clearTimeout(modelChangeCompletionTimerRef.current); } modelChangeCompletionTimerRef.current = window.setTimeout(() => { modelChangeCompletionTimerRef.current = null; suppressAutoPriceRefreshRef.current = false; const latest = pricingInputsRef.current; pricingInputsRef.current = { ...latest, config: nextConfig, activeModelDetail: detail ?? latest.activeModelDetail, }; if (latest.isHidden || !nextConfig.selectedModel.modelId) return; schedulePriceRefresh(); }, 0); }, [schedulePriceRefresh]); useEffect(() => () => { if (modelChangeCompletionTimerRef.current !== null) { window.clearTimeout(modelChangeCompletionTimerRef.current); } if (priceRefreshTimerRef.current !== null) { window.clearTimeout(priceRefreshTimerRef.current); } }, []); const processBody = isProcessNode ? ( ) : undefined; const mentionReferenceItems = useMemo(() => { if (referenceMaterials.length === 0) { return [{ key: "canvas-assets", label: t("composer.canvasAssets"), value: `@${t("composer.canvasAssets")} `, }]; } return referenceMaterials.map((material) => ({ key: `${material.type}-${material.id}-${material.url.slice(0, 24)}`, label: material.name.replace(/^@/, ""), value: `${material.name} `, image: material.type === "image" ? material.thumbnailUrl ?? material.url : undefined, sourceNodeId: material.sourceNodeId, referenceType: material.type, url: material.url, thumbnailUrl: material.thumbnailUrl, duration: material.duration, })); }, [referenceMaterials, t]); const runAction = useCallback(() => { if (isCurrentNodeRunning) { onCancelRunning(); return; } const generationConfig = buildGenerationConfigForContext(context); onPersistConfig(context, generationConfig); const { prompt: _prompt, ...nextConfig } = generationConfig; void onRun(context, prompt, nextConfig); }, [ buildGenerationConfigForContext, context, isCurrentNodeRunning, onCancelRunning, onPersistConfig, onRun, prompt, ]); return ( <>
event.stopPropagation()} onMouseDown={(event) => event.stopPropagation()} onClick={(event) => event.stopPropagation()} >
{ if (!isComposerCollapsed) persistCurrentComposerConfig(); setIsComposerCollapsed((value) => !value); }} > {isComposerCollapsed ? : }
{isComposerCollapsed ? ( ) : ( <> onRemoveEdge(edgeId)} onReferenceMaterialDragStart={(event, material) => { event.dataTransfer.effectAllowed = "copyMove"; event.dataTransfer.setData("application/x-popiart-reference-material", getReferenceMaterialKey(material)); }} onReferenceMaterialDrop={handleReferenceMaterialDrop} onRemoveReferenceMaterial={(material) => material.edgeId && onRemoveEdge(material.edgeId)} onToggleReferenceAudio={handleReferenceAudioPreviewToggle} onStopReferenceAudio={stopReferenceAudioPreview} onVideoSubTypeChange={(subType) => { if (context.mode !== "node-edit" || !context.node) return; const nextConfig = { ...config, subType, parameters: filterTaskRoutingParameters(config.parameters), }; setConfig(nextConfig); updateNodeData(context.node.id, { subType, parameters: nextConfig.parameters, } as Partial); }} promptInputRef={promptInputRef} promptValue={prompt} promptReadOnly={false} promptMinHeightPx={PROMPT_MIN_HEIGHT_PX} promptMaxHeightPx={PROMPT_MAX_HEIGHT_PX} referenceItems={mentionReferenceItems} onDraftPatch={(patch) => { const nextPrompt = typeof patch.prompt === "string" ? patch.prompt : prompt; if (typeof patch.prompt === "string") { setPrompt(patch.prompt); } if (patch.parameters) { const nextParameters = patch.parameters; const nextConfig = { ...config, parameters: nextParameters, }; setConfig(nextConfig); } }} onSubmitShortcut={runAction} bodyContent={processBody} /> {errorMessage && (
{errorMessage}
)}
{isProcessNode ? (
{context.node?.type === "easeCurve" ? ((connectedInputs?.videos.length ?? 0) >= 1 ? t("composer.easeCurveReady") : t("composer.easeCurveNoVideo")) : ((connectedInputs?.videos.length ?? 0) >= 2 ? t("composer.videoStitchReady") : t("composer.videoStitchNeedTwo"))}
) : ( { if (context.mode === "node-edit" && context.node) onApplyNodeSize(context.node.id, value); }} getPopupContainer={getComposerModelPopupContainer} selectModelPlaceholder={t("composer.selectModel")} showLlmModelConfig={false} llmModelConfigContent={null} llmModelConfigSummary="" /> )}
{priceLabel ? (
{priceLabel}
) : null}
)}
); }