From 109012cbbfa05e58a6fdf20a005bc7deae06f6d7 Mon Sep 17 00:00:00 2001 From: jiajia Date: Fri, 22 May 2026 18:17:47 +0800 Subject: [PATCH] Persist browserfs workflow media locally Browser-file-system workflow saves were still bypassing external media storage, so local projects could keep large image, video, and audio data embedded instead of writing refs into the selected browser-backed workspace. The storage path now uses the browser file helpers for image, video, and audio refs and hydrates them back without API calls. Constraint: Keep local browserfs storage on the existing external media storage path instead of adding a new persistence mode. Rejected: Keep browserfs excluded from externalization | it preserves the old large-payload behavior for local projects. Confidence: high Scope-risk: moderate Directive: Do not route browserfs media through server API endpoints; browserfs paths must stay client-local. Tested: npm run test:run Tested: npm run build Tested: git diff --check -- src/store/workflowStore.ts src/utils/browserFileSystem.ts src/utils/mediaStorage.ts src/utils/__tests__/mediaStorage.browserfs.test.ts Not-tested: npm run lint is blocked because next lint is no longer a valid Next 16 command in this repo Not-tested: npx tsc --noEmit reports existing test-file type debt outside this change while next build TypeScript passes --- src/store/workflowStore.ts | 6 +- .../__tests__/mediaStorage.browserfs.test.ts | 204 ++++++++++++++++++ src/utils/browserFileSystem.ts | 48 ++++- src/utils/mediaStorage.ts | 54 +++++ 4 files changed, 304 insertions(+), 8 deletions(-) create mode 100644 src/utils/__tests__/mediaStorage.browserfs.test.ts diff --git a/src/store/workflowStore.ts b/src/store/workflowStore.ts index 59d82505..df989e10 100644 --- a/src/store/workflowStore.ts +++ b/src/store/workflowStore.ts @@ -2689,7 +2689,7 @@ const workflowStoreImpl: StateCreator = (set, get) => ({ const isBrowserPath = isBrowserFileSystemPath(saveDirectoryPath); // If external media storage is enabled, externalize media before saving - if (useExternalImageStorage && !isBrowserPath) { + if (useExternalImageStorage) { workflow = await externalizeWorkflowMedia(workflow, saveDirectoryPath); } @@ -2710,7 +2710,7 @@ const workflowStoreImpl: StateCreator = (set, get) => ({ // If we externalized media, update store nodes with the refs // This prevents duplicate media on subsequent saves - if (useExternalImageStorage && !isBrowserPath && workflow.nodes !== currentNodes) { + if (useExternalImageStorage && workflow.nodes !== currentNodes) { // Merge refs from externalized nodes into current nodes (keeping media data) const nodesWithRefs = currentNodes.map((node, index) => { const externalizedNode = workflow.nodes[index]; @@ -2776,7 +2776,7 @@ const workflowStoreImpl: StateCreator = (set, get) => ({ lastSavedAt: timestamp, hasUnsavedChanges: false, // Update imageRefBasePath to reflect save location - imageRefBasePath: useExternalImageStorage && !isBrowserPath ? saveDirectoryPath : null, + imageRefBasePath: useExternalImageStorage ? saveDirectoryPath : null, }); } diff --git a/src/utils/__tests__/mediaStorage.browserfs.test.ts b/src/utils/__tests__/mediaStorage.browserfs.test.ts new file mode 100644 index 00000000..2279bc03 --- /dev/null +++ b/src/utils/__tests__/mediaStorage.browserfs.test.ts @@ -0,0 +1,204 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { externalizeWorkflowMedia, hydrateWorkflowMedia } from "../mediaStorage"; +import type { WorkflowFile } from "@/store/workflowStore"; + +const browserFsMocks = vi.hoisted(() => ({ + writeBrowserGenerationFile: vi.fn(), + loadBrowserGenerationFile: vi.fn(), +})); + +vi.mock("../browserFileSystem", () => ({ + isBrowserFileSystemPath: (path: string | null | undefined) => + typeof path === "string" && path.startsWith("browserfs://"), + joinBrowserFileSystemPath: (basePath: string, folderName: string) => + `${basePath.replace(/\/+$/, "")}/${encodeURIComponent(folderName)}`, + writeBrowserGenerationFile: browserFsMocks.writeBrowserGenerationFile, + loadBrowserGenerationFile: browserFsMocks.loadBrowserGenerationFile, +})); + +const imageData = "data:image/png;base64,aW1hZ2U="; +const videoData = "data:video/mp4;base64,dmlkZW8="; +const audioData = "data:audio/mpeg;base64,YXVkaW8="; +const workflowPath = "browserfs://root/project"; + +function makeWorkflow(): WorkflowFile { + return { + version: 1, + id: "wf-browserfs", + name: "browserfs media", + nodes: [ + { + id: "imageInput-1", + type: "imageInput", + position: { x: 0, y: 0 }, + data: { + image: imageData, + filename: "input.png", + dimensions: null, + }, + }, + { + id: "generateVideo-1", + type: "generateVideo", + position: { x: 200, y: 0 }, + data: { + inputImages: [imageData], + inputPrompt: "move", + outputVideo: videoData, + outputVideoRef: undefined, + outputVideoRemoteUrl: undefined, + outputVideoStorageStatus: undefined, + selectedVideoHistoryIndex: 0, + videoHistory: [ + { + id: "vid-history-1", + video: videoData, + timestamp: 1, + prompt: "move", + }, + ], + status: "completed", + error: null, + }, + }, + { + id: "generateAudio-1", + type: "generateAudio", + position: { x: 400, y: 0 }, + data: { + inputPrompt: "sound", + outputAudio: audioData, + outputAudioRef: undefined, + selectedAudioHistoryIndex: 0, + audioHistory: [ + { + id: "aud-history-1", + audio: audioData, + timestamp: 1, + prompt: "sound", + }, + ], + status: "completed", + error: null, + }, + }, + ], + edges: [], + edgeStyle: "angular", + } as WorkflowFile; +} + +describe("mediaStorage browserfs support", () => { + beforeEach(() => { + browserFsMocks.writeBrowserGenerationFile.mockReset(); + browserFsMocks.loadBrowserGenerationFile.mockReset(); + browserFsMocks.writeBrowserGenerationFile.mockResolvedValue({ filePath: "browserfs://root/project/media.bin" }); + vi.stubGlobal("fetch", vi.fn(async () => ({ + json: async () => ({ success: true, imageId: "server-ref" }), + }))); + }); + + it("externalizes browserfs media through browser file helpers instead of API fetch", async () => { + const workflow = makeWorkflow(); + + const externalized = await externalizeWorkflowMedia(workflow, workflowPath); + + expect(fetch).not.toHaveBeenCalled(); + expect(browserFsMocks.writeBrowserGenerationFile).toHaveBeenCalledWith( + "browserfs://root/project/inputs", + expect.any(String), + imageData + ); + expect(browserFsMocks.writeBrowserGenerationFile).toHaveBeenCalledWith( + "browserfs://root/project/generations", + "vid-history-1", + videoData + ); + expect(browserFsMocks.writeBrowserGenerationFile).toHaveBeenCalledWith( + "browserfs://root/project/generations", + "aud-history-1", + audioData + ); + + expect(externalized.nodes[0].data).toMatchObject({ + image: null, + imageRef: expect.any(String), + }); + expect(externalized.nodes[1].data).toMatchObject({ + inputImages: [], + inputImageRefs: [expect.any(String)], + outputVideo: null, + outputVideoRef: "vid-history-1", + outputVideoStorageStatus: "localized", + videoHistory: [ + { + id: "vid-history-1", + timestamp: 1, + prompt: "move", + }, + ], + }); + expect(externalized.nodes[2].data).toMatchObject({ + outputAudio: null, + outputAudioRef: "aud-history-1", + }); + }); + + it("hydrates browserfs media refs through browser file helpers instead of API fetch", async () => { + browserFsMocks.loadBrowserGenerationFile.mockImplementation(async (directoryPath: string, mediaId: string) => { + if (directoryPath.endsWith("/inputs") && mediaId === "input-ref") return imageData; + if (directoryPath.endsWith("/generations") && mediaId === "video-ref") return videoData; + if (directoryPath.endsWith("/generations") && mediaId === "audio-ref") return audioData; + return null; + }); + + const workflow: WorkflowFile = { + ...makeWorkflow(), + nodes: [ + { + id: "imageInput-1", + type: "imageInput", + position: { x: 0, y: 0 }, + data: { image: null, imageRef: "input-ref" }, + }, + { + id: "generateVideo-1", + type: "generateVideo", + position: { x: 200, y: 0 }, + data: { + inputImages: [], + inputImageRefs: ["input-ref"], + outputVideo: null, + outputVideoRef: "video-ref", + outputVideoStorageStatus: "localized", + status: "idle", + error: null, + }, + }, + { + id: "generateAudio-1", + type: "generateAudio", + position: { x: 400, y: 0 }, + data: { + inputPrompt: "sound", + outputAudio: null, + outputAudioRef: "audio-ref", + status: "idle", + error: null, + }, + }, + ], + } as WorkflowFile; + + const hydrated = await hydrateWorkflowMedia(workflow, workflowPath); + + expect(fetch).not.toHaveBeenCalled(); + expect(hydrated.nodes[0].data).toMatchObject({ image: imageData }); + expect(hydrated.nodes[1].data).toMatchObject({ + inputImages: [imageData], + outputVideo: videoData, + outputVideoStorageStatus: "localized", + }); + expect(hydrated.nodes[2].data).toMatchObject({ outputAudio: audioData }); + }); +}); diff --git a/src/utils/browserFileSystem.ts b/src/utils/browserFileSystem.ts index 334722e8..d3ce6837 100644 --- a/src/utils/browserFileSystem.ts +++ b/src/utils/browserFileSystem.ts @@ -52,15 +52,49 @@ export function joinBrowserFileSystemPath(basePath: string, folderName: string): return `${basePath.replace(/\/+$/, "")}/${encodeURIComponent(folderName)}`; } -const IMAGE_EXTENSIONS = ["png", "jpg", "jpeg", "webp", "gif"] as const; +const MEDIA_EXTENSIONS = [ + "png", + "jpg", + "jpeg", + "webp", + "gif", + "mp4", + "webm", + "mov", + "mp3", + "wav", + "ogg", + "flac", + "aac", +] as const; function extensionFromMime(mime: string | null | undefined): string { if (mime === "image/jpeg") return "jpg"; if (mime === "image/webp") return "webp"; if (mime === "image/gif") return "gif"; + if (mime === "video/webm") return "webm"; + if (mime === "video/quicktime") return "mov"; + if (mime?.startsWith("video/")) return "mp4"; + if (mime === "audio/wav") return "wav"; + if (mime === "audio/ogg") return "ogg"; + if (mime === "audio/flac") return "flac"; + if (mime === "audio/aac") return "aac"; + if (mime?.startsWith("audio/")) return "mp3"; return "png"; } +function mimeFromExtension(extension: string): string { + if (extension === "jpg" || extension === "jpeg") return "image/jpeg"; + if (extension === "mov") return "video/quicktime"; + if (extension === "mp3") return "audio/mpeg"; + if (extension === "wav") return "audio/wav"; + if (extension === "ogg") return "audio/ogg"; + if (extension === "flac") return "audio/flac"; + if (extension === "aac") return "audio/aac"; + if (extension === "mp4" || extension === "webm") return `video/${extension}`; + return `image/${extension}`; +} + function extensionFromDataUrl(dataUrl: string): string { return extensionFromMime(dataUrl.match(/^data:([^;]+);base64,/)?.[1]); } @@ -75,12 +109,16 @@ async function imageSourceToBlob(src: string): Promise { return response.blob(); } -async function fileToDataUrl(file: File): Promise { +async function fileToDataUrl(file: File, fallbackMime?: string): Promise { + const blob = file.type || !fallbackMime + ? file + : file.slice(0, file.size, fallbackMime); + return new Promise((resolve, reject) => { const reader = new FileReader(); reader.onload = () => resolve(String(reader.result)); reader.onerror = () => reject(reader.error); - reader.readAsDataURL(file); + reader.readAsDataURL(blob); }); } @@ -154,10 +192,10 @@ export async function loadBrowserGenerationFile( const directoryHandle = await resolveDirectoryHandle(directoryPath, false); await ensurePermission(directoryHandle, "readwrite"); - for (const extension of IMAGE_EXTENSIONS) { + for (const extension of MEDIA_EXTENSIONS) { try { const fileHandle = await directoryHandle.getFileHandle(`${imageId}.${extension}`); - return fileToDataUrl(await fileHandle.getFile()); + return fileToDataUrl(await fileHandle.getFile(), mimeFromExtension(extension)); } catch { continue; } diff --git a/src/utils/mediaStorage.ts b/src/utils/mediaStorage.ts index ec1c96aa..40266610 100644 --- a/src/utils/mediaStorage.ts +++ b/src/utils/mediaStorage.ts @@ -1,6 +1,12 @@ import { WorkflowNode, WorkflowNodeData } from "@/types"; import { WorkflowFile } from "@/store/workflowStore"; import crypto from "crypto"; +import { + isBrowserFileSystemPath, + joinBrowserFileSystemPath, + loadBrowserGenerationFile, + writeBrowserGenerationFile, +} from "./browserFileSystem"; /** * Fetch with timeout support using AbortController @@ -672,6 +678,16 @@ async function saveImageAndGetId( const imageId = existingId || generateImageId(); const savePromise = (async () => { + if (isBrowserFileSystemPath(workflowPath)) { + await writeBrowserGenerationFile( + joinBrowserFileSystemPath(workflowPath, folder), + imageId, + imageData + ); + savedImageIds.set(hash, imageId); + return imageId; + } + const response = await fetchWithTimeout( "/api/workflow-images", { @@ -738,6 +754,16 @@ async function saveVideoAndGetRef( const videoId = existingId || generateMediaId("vid"); const savePromise = (async () => { + if (isBrowserFileSystemPath(workflowPath)) { + await writeBrowserGenerationFile( + joinBrowserFileSystemPath(workflowPath, "generations"), + videoId, + videoData + ); + savedMediaIds.set(hash, videoId); + return videoId; + } + const response = await fetchWithTimeout( "/api/save-generation", { @@ -806,6 +832,16 @@ async function saveAudioAndGetRef( const audioId = existingId || generateMediaId("aud"); const savePromise = (async () => { + if (isBrowserFileSystemPath(workflowPath)) { + await writeBrowserGenerationFile( + joinBrowserFileSystemPath(workflowPath, "generations"), + audioId, + audioData + ); + savedMediaIds.set(hash, audioId); + return audioId; + } + const response = await fetchWithTimeout( "/api/save-generation", { @@ -1242,6 +1278,24 @@ async function loadMediaById( return loadedMedia.get(mediaId)!; } + if (isBrowserFileSystemPath(workflowPath)) { + const folderNames = mediaType === "image" ? ["inputs", "generations"] : ["generations"]; + + for (const folderName of folderNames) { + const mediaData = await loadBrowserGenerationFile( + joinBrowserFileSystemPath(workflowPath, folderName), + mediaId + ); + if (mediaData) { + loadedMedia.set(mediaId, mediaData); + return mediaData; + } + } + + console.log(`${mediaType} not found: ${mediaId}`); + return ""; + } + let response: Response; if (mediaType === "image") {