From 15f8ec6c87f081617882d638b774a2800f886009 Mon Sep 17 00:00:00 2001 From: shrimbly Date: Sat, 28 Mar 2026 22:32:26 +1300 Subject: [PATCH] feat: discover workflows in nested subdirectories Add recursive directory scanning (up to 3 levels deep) to the list-workflows API so projects organized into subdirectories are visible in the workflow browser. Skips hidden dirs and node_modules. Returns relativePath field displayed as the subtitle in the UI. Co-Authored-By: Claude Opus 4.6 --- .../list-workflows/__tests__/route.test.ts | 254 +++++++++++------- src/app/api/list-workflows/route.ts | 51 +++- .../quickstart/WorkflowBrowserView.tsx | 3 +- 3 files changed, 211 insertions(+), 97 deletions(-) diff --git a/src/app/api/list-workflows/__tests__/route.test.ts b/src/app/api/list-workflows/__tests__/route.test.ts index b40ba110..4b942ef4 100644 --- a/src/app/api/list-workflows/__tests__/route.test.ts +++ b/src/app/api/list-workflows/__tests__/route.test.ts @@ -29,26 +29,58 @@ function createRequest(path?: string): NextRequest { return new NextRequest(url); } -function createFileHandle(content: string) { - const buf = Buffer.from(content); +function makeWorkflowHeader(name?: string): string { + if (name) { + return `{"version":1,"name":"${name}","nodes":[]}`; + } + return '{"version":1,"nodes":[{"id":"1"}],"edges":[]}'; +} + +function makeBufReader(content: string) { + const srcBuf = Buffer.from(content); return { - read: vi.fn().mockResolvedValue({ bytesRead: buf.length }), + read: vi.fn().mockImplementation(async (buf: Buffer) => { + srcBuf.copy(buf, 0, 0, Math.min(srcBuf.length, buf.length)); + return { bytesRead: Math.min(srcBuf.length, buf.length) }; + }), close: vi.fn().mockResolvedValue(undefined), - // Store the buffer for inspection — the actual route allocates its own buffer - _content: content, }; } +// Directory entry helpers +function dirEntry(name: string) { + return { name, isDirectory: () => true }; +} +function fileEntry(name: string) { + return { name, isDirectory: () => false }; +} + +// Filesystem layout definition — maps path to directory entries or file list +type FsLayout = Record | ReturnType)[] | string[]>; + +function setupFs(layout: FsLayout, files: Record = {}) { + mockReaddir.mockImplementation(async (dirPath: string, opts?: { withFileTypes?: boolean }) => { + const entries = layout[dirPath]; + if (!entries) throw new Error(`ENOENT: no such directory '${dirPath}'`); + // If called withFileTypes, return the dirent-like objects; otherwise return string[] + if (opts?.withFileTypes) { + // Entries should be dirent objects + return entries; + } + // Plain readdir (used inside probeWorkflow) — return string[] + return entries as string[]; + }); + + mockOpen.mockImplementation(async (filePath: string) => { + const content = files[filePath]; + if (content === undefined) throw new Error(`ENOENT: ${filePath}`); + return makeBufReader(content); + }); +} + describe("/api/list-workflows route", () => { beforeEach(() => { vi.clearAllMocks(); - - // Override mockOpen to simulate filling the buffer with file content - mockOpen.mockImplementation(async (filePath: string) => { - const handle = createFileHandle(""); - // We'll configure per-test below by re-mocking - return handle; - }); }); describe("parameter validation", () => { @@ -81,7 +113,9 @@ describe("/api/list-workflows route", () => { describe("directory probing", () => { it("should return empty array when parent has no subdirectories", async () => { - mockReaddir.mockResolvedValue([]); + setupFs({ + "/home/user/projects": [], + }); const response = await GET(createRequest("/home/user/projects")); const data = await response.json(); @@ -91,11 +125,10 @@ describe("/api/list-workflows route", () => { }); it("should skip subdirectories without JSON files", async () => { - mockReaddir.mockResolvedValueOnce([ - { name: "empty-dir", isDirectory: () => true }, - ]); - // readdir for the subdirectory itself returns no files - mockReaddir.mockResolvedValueOnce([]); + setupFs({ + "/home/user/projects": [dirEntry("empty-dir")], + "/home/user/projects/empty-dir": [], + }); const response = await GET(createRequest("/home/user/projects")); const data = await response.json(); @@ -105,24 +138,15 @@ describe("/api/list-workflows route", () => { }); it("should detect a valid workflow and extract name from header", async () => { - // Parent dir has one subdirectory - mockReaddir.mockResolvedValueOnce([ - { name: "my-project", isDirectory: () => true }, - ]); - // Subdirectory has one JSON file - mockReaddir.mockResolvedValueOnce(["workflow.json"]); - - const workflowHeader = '{"version":1,"name":"Cool Project","nodes":['; - const headerBuf = Buffer.from(workflowHeader); - - mockOpen.mockResolvedValue({ - read: vi.fn().mockImplementation(async (buf: Buffer) => { - headerBuf.copy(buf, 0, 0, Math.min(headerBuf.length, buf.length)); - return { bytesRead: Math.min(headerBuf.length, buf.length) }; - }), - close: vi.fn().mockResolvedValue(undefined), - }); - + setupFs( + { + "/home/user/projects": [dirEntry("my-project")], + "/home/user/projects/my-project": ["workflow.json"], + }, + { + "/home/user/projects/my-project/workflow.json": makeWorkflowHeader("Cool Project"), + } + ); mockStat.mockResolvedValue({ mtimeMs: 1700000000000 }); const response = await GET(createRequest("/home/user/projects")); @@ -132,26 +156,20 @@ describe("/api/list-workflows route", () => { expect(data.workflows).toHaveLength(1); expect(data.workflows[0].name).toBe("Cool Project"); expect(data.workflows[0].directoryPath).toBe("/home/user/projects/my-project"); + expect(data.workflows[0].relativePath).toBe("my-project"); expect(data.workflows[0].lastModified).toBe(1700000000000); }); it("should skip JSON files that are not workflow files", async () => { - mockReaddir.mockResolvedValueOnce([ - { name: "my-project", isDirectory: () => true }, - ]); - mockReaddir.mockResolvedValueOnce(["package.json"]); - - // package.json doesn't contain "version" + "nodes" - const notWorkflow = '{"name":"my-app","version":"1.0.0","dependencies":{}}'; - const notBuf = Buffer.from(notWorkflow); - - mockOpen.mockResolvedValue({ - read: vi.fn().mockImplementation(async (buf: Buffer) => { - notBuf.copy(buf, 0, 0, Math.min(notBuf.length, buf.length)); - return { bytesRead: Math.min(notBuf.length, buf.length) }; - }), - close: vi.fn().mockResolvedValue(undefined), - }); + setupFs( + { + "/home/user/projects": [dirEntry("my-project")], + "/home/user/projects/my-project": ["package.json"], + }, + { + "/home/user/projects/my-project/package.json": '{"name":"my-app","version":"1.0.0","dependencies":{}}', + } + ); const response = await GET(createRequest("/home/user/projects")); const data = await response.json(); @@ -161,23 +179,15 @@ describe("/api/list-workflows route", () => { }); it("should fall back to directory name when workflow has no name field", async () => { - mockReaddir.mockResolvedValueOnce([ - { name: "unnamed-project", isDirectory: () => true }, - ]); - mockReaddir.mockResolvedValueOnce(["workflow.json"]); - - // No "name" field but has version + nodes - const noNameHeader = '{"version":1,"nodes":[{"id":"1"}],"edges":[]}'; - const noNameBuf = Buffer.from(noNameHeader); - - mockOpen.mockResolvedValue({ - read: vi.fn().mockImplementation(async (buf: Buffer) => { - noNameBuf.copy(buf, 0, 0, Math.min(noNameBuf.length, buf.length)); - return { bytesRead: Math.min(noNameBuf.length, buf.length) }; - }), - close: vi.fn().mockResolvedValue(undefined), - }); - + setupFs( + { + "/home/user/projects": [dirEntry("unnamed-project")], + "/home/user/projects/unnamed-project": ["workflow.json"], + }, + { + "/home/user/projects/unnamed-project/workflow.json": makeWorkflowHeader(), + } + ); mockStat.mockResolvedValue({ mtimeMs: 1700000000000 }); const response = await GET(createRequest("/home/user/projects")); @@ -189,28 +199,18 @@ describe("/api/list-workflows route", () => { }); it("should sort workflows by most recently modified first", async () => { - mockReaddir.mockResolvedValueOnce([ - { name: "old-project", isDirectory: () => true }, - { name: "new-project", isDirectory: () => true }, - ]); - - // Old project - mockReaddir.mockResolvedValueOnce(["workflow.json"]); - // New project - mockReaddir.mockResolvedValueOnce(["workflow.json"]); - - const workflowHeader = '{"version":1,"name":"Test","nodes":[]}'; - const headerBuf = Buffer.from(workflowHeader); - - mockOpen.mockResolvedValue({ - read: vi.fn().mockImplementation(async (buf: Buffer) => { - headerBuf.copy(buf, 0, 0, Math.min(headerBuf.length, buf.length)); - return { bytesRead: Math.min(headerBuf.length, buf.length) }; - }), - close: vi.fn().mockResolvedValue(undefined), - }); + setupFs( + { + "/home/user/projects": [dirEntry("old-project"), dirEntry("new-project")], + "/home/user/projects/old-project": ["workflow.json"], + "/home/user/projects/new-project": ["workflow.json"], + }, + { + "/home/user/projects/old-project/workflow.json": makeWorkflowHeader("Old"), + "/home/user/projects/new-project/workflow.json": makeWorkflowHeader("New"), + } + ); - // old-project was modified before new-project mockStat .mockResolvedValueOnce({ mtimeMs: 1600000000000 }) .mockResolvedValueOnce({ mtimeMs: 1700000000000 }); @@ -220,12 +220,84 @@ describe("/api/list-workflows route", () => { expect(data.success).toBe(true); expect(data.workflows).toHaveLength(2); - // Newer project should come first expect(data.workflows[0].lastModified).toBe(1700000000000); expect(data.workflows[1].lastModified).toBe(1600000000000); }); }); + describe("recursive discovery", () => { + it("should discover workflows in nested subdirectories", async () => { + setupFs( + { + "/home/user/projects": [dirEntry("top-project"), dirEntry("category")], + "/home/user/projects/top-project": ["workflow.json"], + "/home/user/projects/category": [dirEntry("nested-project")], + "/home/user/projects/category/nested-project": ["workflow.json"], + }, + { + "/home/user/projects/top-project/workflow.json": makeWorkflowHeader("Top Project"), + "/home/user/projects/category/nested-project/workflow.json": makeWorkflowHeader("Nested Project"), + } + ); + mockStat.mockResolvedValue({ mtimeMs: 1700000000000 }); + + const response = await GET(createRequest("/home/user/projects")); + const data = await response.json(); + + expect(data.success).toBe(true); + expect(data.workflows).toHaveLength(2); + + const names = data.workflows.map((w: { name: string }) => w.name); + expect(names).toContain("Top Project"); + expect(names).toContain("Nested Project"); + + const nested = data.workflows.find((w: { name: string }) => w.name === "Nested Project"); + expect(nested.relativePath).toBe("category/nested-project"); + + const top = data.workflows.find((w: { name: string }) => w.name === "Top Project"); + expect(top.relativePath).toBe("top-project"); + }); + + it("should skip hidden directories and node_modules", async () => { + setupFs( + { + "/home/user/projects": [ + dirEntry("good-project"), + dirEntry(".hidden"), + dirEntry("node_modules"), + dirEntry(".git"), + ], + "/home/user/projects/good-project": ["workflow.json"], + // These should never be read — if they are, the mock will return entries + // but we don't set up workflow files so they'd be empty anyway + "/home/user/projects/.hidden": [dirEntry("secret")], + "/home/user/projects/node_modules": [dirEntry("some-package")], + "/home/user/projects/.git": [dirEntry("objects")], + }, + { + "/home/user/projects/good-project/workflow.json": makeWorkflowHeader("Good"), + } + ); + mockStat.mockResolvedValue({ mtimeMs: 1700000000000 }); + + const response = await GET(createRequest("/home/user/projects")); + const data = await response.json(); + + expect(data.success).toBe(true); + expect(data.workflows).toHaveLength(1); + expect(data.workflows[0].name).toBe("Good"); + + // Verify hidden/skip dirs were never traversed as withFileTypes (recursive collection) + const readdirCalls = mockReaddir.mock.calls; + const withFileTypeCalls = readdirCalls + .filter((call: unknown[]) => call[1]?.withFileTypes) + .map((call: unknown[]) => call[0]); + expect(withFileTypeCalls).not.toContain("/home/user/projects/.hidden"); + expect(withFileTypeCalls).not.toContain("/home/user/projects/node_modules"); + expect(withFileTypeCalls).not.toContain("/home/user/projects/.git"); + }); + }); + describe("error handling", () => { it("should return 500 when readdir fails on parent directory", async () => { mockReaddir.mockRejectedValue(new Error("ENOENT: no such directory")); diff --git a/src/app/api/list-workflows/route.ts b/src/app/api/list-workflows/route.ts index d4084d96..ddfc0f02 100644 --- a/src/app/api/list-workflows/route.ts +++ b/src/app/api/list-workflows/route.ts @@ -3,9 +3,13 @@ import * as fs from "fs/promises"; import * as path from "path"; import { validateWorkflowPath } from "@/utils/pathValidation"; +const MAX_DEPTH = 3; +const SKIP_DIRS = new Set([".git", "node_modules", "__pycache__", ".next"]); + interface WorkflowListEntry { name: string; directoryPath: string; + relativePath: string; lastModified: number; } @@ -13,9 +17,46 @@ interface WorkflowListEntry { // Workflow JSON starts with { "version": 1, "name": "..." ... } so 1KB is plenty. const HEAD_BYTES = 1024; +interface DirEntry { + absPath: string; + relativePath: string; +} + +async function collectAllDirectories( + rootPath: string, + currentPath: string, + depth: number +): Promise { + if (depth > MAX_DEPTH) return []; + + const entries = await fs.readdir(currentPath, { withFileTypes: true }); + const dirs: DirEntry[] = []; + + for (const entry of entries) { + if (!entry.isDirectory()) continue; + if (entry.name.startsWith(".") || SKIP_DIRS.has(entry.name)) continue; + + const absPath = path.join(currentPath, entry.name); + const relativePath = path.relative(rootPath, absPath); + dirs.push({ absPath, relativePath }); + + if (depth < MAX_DEPTH) { + try { + const nested = await collectAllDirectories(rootPath, absPath, depth + 1); + dirs.push(...nested); + } catch { + // Can't read subdirectory — skip but keep the dir entry itself + } + } + } + + return dirs; +} + async function probeWorkflow( dirPath: string, - dirName: string + dirName: string, + relativePath: string ): Promise { try { const files = await fs.readdir(dirPath); @@ -43,6 +84,7 @@ async function probeWorkflow( return { name, directoryPath: dirPath, + relativePath, lastModified: stat.mtimeMs, }; } finally { @@ -77,13 +119,12 @@ export async function GET(request: NextRequest) { } try { - const entries = await fs.readdir(parentPath, { withFileTypes: true }); - const directories = entries.filter((e) => e.isDirectory()); + const allDirs = await collectAllDirectories(parentPath, parentPath, 0); // Probe all directories in parallel const results = await Promise.all( - directories.map((dir) => - probeWorkflow(path.join(parentPath, dir.name), dir.name) + allDirs.map((dir) => + probeWorkflow(dir.absPath, path.basename(dir.absPath), dir.relativePath) ) ); diff --git a/src/components/quickstart/WorkflowBrowserView.tsx b/src/components/quickstart/WorkflowBrowserView.tsx index 8dd13dd0..b6cf6785 100644 --- a/src/components/quickstart/WorkflowBrowserView.tsx +++ b/src/components/quickstart/WorkflowBrowserView.tsx @@ -11,6 +11,7 @@ import { interface WorkflowListEntry { name: string; directoryPath: string; + relativePath?: string; lastModified: number; } @@ -306,7 +307,7 @@ export function WorkflowBrowserView({ {entry.name}
- {dirBasename(entry.directoryPath)} + {entry.relativePath || dirBasename(entry.directoryPath)}