From d23b0c4faedff72d15e509452f473d8523f9fb9f Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Sat, 18 Apr 2026 01:19:22 +0800 Subject: [PATCH 1/5] =?UTF-8?q?fix(import-markdown):=20CI=E6=B8=AC?= =?UTF-8?q?=E8=A9=A6=E7=99=BB=E8=A8=98=20+=20.md=E7=9B=AE=E9=8C=84skip?= =?UTF-8?q?=E4=BF=9D=E8=AD=B7=20(Issue=20#588)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cli.ts: 讀檔前先stat確認isFile()，非檔案skip並log warn - ci-test-manifest.mjs: 把import-markdown.test.mjs加入cli-smoke群組 - import-markdown.test.mjs: 新增測試「skip non-file .md entries」 --- cli.ts | 27 +++++++++++++-- scripts/ci-test-manifest.mjs | 2 +- test/import-markdown/import-markdown.test.mjs | 34 ++++++++++++++++++- 3 files changed, 59 insertions(+), 4 deletions(-) diff --git a/cli.ts b/cli.ts index c969b755..6607e86d 100644 --- a/cli.ts +++ b/cli.ts @@ -658,8 +658,31 @@ export async function runImportMarkdown( // Parse each file for memory entries (lines starting with "- ") for (const { filePath, scope: discoveredScope } of mdFiles) { - foundFiles++; - let content = await fsPromises.readFile(filePath, "utf-8"); + let content: string; + try { + const stats = await fsPromises.stat(filePath); + if (!stats.isFile()) { + // Skip non-file entries (e.g. a directory named "YYYY-MM-DD.md") + console.warn(` [skip] not a file: ${filePath}`); + skipped++; + continue; + } + foundFiles++; // count only actual files, not directories + content = await fsPromises.readFile(filePath, "utf-8"); + } catch (err) { + const e = err as NodeJS.ErrnoException; + if (e.code === "EISDIR") { + // .md directory — already handled above by isFile() check, but catch + // this explicitly so genuine I/O errors (permissions, corruption) + // are not silently swallowed + console.warn(` [skip] not a file: ${filePath}`); + skipped++; + } else { + throw err; // re-throw genuine I/O errors + } + continue; + } + // (fix(import-markdown): CI測試登記 + .md目錄skip保護) // Strip UTF-8 BOM (e.g. from Windows Notepad-saved files) content = content.replace(/^\uFEFF/, ""); // Normalize line endings: handle both CRLF (\r\n) and LF (\n) diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index fcf90587..7bbc10be 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -18,12 +18,12 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/recall-text-cleanup.test.mjs", args: ["--test"] }, { group: "storage-and-schema", runner: "node", file: "test/update-consistency-lancedb.test.mjs" }, { group: "core-regression", runner: "node", file: "test/strip-envelope-metadata.test.mjs", args: ["--test"] }, + { group: "cli-smoke", runner: "node", file: "test/import-markdown/import-markdown.test.mjs", args: ["--test"] }, { group: "cli-smoke", runner: "node", file: "test/cli-smoke.mjs" }, { group: "cli-smoke", runner: "node", file: "test/functional-e2e.mjs" }, { group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-memory-lifecycle.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" }, - { group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" }, { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, { group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] }, diff --git a/test/import-markdown/import-markdown.test.mjs b/test/import-markdown/import-markdown.test.mjs index e2deb247..e07b8420 100644 --- a/test/import-markdown/import-markdown.test.mjs +++ b/test/import-markdown/import-markdown.test.mjs @@ -1,4 +1,4 @@ -/** +/** * import-markdown.test.mjs * Integration tests for the import-markdown CLI command. * Tests: BOM handling, CRLF normalization, bullet formats, dedup logic, @@ -416,6 +416,38 @@ describe("import-markdown CLI", () => { ); }); }); + describe("skip non-file .md entries", () => { + it("skips a directory named YYYY-MM-DD.md without aborting import", async () => { + const wsDir = await setupWorkspace("nonfile-test"); + // Create memory/ subdirectory first + await mkdir(join(wsDir, "memory"), { recursive: true }); + // Create a real .md file + await writeFile( + join(wsDir, "memory", "2026-04-11.md"), + "- Real file entry\n", + "utf-8", + ); + // Create a directory that looks like a .md file (the bug scenario) + const fakeDir = join(wsDir, "memory", "2026-04-12.md"); + await mkdir(fakeDir, { recursive: true }); + + const ctx = { embedder: mockEmbedder, store: mockStore }; + let threw = false; + try { + const { imported, skipped } = await runImportMarkdown(ctx, { + openclawHome: testWorkspaceDir, + workspaceGlob: "nonfile-test", + }); + // Should have imported the real file (1 entry from "- Real file entry") + assert.strictEqual(imported, 1, "should import the real .md file"); + assert.strictEqual(skipped, 1, "should skip exactly 1 (.md directory entry)"); + } catch (err) { + threw = true; + throw new Error(`Import aborted on .md directory: ${err}`); + } + assert.ok(!threw, "import should not abort when encountering .md directory"); + }); + }); }); // ────────────────────────────────────────────────────────────────────────────── Test runner helper ────────────────────────────────────────────────────────────────────────────── From 3a14ab8b2a0dbc1cb4b3cc1629f68914ed73c7ba Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Sat, 18 Apr 2026 14:16:41 +0800 Subject: [PATCH 2/5] fix(import-markdown): address PR #649 review comments - F1: restore smart-extractor-batch-embed.test.mjs to CI manifest - F2: remove BOM from import-markdown.test.mjs - F3: simplify EISDIR catch (unreachable, keep for TOCTOU paranoia) --- cli.ts | 13 ++++--------- scripts/ci-test-manifest.mjs | 1 + 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/cli.ts b/cli.ts index 6607e86d..e44b0f29 100644 --- a/cli.ts +++ b/cli.ts @@ -671,15 +671,10 @@ export async function runImportMarkdown( content = await fsPromises.readFile(filePath, "utf-8"); } catch (err) { const e = err as NodeJS.ErrnoException; - if (e.code === "EISDIR") { - // .md directory — already handled above by isFile() check, but catch - // this explicitly so genuine I/O errors (permissions, corruption) - // are not silently swallowed - console.warn(` [skip] not a file: ${filePath}`); - skipped++; - } else { - throw err; // re-throw genuine I/O errors - } + // EISDIR unreachable here — isFile() check above already filters directories. + // Keep catch for genuine I/O errors (permissions, corruption, etc.). + console.warn(` [skip] not a file: ${filePath}`); + skipped++; continue; } // (fix(import-markdown): CI測試登記 + .md目錄skip保護) diff --git a/scripts/ci-test-manifest.mjs b/scripts/ci-test-manifest.mjs index 7bbc10be..b8275e36 100644 --- a/scripts/ci-test-manifest.mjs +++ b/scripts/ci-test-manifest.mjs @@ -24,6 +24,7 @@ export const CI_TEST_MANIFEST = [ { group: "core-regression", runner: "node", file: "test/retriever-rerank-regression.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-memory-lifecycle.mjs" }, { group: "core-regression", runner: "node", file: "test/smart-extractor-branches.mjs" }, + { group: "core-regression", runner: "node", file: "test/smart-extractor-batch-embed.test.mjs" }, { group: "packaging-and-workflow", runner: "node", file: "test/plugin-manifest-regression.mjs" }, { group: "core-regression", runner: "node", file: "test/session-summary-before-reset.test.mjs", args: ["--test"] }, { group: "packaging-and-workflow", runner: "node", file: "test/sync-plugin-version.test.mjs", args: ["--test"] }, From 3bada90f4010804f4d17eb02b80a6bbb6cfe5410 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Sun, 19 Apr 2026 01:39:06 +0800 Subject: [PATCH 3/5] =?UTF-8?q?fix(import-markdown):=20MR2=20=E5=84=AA?= =?UTF-8?q?=E5=8C=96=20-=20=E4=BD=BF=E7=94=A8=20withFileTypes=20=E9=81=BF?= =?UTF-8?q?=E5=85=8D=E9=A1=8D=E5=A4=96=20stat()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - readdir(memoryDir, { withFileTypes: true }) 取得 Dirent - 用 f.isFile() 過濾，非額外 stat() syscall - 移除迴圈中的 stat() 呼叫（原本每個 .md 檔都會 stat） - 同時優化 agent memory 目錄的掃描 --- cli.ts | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/cli.ts b/cli.ts index e44b0f29..1330a3ef 100644 --- a/cli.ts +++ b/cli.ts @@ -560,10 +560,10 @@ export async function runImportMarkdown( try { const stats = await fsPromises.stat(memoryDir); if (stats.isDirectory()) { - const files = await fsPromises.readdir(memoryDir); + const files = await fsPromises.readdir(memoryDir, { withFileTypes: true }); for (const f of files) { - if (f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f)) { - mdFiles.push({ filePath: path.join(memoryDir, f), scope: entry.name }); + if (f.isFile() && f.name.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f.name)) { + mdFiles.push({ filePath: path.join(memoryDir, f.name), scope: entry.name }); } } } @@ -592,10 +592,10 @@ export async function runImportMarkdown( try { const stats = await fsP.stat(agentMemoryDir); if (stats.isDirectory()) { - const files = await fsP.readdir(agentMemoryDir); + const files = await fsP.readdir(agentMemoryDir, { withFileTypes: true }); for (const f of files) { - if (f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f)) { - mdFiles.push({ filePath: path.join(agentMemoryDir, f), scope: agentId }); + if (f.isFile() && f.name.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f.name)) { + mdFiles.push({ filePath: path.join(agentMemoryDir, f.name), scope: agentId }); } } } @@ -660,20 +660,12 @@ export async function runImportMarkdown( for (const { filePath, scope: discoveredScope } of mdFiles) { let content: string; try { - const stats = await fsPromises.stat(filePath); - if (!stats.isFile()) { - // Skip non-file entries (e.g. a directory named "YYYY-MM-DD.md") - console.warn(` [skip] not a file: ${filePath}`); - skipped++; - continue; - } - foundFiles++; // count only actual files, not directories + // 已在收集時用 withFileTypes: true 過濾，直接讀取 + foundFiles++; content = await fsPromises.readFile(filePath, "utf-8"); } catch (err) { - const e = err as NodeJS.ErrnoException; - // EISDIR unreachable here — isFile() check above already filters directories. - // Keep catch for genuine I/O errors (permissions, corruption, etc.). - console.warn(` [skip] not a file: ${filePath}`); + // I/O errors (permissions, corruption, etc.) + console.warn(` [skip] read failed: ${filePath}: ${(err as Error).message}`); skipped++; continue; } From 1fdbe3ddf25a4ab627bf61f8d2cf38aa04e669a2 Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 21 Apr 2026 01:29:40 +0800 Subject: [PATCH 4/5] =?UTF-8?q?fix(import-markdown):=20=E4=BF=AE=E5=BE=A9?= =?UTF-8?q?=20flatMemoryDir=20=E7=BC=BA=E5=B0=91=20withFileTypes=20(Issue?= =?UTF-8?q?=20#588)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cli.ts line 636: 將 readdir(flatMemoryDir) 改為使用 withFileTypes: true - 與其他 3 處 readdir 保持一致模式（workspaceDir, memoryDir, agentMemoryDir） - 新增 regression test: flatMemoryDir 路徑的 .md directory skip 測試 Claude API adversarial review 後確認： - 4 處 readdir 全部已修復 - 邊界條件測試已新增 --- cli.ts | 6 ++-- test/import-markdown/import-markdown.test.mjs | 34 +++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/cli.ts b/cli.ts index 1330a3ef..c2e0a9b0 100644 --- a/cli.ts +++ b/cli.ts @@ -633,10 +633,10 @@ export async function runImportMarkdown( try { const stats = await fsPromises.stat(flatMemoryDir); if (stats.isDirectory()) { - const files = await fsPromises.readdir(flatMemoryDir); + const files = await fsPromises.readdir(flatMemoryDir, { withFileTypes: true }); for (const f of files) { - if (f.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f)) { - mdFiles.push({ filePath: path.join(flatMemoryDir, f), scope: workspaceScope || "global" }); + if (f.isFile() && f.name.endsWith(".md") && /^\d{4}-\d{2}-\d{2}/.test(f.name)) { + mdFiles.push({ filePath: path.join(flatMemoryDir, f.name), scope: workspaceScope || "global" }); } } } diff --git a/test/import-markdown/import-markdown.test.mjs b/test/import-markdown/import-markdown.test.mjs index e07b8420..a8bab24c 100644 --- a/test/import-markdown/import-markdown.test.mjs +++ b/test/import-markdown/import-markdown.test.mjs @@ -447,6 +447,40 @@ describe("import-markdown CLI", () => { } assert.ok(!threw, "import should not abort when encountering .md directory"); }); + + // Regression test for flatMemoryDir path (workspace/memory/YYYY-MM-DD.md) + // This path was missing withFileTypes: true in cli.ts, causing .md directories + // to be pushed to mdFiles and later causing EISDIR errors during readFile + it("skips a .md directory in flatMemoryDir without aborting import", async () => { + const wsDir = await setupWorkspace("flatmd-dir-test"); + // Create memory/ subdirectory for flat structure + await mkdir(join(wsDir, "memory"), { recursive: true }); + // Create a real .md file + await writeFile( + join(wsDir, "memory", "2026-04-11.md"), + "- Real flat file entry\n", + "utf-8", + ); + // Create a directory that looks like a .md file in flat memory path + const fakeDir = join(wsDir, "memory", "2026-04-12.md"); + await mkdir(fakeDir, { recursive: true }); + + const ctx = { embedder: mockEmbedder, store: mockStore }; + let threw = false; + try { + // This specifically tests the flatMemoryDir path (no workspaceGlob) + const { imported, skipped } = await runImportMarkdown(ctx, { + openclawHome: testWorkspaceDir, + workspaceGlob: "flatmd-dir-test", + }); + assert.strictEqual(imported, 1, "should import the real .md file"); + assert.strictEqual(skipped, 1, "should skip exactly 1 (.md directory entry)"); + } catch (err) { + threw = true; + throw new Error(`Import aborted on .md directory in flatMemoryDir: ${err}`); + } + assert.ok(!threw, "import should not abort when encountering .md directory in flatMemoryDir"); + }); }); }); From fd4b4f69e33a380f891651f2145c4dae8182a54e Mon Sep 17 00:00:00 2001 From: jlin53882 Date: Tue, 21 Apr 2026 01:45:52 +0800 Subject: [PATCH 5/5] =?UTF-8?q?fix(import-markdown.test):=20=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=20skipped=20=E6=9C=9F=E6=9C=9B=E5=80=BC=20=E2=80=94?= =?UTF-8?q?=20f.isFile()=20=E5=9C=A8=E6=94=B6=E9=9B=86=E9=9A=8E=E6=AE=B5?= =?UTF-8?q?=E9=81=8E=E6=BF=BE=E7=9B=AE=E9=8C=84=EF=BC=8C=E4=B8=8D=E8=A7=B8?= =?UTF-8?q?=E7=99=BC=20read=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 測試期望 skipped === 1 的前提是原始實作：stat() 發現目錄 -> EISDIR error -> catch 區塊 skipped++。 但 MR2 優化後使用 withFileTypes + f.isFile() 在收集階段過濾， 目錄根本不會進入 read loop，所以 skipped 保持為 0。 修復：將 skipped 期望值從 1 改為 0，並加上說明註解。 --- test/import-markdown/import-markdown.test.mjs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/import-markdown/import-markdown.test.mjs b/test/import-markdown/import-markdown.test.mjs index a8bab24c..f8786907 100644 --- a/test/import-markdown/import-markdown.test.mjs +++ b/test/import-markdown/import-markdown.test.mjs @@ -440,7 +440,9 @@ describe("import-markdown CLI", () => { }); // Should have imported the real file (1 entry from "- Real file entry") assert.strictEqual(imported, 1, "should import the real .md file"); - assert.strictEqual(skipped, 1, "should skip exactly 1 (.md directory entry)"); + // skipped === 0: f.isFile() silently filters .md directories during mdFiles collection. + // This is correct — the directory doesn't cause EISDIR or increment skipped. + assert.strictEqual(skipped, 0, "directory silently filtered by f.isFile() — not counted as skipped"); } catch (err) { threw = true; throw new Error(`Import aborted on .md directory: ${err}`); @@ -474,7 +476,9 @@ describe("import-markdown CLI", () => { workspaceGlob: "flatmd-dir-test", }); assert.strictEqual(imported, 1, "should import the real .md file"); - assert.strictEqual(skipped, 1, "should skip exactly 1 (.md directory entry)"); + // skipped === 0: f.isFile() in flatMemoryDir scan (cli.ts:639) silently filters + // .md directories during collection — no EISDIR error, no skipped++ increment. + assert.strictEqual(skipped, 0, "directory silently filtered by f.isFile() — not counted as skipped"); } catch (err) { threw = true; throw new Error(`Import aborted on .md directory in flatMemoryDir: ${err}`);