diff --git a/src/functions/remember.ts b/src/functions/remember.ts index 5735b4f23..0d651ab75 100644 --- a/src/functions/remember.ts +++ b/src/functions/remember.ts @@ -1,6 +1,7 @@ import { TriggerAction, type ISdk } from "iii-sdk"; import type { Memory } from "../types.js"; import { KV, generateId, jaccardSimilarity } from "../state/schema.js"; +import { isNegationConflict } from "../state/memory-dedup.js"; import { StateKV } from "../state/kv.js"; import { withKeyedLock } from "../state/keyed-mutex.js"; import { memoryToObservation } from "../state/memory-utils.js"; @@ -78,7 +79,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void { lowerContent, existing.content.toLowerCase(), ); - if (similarity > 0.7) { + if (similarity > 0.7 && !isNegationConflict(lowerContent, existing.content.toLowerCase())) { supersededId = existing.id; supersededVersion = existing.version ?? 1; supersededMemory = existing; diff --git a/src/state/memory-dedup.ts b/src/state/memory-dedup.ts new file mode 100644 index 000000000..77db709ce --- /dev/null +++ b/src/state/memory-dedup.ts @@ -0,0 +1,32 @@ +/** + * Negation guard for memory dedup. + * + * Prevents "do X" from superseding "do not X" (and vice versa) even when + * the two strings share enough tokens to exceed the Jaccard threshold. + * + * The guard is deliberately narrow and conservative: + * - Only triggers when one side has a negation marker and the other does not. + * - Supports English ("not", "never", "don't", "do not") and CJK + * ("不要", "别", "無", "なし", "않") markers. + * - If BOTH sides have negation markers, they are treated as compatible + * (both say "don't do X") and the guard does not fire. + */ + +const EN_NEGATION = /\b(not|never|don't|do not|cannot|should not|must not|no)\b/i; +const CJK_NEGATION = /不要|别|勿|無|无|不|なし|않|안|못/i; + +function hasNegation(text: string): boolean { + return EN_NEGATION.test(text) || CJK_NEGATION.test(text); +} + +/** + * Returns true if the two texts appear to be in a negation conflict + * (one says do X, the other says do NOT do X) and should NOT be superseded. + */ +export function isNegationConflict(a: string, b: string): boolean { + const aNeg = hasNegation(a); + const bNeg = hasNegation(b); + // If both have negation or neither does, no conflict. + if (aNeg === bNeg) return false; + return true; +} \ No newline at end of file diff --git a/src/state/schema.ts b/src/state/schema.ts index cb29d41ad..51f5ffdaf 100644 --- a/src/state/schema.ts +++ b/src/state/schema.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { hasCjk, segmentCjk } from "./cjk-segmenter.js"; export const KV = { sessions: "mem:sessions", @@ -91,9 +92,52 @@ export function fingerprintId(prefix: string, content: string): string { return `${prefix}_${hash.slice(0, 16)}`; } +/** + * CJK-aware tokenizer for Jaccard similarity. + * + * For English/whitespace-delimited text we keep the original behaviour: + * split on whitespace, filter tokens longer than 2 characters. + * + * For CJK text (Chinese/Japanese/Korean) whitespace splitting produces + * near-disjoint sets because CJK text is typically written without spaces. + * We reuse the existing `segmentCjk` function and additionally emit + * character bigrams from CJK runs so that no-space Chinese text still + * produces useful overlap. + */ +function tokenizeForJaccard(text: string): Set { + // Fast path: no CJK characters → original whitespace split + if (!hasCjk(text)) { + return new Set(text.split(/\s+/).filter((t) => t.length > 2)); + } + + // CJK path: segment + add character bigrams for overlap + const segments = segmentCjk(text); + const tokens = new Set(); + + for (const seg of segments) { + // Keep the whole segment as a token (works for word-segmented Chinese + // via jieba, and for English fragments in mixed text). + const cleaned = seg.replace(/[^\p{L}\p{N}]/gu, "").toLowerCase(); + if (cleaned.length > 2) { + tokens.add(cleaned); + } + + // Add CJK character bigrams from each segment so unsegmented CJK + // text can still produce meaningful overlap. + const cjkChars = seg.match(/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu); + if (cjkChars && cjkChars.length >= 2) { + for (let i = 0; i < cjkChars.length - 1; i++) { + tokens.add(cjkChars[i] + cjkChars[i + 1]); + } + } + } + + return tokens; +} + export function jaccardSimilarity(a: string, b: string): number { - const setA = new Set(a.split(/\s+/).filter((t) => t.length > 2)); - const setB = new Set(b.split(/\s+/).filter((t) => t.length > 2)); + const setA = tokenizeForJaccard(a); + const setB = tokenizeForJaccard(b); if (setA.size === 0 && setB.size === 0) return 1; if (setA.size === 0 || setB.size === 0) return 0; let intersection = 0; diff --git a/test/remember-cjk-dedup.test.ts b/test/remember-cjk-dedup.test.ts new file mode 100644 index 000000000..21ab49573 --- /dev/null +++ b/test/remember-cjk-dedup.test.ts @@ -0,0 +1,212 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; + +vi.mock("../src/logger.js", () => ({ + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, +})); + +vi.mock("../src/state/keyed-mutex.js", () => ({ + withKeyedLock: (_key: string, fn: () => Promise) => fn(), +})); + +vi.mock("../src/functions/audit.js", () => ({ + recordAudit: vi.fn(), +})); + +vi.mock("../src/functions/access-tracker.js", () => ({ + recordAccessBatch: vi.fn(), + deleteAccessLog: vi.fn(), +})); + +vi.mock("../src/config.js", () => ({ + getAgentId: () => undefined, + isAgentScopeIsolated: () => false, +})); + +import { registerRememberFunction } from "../src/functions/remember.js"; +import { getSearchIndex, setIndexPersistence } from "../src/functions/search.js"; +import { KV } from "../src/state/schema.js"; + +function makeMockKV() { + const store = new Map>(); + return { + get: async (scope: string, key: string): Promise => + (store.get(scope)?.get(key) as T) ?? null, + set: async (scope: string, key: string, data: T): Promise => { + if (!store.has(scope)) store.set(scope, new Map()); + store.get(scope)!.set(key, data); + return data; + }, + delete: async (scope: string, key: string): Promise => { + store.get(scope)?.delete(key); + }, + list: async (scope: string): Promise => { + const entries = store.get(scope); + return entries ? (Array.from(entries.values()) as T[]) : []; + }, + }; +} + +function makeMockSdk() { + const functions = new Map(); + return { + registerFunction: (id: string, handler: Function) => { + functions.set(id, handler); + }, + registerTrigger: () => {}, + trigger: async (input: { function_id: string; payload: unknown }) => { + const fn = functions.get(input.function_id); + if (!fn) return {}; + return fn(input.payload); + }, + }; +} + +describe("mem::remember — CJK dedup", () => { + let sdk: ReturnType; + let kv: ReturnType; + + beforeEach(() => { + sdk = makeMockSdk(); + kv = makeMockKV(); + getSearchIndex().clear(); + setIndexPersistence(null); + registerRememberFunction(sdk as never, kv as never); + }); + + it("supersedes a pure Chinese duplicate preference", async () => { + const first = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用代码审查来确保代码质量", + type: "preference", + }, + }) as { memory: { id: string } }; + + const second = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用代码审查来确保代码质量", + type: "preference", + }, + }) as { memory: { id: string; supersedes: string[] } }; + + expect(second.memory.supersedes).toContain(first.memory.id); + + const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id); + expect(original?.isLatest).toBe(false); + }); + + it("supersedes a mixed Chinese/English duplicate preference", async () => { + const first = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "preference: 使用 eslint 进行代码检查", + type: "preference", + }, + }) as { memory: { id: string } }; + + const second = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用 eslint 进行代码检查 preference", + type: "preference", + }, + }) as { memory: { id: string; supersedes: string[] } }; + + expect(second.memory.supersedes).toContain(first.memory.id); + }); + + it("does not supersede related but distinct Chinese preferences", async () => { + const first = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用代码审查来确保质量", + type: "preference", + }, + }) as { memory: { id: string } }; + + const second = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用持续集成来确保部署", + type: "preference", + }, + }) as { memory: { id: string; supersedes: string[] } }; + + expect(second.memory.supersedes).toHaveLength(0); + + const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id); + expect(original?.isLatest).toBe(true); + }); + + it("does not supersede negation conflict (English do vs do-not)", async () => { + const first = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "always use strict mode in JavaScript files", + type: "preference", + }, + }) as { memory: { id: string } }; + + const second = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "do not use strict mode in JavaScript files", + type: "preference", + }, + }) as { memory: { id: string; supersedes: string[] } }; + + expect(second.memory.supersedes).toHaveLength(0); + + const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id); + expect(original?.isLatest).toBe(true); + }); + + it("does not supersede negation conflict (Chinese 不要 vs affirmative)", async () => { + const first = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用 var 声明变量", + type: "preference", + }, + }) as { memory: { id: string } }; + + const second = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "不要使用 var 声明变量", + type: "preference", + }, + }) as { memory: { id: string; supersedes: string[] } }; + + expect(second.memory.supersedes).toHaveLength(0); + + const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id); + expect(original?.isLatest).toBe(true); + }); + + it("respects project isolation for CJK dedup", async () => { + const first = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用代码审查来确保质量", + type: "preference", + project: "api", + }, + }) as { memory: { id: string } }; + + const second = await sdk.trigger({ + function_id: "mem::remember", + payload: { + content: "使用代码审查来确保质量", + type: "preference", + project: "web", + }, + }) as { memory: { id: string; supersedes: string[] } }; + + expect(second.memory.supersedes).toHaveLength(0); + + const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id); + expect(original?.isLatest).toBe(true); + }); +}); \ No newline at end of file diff --git a/test/remember-dedup-similarity.test.ts b/test/remember-dedup-similarity.test.ts new file mode 100644 index 000000000..0d4a25673 --- /dev/null +++ b/test/remember-dedup-similarity.test.ts @@ -0,0 +1,100 @@ +import { describe, it, expect } from "vitest"; +import { jaccardSimilarity } from "../src/state/schema.js"; +import { isNegationConflict } from "../src/state/memory-dedup.js"; + +describe("jaccardSimilarity — English (regression)", () => { + it("returns 1 for identical strings", () => { + expect(jaccardSimilarity("hello world", "hello world")).toBe(1); + }); + + it("returns 0 for completely disjoint strings", () => { + expect(jaccardSimilarity("apple banana", "xyz qwerty")).toBe(0); + }); + + it("returns intermediate value for partial overlap", () => { + const sim = jaccardSimilarity("the quick brown fox", "the quick red fox"); + expect(sim).toBeGreaterThan(0); + expect(sim).toBeLessThan(1); + }); +}); + +describe("jaccardSimilarity — CJK-aware", () => { + it("gives high similarity for identical Chinese text", () => { + const a = "使用代码审查来确保质量"; + const b = "使用代码审查来确保质量"; + expect(jaccardSimilarity(a, b)).toBe(1); + }); + + it("gives high similarity for near-identical Chinese with punctuation differences", () => { + const a = "使用代码审查来确保质量。"; + const b = "使用代码审查来确保质量"; + expect(jaccardSimilarity(a, b)).toBeGreaterThan(0.7); + }); + + it("gives high similarity for reordered Chinese preference lists", () => { + const a = "使用独立审查子代理 进行代码审查"; + const b = "进行代码审查 使用独立审查子代理"; + expect(jaccardSimilarity(a, b)).toBeGreaterThan(0.7); + }); + + it("gives lower similarity for distinct Chinese preferences", () => { + const a = "使用代码审查来确保质量"; + const b = "使用持续集成来确保部署"; + const sim = jaccardSimilarity(a, b); + expect(sim).toBeLessThan(0.7); + }); + + it("handles mixed Chinese/English text", () => { + const a = "preference: 使用 eslint 进行代码检查"; + const b = "preference: 使用 eslint 进行代码检查"; + expect(jaccardSimilarity(a, b)).toBe(1); + }); + + it("gives high similarity for near-identical mixed Chinese/English", () => { + const a = "preference: 使用 eslint 进行代码检查"; + const b = "使用 eslint 进行代码检查 preference"; + expect(jaccardSimilarity(a, b)).toBeGreaterThan(0.7); + }); + + it("returns 0 when one side is empty and the other is not", () => { + expect(jaccardSimilarity("使用代码审查", "")).toBe(0); + }); + + it("returns 1 when both sides are empty", () => { + expect(jaccardSimilarity("", "")).toBe(1); + }); +}); + +describe("isNegationConflict", () => { + it("detects English do vs do-not", () => { + expect(isNegationConflict("always use strict mode", "do not use strict mode")).toBe(true); + }); + + it("detects English never vs affirmative", () => { + expect(isNegationConflict("use console.log for debugging", "never use console.log for debugging")).toBe(true); + }); + + it("does not flag both-negated pairs", () => { + expect(isNegationConflict("do not use var", "never use var")).toBe(false); + }); + + it("does not flag both-affirmative pairs", () => { + expect(isNegationConflict("use const", "use const everywhere")).toBe(false); + }); + + it("detects Chinese 不要 vs affirmative", () => { + expect(isNegationConflict("使用 var 声明变量", "不要使用 var 声明变量")).toBe(true); + }); + + it("detects Chinese 别 vs affirmative", () => { + expect(isNegationConflict("提交前运行测试", "别提交前运行测试")).toBe(true); + }); + + it("does not flag both-negated Chinese pairs", () => { + expect(isNegationConflict("不要使用 var", "别使用 var")).toBe(false); + }); + + it("does not flag both-affirmative Chinese pairs", () => { + expect(isNegationConflict("使用 const", "使用 const 声明")).toBe(false); + }); +}); \ No newline at end of file