fix: CJK-aware remember dedup with negation guard (#1021)

src/functions/remember.ts

@@ -1,6 +1,7 @@
 import { TriggerAction, type ISdk } from "iii-sdk";
 import type { Memory } from "../types.js";
 import { KV, generateId, jaccardSimilarity } from "../state/schema.js";
+import { isNegationConflict } from "../state/memory-dedup.js";
 import { StateKV } from "../state/kv.js";
 import { withKeyedLock } from "../state/keyed-mutex.js";
 import { memoryToObservation } from "../state/memory-utils.js";
@@ -78,7 +79,7 @@ export function registerRememberFunction(sdk: ISdk, kv: StateKV): void {
             lowerContent,
             existing.content.toLowerCase(),
           );
-          if (similarity > 0.7) {
+          if (similarity > 0.7 && !isNegationConflict(lowerContent, existing.content.toLowerCase())) {
             supersededId = existing.id;
             supersededVersion = existing.version ?? 1;
             supersededMemory = existing;

src/state/memory-dedup.ts

@@ -0,0 +1,32 @@
+/**
+ * Negation guard for memory dedup.
+ *
+ * Prevents "do X" from superseding "do not X" (and vice versa) even when
+ * the two strings share enough tokens to exceed the Jaccard threshold.
+ *
+ * The guard is deliberately narrow and conservative:
+ * - Only triggers when one side has a negation marker and the other does not.
+ * - Supports English ("not", "never", "don't", "do not") and CJK
+ *   ("不要", "别", "無", "なし", "않") markers.
+ * - If BOTH sides have negation markers, they are treated as compatible
+ *   (both say "don't do X") and the guard does not fire.
+ */
+
+const EN_NEGATION = /\b(not|never|don't|do not|cannot|should not|must not|no)\b/i;
+const CJK_NEGATION = /不要|别|勿|無|无|不|なし|않|안|못/i;
+
+function hasNegation(text: string): boolean {
+  return EN_NEGATION.test(text) || CJK_NEGATION.test(text);
+}
+
+/**
+ * Returns true if the two texts appear to be in a negation conflict
+ * (one says do X, the other says do NOT do X) and should NOT be superseded.
+ */
+export function isNegationConflict(a: string, b: string): boolean {
+  const aNeg = hasNegation(a);
+  const bNeg = hasNegation(b);
+  // If both have negation or neither does, no conflict.
+  if (aNeg === bNeg) return false;
+  return true;
+}

src/state/schema.ts

@@ -1,4 +1,5 @@
 import { createHash } from "node:crypto";
+import { hasCjk, segmentCjk } from "./cjk-segmenter.js";
 
 export const KV = {
   sessions: "mem:sessions",
@@ -91,9 +92,52 @@ export function fingerprintId(prefix: string, content: string): string {
   return `${prefix}_${hash.slice(0, 16)}`;
 }
 
+/**
+ * CJK-aware tokenizer for Jaccard similarity.
+ *
+ * For English/whitespace-delimited text we keep the original behaviour:
+ * split on whitespace, filter tokens longer than 2 characters.
+ *
+ * For CJK text (Chinese/Japanese/Korean) whitespace splitting produces
+ * near-disjoint sets because CJK text is typically written without spaces.
+ * We reuse the existing `segmentCjk` function and additionally emit
+ * character bigrams from CJK runs so that no-space Chinese text still
+ * produces useful overlap.
+ */
+function tokenizeForJaccard(text: string): Set<string> {
+  // Fast path: no CJK characters → original whitespace split
+  if (!hasCjk(text)) {
+    return new Set(text.split(/\s+/).filter((t) => t.length > 2));
+  }
+
+  // CJK path: segment + add character bigrams for overlap
+  const segments = segmentCjk(text);
+  const tokens = new Set<string>();
+
+  for (const seg of segments) {
+    // Keep the whole segment as a token (works for word-segmented Chinese
+    // via jieba, and for English fragments in mixed text).
+    const cleaned = seg.replace(/[^\p{L}\p{N}]/gu, "").toLowerCase();
+    if (cleaned.length > 2) {
+      tokens.add(cleaned);
+    }
+
+    // Add CJK character bigrams from each segment so unsegmented CJK
+    // text can still produce meaningful overlap.
+    const cjkChars = seg.match(/[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu);
+    if (cjkChars && cjkChars.length >= 2) {
+      for (let i = 0; i < cjkChars.length - 1; i++) {
+        tokens.add(cjkChars[i] + cjkChars[i + 1]);
+      }
+    }
+  }
+
+  return tokens;
+}
+
 export function jaccardSimilarity(a: string, b: string): number {
-  const setA = new Set(a.split(/\s+/).filter((t) => t.length > 2));
-  const setB = new Set(b.split(/\s+/).filter((t) => t.length > 2));
+  const setA = tokenizeForJaccard(a);
+  const setB = tokenizeForJaccard(b);
   if (setA.size === 0 && setB.size === 0) return 1;
   if (setA.size === 0 || setB.size === 0) return 0;
   let intersection = 0;

test/remember-cjk-dedup.test.ts

@@ -0,0 +1,212 @@
+import { describe, it, expect, beforeEach, vi } from "vitest";
+
+vi.mock("../src/logger.js", () => ({
+  logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+vi.mock("../src/state/keyed-mutex.js", () => ({
+  withKeyedLock: <T>(_key: string, fn: () => Promise<T>) => fn(),
+}));
+
+vi.mock("../src/functions/audit.js", () => ({
+  recordAudit: vi.fn(),
+}));
+
+vi.mock("../src/functions/access-tracker.js", () => ({
+  recordAccessBatch: vi.fn(),
+  deleteAccessLog: vi.fn(),
+}));
+
+vi.mock("../src/config.js", () => ({
+  getAgentId: () => undefined,
+  isAgentScopeIsolated: () => false,
+}));
+
+import { registerRememberFunction } from "../src/functions/remember.js";
+import { getSearchIndex, setIndexPersistence } from "../src/functions/search.js";
+import { KV } from "../src/state/schema.js";
+
+function makeMockKV() {
+  const store = new Map<string, Map<string, unknown>>();
+  return {
+    get: async <T>(scope: string, key: string): Promise<T | null> =>
+      (store.get(scope)?.get(key) as T) ?? null,
+    set: async <T>(scope: string, key: string, data: T): Promise<T> => {
+      if (!store.has(scope)) store.set(scope, new Map());
+      store.get(scope)!.set(key, data);
+      return data;
+    },
+    delete: async (scope: string, key: string): Promise<void> => {
+      store.get(scope)?.delete(key);
+    },
+    list: async <T>(scope: string): Promise<T[]> => {
+      const entries = store.get(scope);
+      return entries ? (Array.from(entries.values()) as T[]) : [];
+    },
+  };
+}
+
+function makeMockSdk() {
+  const functions = new Map<string, Function>();
+  return {
+    registerFunction: (id: string, handler: Function) => {
+      functions.set(id, handler);
+    },
+    registerTrigger: () => {},
+    trigger: async (input: { function_id: string; payload: unknown }) => {
+      const fn = functions.get(input.function_id);
+      if (!fn) return {};
+      return fn(input.payload);
+    },
+  };
+}
+
+describe("mem::remember — CJK dedup", () => {
+  let sdk: ReturnType<typeof makeMockSdk>;
+  let kv: ReturnType<typeof makeMockKV>;
+
+  beforeEach(() => {
+    sdk = makeMockSdk();
+    kv = makeMockKV();
+    getSearchIndex().clear();
+    setIndexPersistence(null);
+    registerRememberFunction(sdk as never, kv as never);
+  });
+
+  it("supersedes a pure Chinese duplicate preference", async () => {
+    const first = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用代码审查来确保代码质量",
+        type: "preference",
+      },
+    }) as { memory: { id: string } };
+
+    const second = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用代码审查来确保代码质量",
+        type: "preference",
+      },
+    }) as { memory: { id: string; supersedes: string[] } };
+
+    expect(second.memory.supersedes).toContain(first.memory.id);
+
+    const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id);
+    expect(original?.isLatest).toBe(false);
+  });
+
+  it("supersedes a mixed Chinese/English duplicate preference", async () => {
+    const first = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "preference: 使用 eslint 进行代码检查",
+        type: "preference",
+      },
+    }) as { memory: { id: string } };
+
+    const second = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用 eslint 进行代码检查 preference",
+        type: "preference",
+      },
+    }) as { memory: { id: string; supersedes: string[] } };
+
+    expect(second.memory.supersedes).toContain(first.memory.id);
+  });
+
+  it("does not supersede related but distinct Chinese preferences", async () => {
+    const first = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用代码审查来确保质量",
+        type: "preference",
+      },
+    }) as { memory: { id: string } };
+
+    const second = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用持续集成来确保部署",
+        type: "preference",
+      },
+    }) as { memory: { id: string; supersedes: string[] } };
+
+    expect(second.memory.supersedes).toHaveLength(0);
+
+    const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id);
+    expect(original?.isLatest).toBe(true);
+  });
+
+  it("does not supersede negation conflict (English do vs do-not)", async () => {
+    const first = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "always use strict mode in JavaScript files",
+        type: "preference",
+      },
+    }) as { memory: { id: string } };
+
+    const second = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "do not use strict mode in JavaScript files",
+        type: "preference",
+      },
+    }) as { memory: { id: string; supersedes: string[] } };
+
+    expect(second.memory.supersedes).toHaveLength(0);
+
+    const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id);
+    expect(original?.isLatest).toBe(true);
+  });
+
+  it("does not supersede negation conflict (Chinese 不要 vs affirmative)", async () => {
+    const first = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用 var 声明变量",
+        type: "preference",
+      },
+    }) as { memory: { id: string } };
+
+    const second = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "不要使用 var 声明变量",
+        type: "preference",
+      },
+    }) as { memory: { id: string; supersedes: string[] } };
+
+    expect(second.memory.supersedes).toHaveLength(0);
+
+    const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id);
+    expect(original?.isLatest).toBe(true);
+  });
+
+  it("respects project isolation for CJK dedup", async () => {
+    const first = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用代码审查来确保质量",
+        type: "preference",
+        project: "api",
+      },
+    }) as { memory: { id: string } };
+
+    const second = await sdk.trigger({
+      function_id: "mem::remember",
+      payload: {
+        content: "使用代码审查来确保质量",
+        type: "preference",
+        project: "web",
+      },
+    }) as { memory: { id: string; supersedes: string[] } };
+
+    expect(second.memory.supersedes).toHaveLength(0);
+
+    const original = await kv.get<{ isLatest: boolean }>(KV.memories, first.memory.id);
+    expect(original?.isLatest).toBe(true);
+  });
+});