fix: align import validation and canonical serialization with python

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rlippmann/context-compiler",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "Deterministic TypeScript control layer for LLM apps. Compiles explicit user directives into authoritative context state before model calls.",
   "keywords": [
     "llm",

src/engine.ts

@@ -54,7 +54,7 @@ class EngineImpl implements Engine {
   }
 
   exportJson(): string {
-    return JSON.stringify(sortKeysDeep(this._state));
+    return stringifyCanonicalJson(sortKeysDeep(this._state));
   }
 
   importJson(payload: string): void {
@@ -301,12 +301,12 @@ export function getPremiseValue(state: EngineState): string | null {
 
 export function getPolicyItems(state: EngineState, value?: 'use' | 'prohibit' | null): string[] {
   if (value == null) {
-    return Object.keys(state.policies).sort((a, b) => a.localeCompare(b));
+    return Object.keys(state.policies).sort(compareStringsByCodepoint);
   }
   return Object.entries(state.policies)
     .filter(([, policy]) => policy === value)
     .map(([item]) => item)
-    .sort((a, b) => a.localeCompare(b));
+    .sort(compareStringsByCodepoint);
 }
 
 function initialState(): EngineState {
@@ -359,10 +359,14 @@ function loadStateObject(raw: unknown): EngineState {
     if (value !== 'use' && value !== 'prohibit') {
       throw new Error('Invalid state payload.');
     }
-    normalizedPolicies[normalizeItem(key)] = value;
+    const normalizedKey = normalizeItem(key);
+    if (normalizedKey === '') {
+      throw new Error('Invalid state payload.');
+    }
+    normalizedPolicies[normalizedKey] = value;
   }
 
-  const sortedEntries = Object.entries(normalizedPolicies).sort(([a], [b]) => a.localeCompare(b));
+  const sortedEntries = Object.entries(normalizedPolicies).sort(([a], [b]) => compareStringsByCodepoint(a, b));
   const sortedPolicies: Record<string, 'use' | 'prohibit'> = {};
   for (const [key, value] of sortedEntries) {
     sortedPolicies[key] = value;
@@ -497,7 +501,7 @@ function diagnosticPolicyContainsHints(policies: Record<string, 'use' | 'prohibi
   if (probe === '') {
     return '';
   }
-  const matches = Object.keys(policies).filter((key) => key.includes(probe)).sort((a, b) => a.localeCompare(b));
+  const matches = Object.keys(policies).filter((key) => key.includes(probe)).sort(compareStringsByCodepoint);
   if (matches.length === 0) {
     return '';
   }
@@ -540,7 +544,7 @@ function sortKeysDeep(value: unknown): unknown {
     return value.map((v) => sortKeysDeep(v));
   }
   if (value !== null && typeof value === 'object') {
-    const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b));
+    const entries = Object.entries(value as Record<string, unknown>).sort(([a], [b]) => compareStringsByCodepoint(a, b));
     const out: Record<string, unknown> = {};
     for (const [k, v] of entries) {
       out[k] = sortKeysDeep(v);
@@ -550,4 +554,35 @@ function sortKeysDeep(value: unknown): unknown {
   return value;
 }
 
+function compareStringsByCodepoint(left: string, right: string): number {
+  const leftCodepoints = Array.from(left);
+  const rightCodepoints = Array.from(right);
+  const limit = Math.min(leftCodepoints.length, rightCodepoints.length);
+
+  for (let idx = 0; idx < limit; idx += 1) {
+    const leftCodepoint = leftCodepoints[idx].codePointAt(0) as number;
+    const rightCodepoint = rightCodepoints[idx].codePointAt(0) as number;
+    if (leftCodepoint < rightCodepoint) {
+      return -1;
+    }
+    if (leftCodepoint > rightCodepoint) {
+      return 1;
+    }
+  }
+
+  if (leftCodepoints.length < rightCodepoints.length) {
+    return -1;
+  }
+  if (leftCodepoints.length > rightCodepoints.length) {
+    return 1;
+  }
+  return 0;
+}
+
+function stringifyCanonicalJson(value: unknown): string {
+  return JSON.stringify(value).replace(/[\u0080-\uFFFF]/g, (char) =>
+    `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`
+  );
+}
+
 export type { Decision, EngineState, TranscriptResult };

tests/import_json_atomicity.test.ts

@@ -0,0 +1,23 @@
+import { describe, expect, it } from 'vitest';
+import { createEngine } from '../src/index.js';
+
+describe('importJson atomicity', () => {
+  it('importJson is atomic when encountering invalid normalized keys', () => {
+    const engine = createEngine();
+    engine.importJson('{"premise":null,"policies":{"Docker":"use"},"version":2}');
+
+    const snapshot = JSON.parse(JSON.stringify(engine.state));
+    const invalidPayload = '{"premise":null,"policies":{"Docker":"use","a":"use"},"version":2}';
+
+    expect(() => engine.importJson(invalidPayload)).toThrowError('Invalid state payload.');
+    expect(engine.state).toEqual(snapshot);
+  });
+
+  it('no partial policy insertion before failure', () => {
+    const engine = createEngine();
+    const invalidPayload = '{"premise":null,"policies":{"Docker":"use","a":"use"},"version":2}';
+
+    expect(() => engine.importJson(invalidPayload)).toThrowError('Invalid state payload.');
+    expect(engine.state.policies).toEqual({});
+  });
+});

tests/serialization_parity.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, it } from 'vitest';
+import { createEngine, getPolicyItems } from '../src/index.js';
+
+describe('serialization parity', () => {
+  it('orders policy keys by codepoint and escapes non-ascii like Python', () => {
+    const engine = createEngine();
+    engine.importJson('{"premise":null,"policies":{"ä":"use","z":"use"},"version":2}');
+
+    expect(getPolicyItems(engine.state)).toEqual(['z', 'ä']);
+    expect(engine.exportJson()).toBe('{"policies":{"z":"use","\\u00e4":"use"},"premise":null,"version":2}');
+  });
+
+  it('keeps export/import round-trip stable for non-ascii policy keys', () => {
+    const engine = createEngine();
+    engine.importJson('{"premise":null,"policies":{"ä":"use","z":"use"},"version":2}');
+
+    const first = engine.exportJson();
+    const restored = createEngine();
+    restored.importJson(first);
+    const second = restored.exportJson();
+
+    expect(second).toBe(first);
+  });
+
+  it('produces deterministic canonical export across repeated runs', () => {
+    const payload = '{"premise":null,"policies":{"ä":"use","z":"use","alpha":"prohibit"},"version":2}';
+
+    const firstEngine = createEngine();
+    firstEngine.importJson(payload);
+    const secondEngine = createEngine();
+    secondEngine.importJson(payload);
+
+    const first = firstEngine.exportJson();
+    const second = secondEngine.exportJson();
+
+    expect(first).toBe(second);
+    expect(first).toBe(
+      '{"policies":{"alpha":"prohibit","z":"use","\\u00e4":"use"},"premise":null,"version":2}'
+    );
+  });
+
+  it('rejects policy keys that normalize to empty during import', () => {
+    const cases = [
+      '{"premise":null,"policies":{"A":"use"},"version":2}',
+      '{"premise":null,"policies":{"a":"use"},"version":2}',
+      '{"premise":null,"policies":{"A":"use","a":"use"},"version":2}',
+      '{"premise":null,"policies":{"the":"use"},"version":2}'
+    ];
+
+    for (const payload of cases) {
+      const engine = createEngine();
+      expect(() => engine.importJson(payload)).toThrowError('Invalid state payload.');
+    }
+  });
+
+  it('continues to accept valid non-empty normalized policy keys', () => {
+    const engine = createEngine();
+    engine.importJson('{"premise":null,"policies":{"Docker":"use"},"version":2}');
+    expect(engine.exportJson()).toBe('{"policies":{"docker":"use"},"premise":null,"version":2}');
+  });
+});