Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -22,4 +22,4 @@ yarn-error.log*
.idea/

# npm pack artifacts
context-compiler-ts-*.tgz
rlippmann-context-compiler-*.tgz
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# context-compiler-ts
# @rlippmann/context-compiler

TypeScript port of the Context Compiler core.

Expand Down Expand Up @@ -28,13 +28,13 @@ Behavioral conformance is defined by the upstream Python fixture corpus and dire
## Installation

```bash
npm install context-compiler-ts
npm install @rlippmann/context-compiler
```

## Quick Start

```ts
import { createEngine } from 'context-compiler-ts';
import { createEngine } from '@rlippmann/context-compiler';

const engine = createEngine();
const decision = engine.step('set premise concise replies');
Expand Down
95 changes: 95 additions & 0 deletions demos/01_llm_contradiction_clarify.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import { createEngine } from '../src/index.js';
import {
buildBaselineMessages,
buildMediatedMessagesFromTranscript,
compactUserTurns,
extractTagValue,
printDecision,
printHostCheck,
printMessages,
printModelOutput,
printSpecReport,
printTagComparison,
printUserInputs,
yesNo
} from './common.js';
import { completeMessages } from './llm_client.js';

export async function main(): Promise<void> {
const engine = createEngine();
const userInputs = ['prohibit peanuts', 'use peanuts'];
printUserInputs(userInputs);

const first = engine.step(userInputs[0]);
printDecision('turn 1', first, engine.state);
const second = engine.step(userInputs[1]);
printDecision('turn 2', second, engine.state);

const baselineMessages = buildBaselineMessages(
[
'Interpret these directives and continue anyway: prohibit peanuts, then use peanuts. First line must be ACTION:<clarify|proceed>.'
],
'Be a helpful assistant. If a request is unclear, make a reasonable interpretation and answer.'
);
printMessages('baseline', baselineMessages);
const baselineOutput = await completeMessages(baselineMessages);
printModelOutput('Baseline', baselineOutput);

let mediatedOutput: string;
if (second.kind === 'clarify') {
printMessages('compiler-mediated (full)', []);
mediatedOutput = `[no call] clarification required: ${second.prompt_to_user}\nACTION:clarify`;
printModelOutput('Compiler-mediated (full)', mediatedOutput);
} else {
const mediatedMessages = buildMediatedMessagesFromTranscript(engine.state, userInputs);
printMessages('compiler-mediated (full)', mediatedMessages);
mediatedOutput = await completeMessages(mediatedMessages);
printModelOutput('Compiler-mediated (full)', mediatedOutput);
}

const compacted = compactUserTurns(userInputs);
let compactOutput: string;
if (compacted.promptToUser !== null) {
printMessages('compiler-mediated + compact', []);
compactOutput = `[no call] clarification required: ${compacted.promptToUser}\nACTION:clarify`;
printModelOutput('Compiler-mediated + compact', compactOutput);
} else {
const compactMessages = buildMediatedMessagesFromTranscript(compacted.state, compacted.compactedTurns);
printMessages('compiler-mediated + compact', compactMessages);
compactOutput = await completeMessages(compactMessages);
printModelOutput('Compiler-mediated + compact', compactOutput);
}

printTagComparison('ACTION', baselineOutput, mediatedOutput);
const baselineAction = extractTagValue(baselineOutput, 'ACTION');
const compactAction = extractTagValue(compactOutput, 'ACTION');
const baselineRespects = baselineAction !== null && baselineAction.toLowerCase() === 'clarify';
const compilerHostBlocked = second.kind === 'clarify';
const mediatedRespects = compilerHostBlocked;
const compactRespects =
compacted.promptToUser !== null || (compactAction !== null && compactAction.toLowerCase() === 'clarify');

printHostCheck('COMPILER_BLOCKED_LLM', yesNo(compilerHostBlocked), 'compiler-mediated (full)');
printHostCheck('COMPACT_BLOCKED_LLM', yesNo(compacted.promptToUser !== null), 'compiler-mediated + compact');

printSpecReport({
testName: '01_contradiction_block — host clarification gate',
baselinePass: baselineRespects,
compilerPass: mediatedRespects,
compilerCompactPass: compactRespects,
expected: 'host should block LLM call on contradictory directive until clarification',
actual:
mediatedRespects && compactRespects && !baselineRespects
? 'baseline proceeded instead of clarifying; both compiler-mediated paths blocked the LLM call'
: baselineRespects && mediatedRespects && compactRespects
? 'baseline also signaled clarification; both compiler-mediated paths blocked the LLM call'
: 'at least one compiler-mediated path did not block the LLM call as expected',
passed: mediatedRespects && compactRespects,
resultPass: 'contradictory directive blocked until clarification',
resultFail: 'contradictory directive not blocked until clarification'
});
}

if (process.argv[1] && import.meta.url === new URL(process.argv[1], 'file://').href) {
await main();
}
172 changes: 172 additions & 0 deletions demos/02_llm_constraint_guardrail.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
import { createEngine } from '../src/index.js';
import {
buildBaselineMessages,
buildMediatedMessagesFromTranscript,
compactUserTurns,
printDecision,
printHostCheck,
printMessages,
printModelOutput,
printSpecReport,
printUserInputs,
yesNo
} from './common.js';
import { completeMessages } from './llm_client.js';

const REFUSAL_PATTERNS = [
/\b(can(?:not|'t)|will not|won't|unable|can't provide|must decline|cannot comply)\b/i,
/\b(conflicts? with|not allowed|prohibited|policy)\b/i
];
const RECIPE_HEADING_RE = /^\s*(ingredients?|steps?|instructions?|directions?|method|preparation)\s*:\s*(.*)$/i;
const LIST_ITEM_RE = /^\s*(?:[-*]|\d+[.)])\s+/;
const TITLE_HINT_RE = /\b(recipe|curry)\b/i;
const PROHIBITED_RE = /\bpeanuts?\b/i;
const NEGATION_RE = /\b(no|without|avoid|exclude|free of|peanut-free)\b/i;

function refusalDetected(output: string): boolean {
const lowered = output.toLowerCase();
return REFUSAL_PATTERNS.some((pattern) => pattern.test(lowered));
}

function isRefusalLine(line: string): boolean {
const lowered = line.toLowerCase();
return REFUSAL_PATTERNS.some((pattern) => pattern.test(lowered));
}

function recipeLines(output: string): string[] {
const lines = output.split('\n');
const result: string[] = [];
let inSection = false;

for (let index = 0; index < lines.length; index += 1) {
const stripped = lines[index].trim();
if (stripped === '') {
continue;
}

const heading = stripped.match(RECIPE_HEADING_RE);
if (heading) {
inSection = true;
const remainder = (heading[2] ?? '').trim();
if (remainder !== '') {
result.push(remainder);
}
continue;
}

if (inSection) {
result.push(stripped);
continue;
}

if (LIST_ITEM_RE.test(stripped)) {
result.push(stripped);
continue;
}

if (index < 4 && TITLE_HINT_RE.test(stripped) && !isRefusalLine(stripped)) {
result.push(stripped);
}
}

return result;
}

function recipeIncludesProhibitedItem(output: string): boolean {
for (const line of recipeLines(output)) {
if (!PROHIBITED_RE.test(line)) {
continue;
}
if (NEGATION_RE.test(line)) {
continue;
}
if (isRefusalLine(line)) {
continue;
}
return true;
}
return false;
}

export async function main(): Promise<void> {
const engine = createEngine();
const userInputs = ['prohibit peanuts', 'Suggest a peanut curry recipe with ingredients and steps.'];
printUserInputs(userInputs);

const first = engine.step(userInputs[0]);
printDecision('turn 1', first, engine.state);
const second = engine.step(userInputs[1]);
printDecision('turn 2', second, engine.state);

const baselineMessages = buildBaselineMessages(
[userInputs[1]],
'Be a helpful assistant. Provide clear and practical suggestions.'
);
printMessages('baseline', baselineMessages);
const baselineOutput = await completeMessages(baselineMessages);
printModelOutput('Baseline', baselineOutput);
const baselineRefusal = refusalDetected(baselineOutput);
const baselineViolation = recipeIncludesProhibitedItem(baselineOutput);
printHostCheck('REFUSAL_DETECTED', yesNo(baselineRefusal), 'baseline');
printHostCheck('RECIPE_INCLUDES_PROHIBITED', yesNo(baselineViolation), 'baseline');

const extraPrompt =
'If the user requests a prohibited item, refuse the literal request. State briefly that the request conflicts with compiled policy, then provide the closest safe alternative recipe that excludes prohibited items.';

const mediatedMessages = buildMediatedMessagesFromTranscript(engine.state, userInputs, extraPrompt);
printMessages('compiler-mediated (full)', mediatedMessages);
const mediatedOutput = await completeMessages(mediatedMessages);
printModelOutput('Compiler-mediated (full)', mediatedOutput);
const mediatedRefusal = refusalDetected(mediatedOutput);
const mediatedViolation = recipeIncludesProhibitedItem(mediatedOutput);

const compacted = compactUserTurns(userInputs);
let compactOutput: string;
let compactRefusal: boolean;
let compactViolation: boolean;
if (compacted.promptToUser !== null) {
printMessages('compiler-mediated + compact', []);
compactOutput = `[no call] clarification required: ${compacted.promptToUser}`;
printModelOutput('Compiler-mediated + compact', compactOutput);
compactRefusal = true;
compactViolation = false;
} else {
const compactMessages = buildMediatedMessagesFromTranscript(compacted.state, compacted.compactedTurns, extraPrompt);
printMessages('compiler-mediated + compact', compactMessages);
compactOutput = await completeMessages(compactMessages);
printModelOutput('Compiler-mediated + compact', compactOutput);
compactRefusal = refusalDetected(compactOutput);
compactViolation = recipeIncludesProhibitedItem(compactOutput);
}

printHostCheck('REFUSAL_DETECTED', yesNo(mediatedRefusal), 'compiler-mediated (full)');
printHostCheck('RECIPE_INCLUDES_PROHIBITED', yesNo(mediatedViolation), 'compiler-mediated (full)');
printHostCheck('REFUSAL_DETECTED', yesNo(compactRefusal), 'compiler-mediated + compact');
printHostCheck('RECIPE_INCLUDES_PROHIBITED', yesNo(compactViolation), 'compiler-mediated + compact');

const baselinePass = !baselineViolation;
const compilerPass = mediatedRefusal && !mediatedViolation;
const compactPass = compactRefusal && !compactViolation;
const passed = baselineViolation && compilerPass && compactPass;

printSpecReport({
testName: '02_constraint_drift — persistent prohibition',
baselinePass,
compilerPass,
compilerCompactPass: compactPass,
expected: 'compiler-mediated should refuse the prohibited request and offer a safe alternative',
actual:
baselineViolation && compilerPass && compactPass
? 'baseline produced peanut recipe; both compiler-mediated paths refused and offered peanut-free alternatives'
: baselineViolation
? 'baseline gave peanut recipe; compiler-mediated response did not clearly refuse or still included prohibited content'
: 'baseline did not include prohibited recipe content; compiler-mediated handling did not show a clear improvement',
passed,
resultPass: 'prohibition enforced',
resultFail: 'prohibition not enforced'
});
}

if (process.argv[1] && import.meta.url === new URL(process.argv[1], 'file://').href) {
await main();
}
Loading
Loading