From 28715d3b47d605cc5ef0857644c9e305da86163c Mon Sep 17 00:00:00 2001 From: "agon (KERN)" <292465531+KERN-Agon@users.noreply.github.com> Date: Sat, 18 Jul 2026 00:00:12 +0200 Subject: [PATCH 1/2] feat(runtime): link public sibling helpers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⚔️ Forged by [Agon](https://github.com/KERNlang/agon) Co-Authored-By: agon (KERN) <292465531+KERN-Agon@users.noreply.github.com> --- .../spec.md | 221 ++++++++++++++++ packages/core/src/runner-runtime-scope.ts | 29 ++- .../src/runtime-envelope/handler-entry.ts | 7 +- .../src/runtime-envelope/source-handler.ts | 5 +- .../runtime-envelope-source-handler.test.ts | 79 ++++++ .../tests/runtime-handler-helper-link.test.ts | 236 ++++++++++++++++++ .../tests/runtime-handler-public-api.test.ts | 22 ++ 7 files changed, 594 insertions(+), 5 deletions(-) create mode 100644 .Codex/specs/kern-5-r2-m3-31d-runtime-handler-helper-link/spec.md create mode 100644 packages/core/tests/runtime-handler-helper-link.test.ts diff --git a/.Codex/specs/kern-5-r2-m3-31d-runtime-handler-helper-link/spec.md b/.Codex/specs/kern-5-r2-m3-31d-runtime-handler-helper-link/spec.md new file mode 100644 index 000000000..9baff671f --- /dev/null +++ b/.Codex/specs/kern-5-r2-m3-31d-runtime-handler-helper-link/spec.md @@ -0,0 +1,221 @@ +# KERN 5 R2 M3.31d — Public Runtime-Handler Sibling Helper Link + +**Status:** IMPLEMENTED AND VERIFIED +**Date:** 2026-07-17 +**Confidence:** 0.99 after implementation, full fitness wall, and clean terminal review +**Parent objective:** close the proven public-handler helper-link prerequisite for M4.1 + +## Executive Summary + +The public typed runtime-handler entry currently parses a complete KERN source +module but extracts only the selected handler body and parameter names. Its +fresh execution environment receives no sibling function scope. A direct body +succeeds, while the same body calling a valid sibling KERN helper fails with +`unsupported-runtime-input` before execution. + +M3.31d links the selected public handler to an authenticated, function-only +single-module runner scope built from that same parsed source. The scope is +installed on the per-invocation owned semantic environment before machine +admission. Reachable sibling helpers can then execute through the existing +canonical helper graph, snapshot, cache, recursion, and iteration contracts. + +This slice does not add imports, classes, module loading, public ABI fields, +fallback execution, or new helper semantics. It only stops discarding already +valid same-source KERN helper declarations at the public handler boundary. + +## Current State and Root Cause + +- **VERIFIED:** `linkedEntry` parses the whole source and selects one `fn`, but + returns only `body`, `parameters`, `identity`, and `signature`; sibling roots + are discarded. Evidence: `runtime-envelope/source-handler.ts`. +- **VERIFIED:** `handlerEnvironment` creates an owned environment for arguments + but does not install `runnerFunctions` or `runnerClasses`. Evidence: + `runtime-envelope/handler-entry.ts`. +- **VERIFIED:** the ordinary source runner already builds an authenticated + single-module scope whose function bindings retain defining-module identity. + Evidence: `runner-runtime-scope.ts`. +- **VERIFIED:** the canonical helper graph, preflight, snapshot, cache, + recursion, loop-budget, and failure containment already operate when an owned + `runnerFunctions` scope is present. +- **VERIFIED RED:** a public handler that directly returns its list argument + succeeds; an otherwise identical handler that calls a sibling identity helper + fails with `unsupported-runtime-input` and no result/events. +- **VERIFIED:** M4.1's KERN-authored canonicalizer needs same-source helpers for + bounded table validation, quoting, type rendering, and expression recursion. + +## Claim Ledger + +| Claim | Status | Evidence or oracle | +| --- | --- | --- | +| Public runtime handlers already link sibling helpers | REJECTED | direct/helper differential RED probe | +| Same-source helper declarations may be linked without public ABI change | VERIFIED | private linked-entry scope plus public envelope parity | +| Host JavaScript may execute a helper as fallback | REJECTED | machine-only runtime envelope remains unchanged | +| Imports or module loaders are part of M3.31d | REJECTED | source handler already rejects import roots | +| Classes are part of M3.31d | REJECTED | function-only scope; class roots remain inert/unreachable | +| Unreachable valid helper bodies may affect execution | REJECTED | graph analysis/execution remains reachability-based; module metadata snapshot is broader | +| Duplicate admitted sibling helper names may link ambiguously | REJECTED | deterministic link failure before effects | +| Runtime-handler request/envelope ABI changes | REJECTED | no new public fields, tags, diagnostics, or options | +| M3.31d makes siblings newly targetable by `handlerName` | REJECTED | current linker already selects any matching valid top-level function | +| `makeEnv` clones the installed runner-function map | REJECTED | verified `ownPlainMap` preserves the exact map object | + +## Contract + +| Surface | M3.31d contract | Failure rule | +| --- | --- | --- | +| Source authority | selected handler and helpers come from one parsed request source | no external file/module resolution | +| Helper set | valid top-level synchronous `fn` declarations with one KERN handler and non-void declared return | invalid declarations are not callable | +| Scope | fresh authenticated function-only root scope per public invocation | no class registry is installed | +| Reachability | only helpers transitively called from the selected handler are analyzed/executed; valid installed module metadata may be snapshotted as a whole | unreachable bodies do not run or add effects | +| Identity | helper lookup and recursion use linker binding identity; `env.runnerFunctions === scope.functions` | copied or caller-forged maps are forbidden | +| Arguments/results | existing portable scalar/list/record rules | no ABI broadening | +| Effects | existing helper graph contract; direct helper capabilities/print remain rejected | no host fallback or partial events | +| Loops/recursion | existing bounded iteration and helper recursion contracts | existing deterministic rejection applies | +| Snapshot | parsed helper bodies and the installed valid function scope freeze before execution | request/source mutation cannot affect an admitted run | +| Isolation | scope, bindings, cache, stack, arrays, scheduler, and capabilities are per invocation | overlapping calls share no mutable helper state | +| Sync/async | immediately resolved behavior and envelopes stay byte-identical | divergence fails the gate | +| Diagnostics | existing link/execution diagnostic taxonomy only | no new public diagnostic code | + +## Selected Design + +### 1. Function-only scope constructor + +Add a focused internal constructor beside `buildSingleModuleRunnerRootScope`. +It collects valid runner functions from the parsed document, removes the +selected public entry from the helper set, allocates one `RunnerModuleScope` +with an empty class map, clones each remaining binding with `module: scope`, and +marks that scope for machine ownership. Existing source-runner construction +continues to include classes and is unchanged. + +The constructor deliberately excludes classes rather than building the broad +source-runner scope and hoping class paths stay unreachable. This makes the +M3.31d boundary explicit and prevents accidental public class promotion. +Before allocation it scans top-level class names and rejects a callable-name +collision with an admitted sibling function without installing the class. + +### 2. Private linked-entry scope + +Extend the internal linked handler entry—not the public request—with the fresh +function scope. `linkedEntry` creates it from the same parse result after the +selected handler/signature checks. Imports remain link-rejected exactly as now. + +Duplicate admitted helper bindings fail link deterministically. Malformed or +unsupported function declarations remain absent from the callable scope as in +the existing runner collector and cannot shadow an admitted binding; both source +orderings are regression-tested. The selected public entry never appears in the +helper scope, including when its name is not legacy `main`, so M3.31d does not +introduce selected-entry self-recursion or let a sibling call back into the +external entry. Sibling helper self/mutual recursion remains governed by the +existing bounded helper contract. + +Public `handlerName` dispatch is unchanged: before M3.31d, any valid top-level +function can already be selected directly as the request entry. The new scope +does not add an entry namespace or make previously non-selectable declarations +selectable; it only changes what a selected entry may call. + +### 3. Owned invocation environment + +`handlerEnvironment` receives the internal entry and passes the scope's exact +function/class maps to `makeEnv`. `ownPlainMap` is verified to return that exact +map object; the implementation must not clone, spread, or reconstruct it after +`markRunnerMachineRootScope`. Argument bindings are then installed using the +existing ownership helpers. The environment keeps its own call cache and call +stack. Machine admission authenticates and snapshots the linked scope before +any effect. No admission predicate or ownership check is relaxed. + +Manual/internal handler entries without a linked scope keep current behavior. +There is no global registry and no reuse across requests. + +## Alternatives + +### A. Inline M4.1 into one enormous handler + +Rejected. It evades the proven handler-language ownership gap, removes reusable +KERN functions, and makes the canonicalizer harder to audit without fixing the +runtime contract KERN 5 actually needs. + +### B. Execute helpers through the compatibility/reference runner + +Rejected. The public runtime envelope is machine-only and must not acquire a +silent host fallback. + +### C. Install the full class/function source-runner scope + +Rejected for this slice. It would implicitly promote public class construction +and class effects without a dedicated contract or RED corpus. + +### D. Reparse or dynamically look up a helper on every call + +Rejected. It loses one-shot link identity, snapshot guarantees, and bounded +graph preflight. + +### E. Copy the marked function map into the handler environment + +Rejected. Machine ownership is keyed by exact scope/map identity. The marked +scope map is installed unchanged; a paired negative test proves a raw or copied +caller map still fails admission. + +## Blast Radius + +| Path | Action | Purpose | +| --- | --- | --- | +| `runner-runtime-scope.ts` | add focused function-only scope builder | reuse canonical binding/link ownership | +| `runtime-envelope/source-handler.ts` | modify private linked entry | retain same-source helper scope | +| `runtime-envelope/handler-entry.ts` | modify private environment construction | install per-run scope before admission | +| `runtime-handler-public-api.test.ts` | add RED/GREEN public contract cases | direct/helper, containment, parity, isolation | +| focused source-handler tests | extend if needed | duplicate/unreachable/class/import boundary | +| release train/spec | update after proof | record prerequisite without closing M4/R2 | + +No public type export, request/envelope shape, package export, capability API, or +diagnostic code changes. + +## Acceptance Criteria + +- [x] Full-roster Agon tribunal completed 3/3 as + `tribunal-1784290979470-yi1m9d-kern-5-r2-m3-31d-prebuild`; its real + ownership/link-contract risks are incorporated. Two claims were rejected + against current code: `makeEnv` preserves map identity, and sibling direct + selection already exists. +- [x] RED proves direct public execution succeeds while a sibling helper call + fails `unsupported-runtime-input` on the current branch. +- [x] A public handler calls one and multiple transitive same-source helpers in + authored argument order with correct scalar/list results. +- [x] Helper loops, self-recursion, and mutual recursion retain existing + iteration/recursion bounds; the selected public entry is not self-callable + through the helper scope and wrong arity rejects. +- [x] Direct helper capability/print/unsupported shape rejects before effects; + no compatibility/reference path executes. +- [x] Unreachable valid effectful helper bodies do not execute, reject, or emit + events; malformed declarations remain non-callable and non-shadowing. +- [x] Duplicate admitted helper names fail during link before effects in either + source order; malformed+valid same-name declarations follow the existing + collector contract deterministically. +- [x] Class/function callable-name collision rejects, class construction stays + unsupported, and imports remain link-rejected. +- [x] The exact marked function map reaches the environment unchanged; a copied + or raw caller-provided map still fails machine admission. +- [x] Sync and immediately resolved async envelopes are byte-identical for + scalar, transitive, rejected-side-effect, recursion, and loop-limit cases. +- [x] Overlapping invocations isolate helper scope, arrays, cache, stack, + scheduler, capabilities, and results; full installed metadata snapshot + cannot be mutated across invocations. +- [x] Public request/envelope shapes, diagnostics, package declarations, and + existing direct `handlerName` target selection remain unchanged. +- [x] Existing public runtime-handler ABI tests pass unchanged. +- [x] Focused runtime/source-runner gates and full `pnpm fitness:kern-5` pass. +- [x] Terminal `agon review -e claude,codex,agy` has no unresolved material + finding before commit: + `review-1784294556508-m2mp6g-kern-5-r2-m3-31d-terminal-v2` completed 3/3 + with zero findings. + +## Confidence Dependencies + +- **0.99:** root cause—the linked entry and environment visibly discard sibling + functions, and the direct/helper differential reproduces it. +- **0.99:** exact ownership path—`ownPlainMap` preserves the marked map identity; + paired positive/negative admission tests prevent a trust relaxation. +- **0.97:** reuse of the canonical scope/helper graph is correct—depends on + function-only construction retaining current ownership markers and identity. +- **0.96:** no accidental behavior broadening—depends on selected-entry removal, + explicit empty class scope, collision/import, and unreachable-body tests. +- **Overall: 0.99** after tribunal amendments, source verification, the full + fitness wall, and a clean terminal review. diff --git a/packages/core/src/runner-runtime-scope.ts b/packages/core/src/runner-runtime-scope.ts index bb30b31a0..4343ae553 100644 --- a/packages/core/src/runner-runtime-scope.ts +++ b/packages/core/src/runner-runtime-scope.ts @@ -1,4 +1,4 @@ -import { isPortableBindingName } from './ir/semantics/portable-scalar.js'; +import { isPortableBindingName } from './ir/semantics/portable-scalar-domain.js'; import { markRunnerMachineClassBinding, markRunnerMachineRootScope } from './ir/semantics/runner-machine-scope.js'; import type { RunnerClassBinding, @@ -161,10 +161,10 @@ function runnerFunctionBinding(node: IRNode): RunnerFunctionBinding | undefined } } -export function collectRunnerFunctions(root: IRNode): Map { +function collectRunnerFunctionsExcluding(root: IRNode, excludedName: string): Map { const functions = new Map(); for (const node of topLevelNodes(root)) { - if (node.type !== 'fn' || node.props?.name === 'main') continue; + if (node.type !== 'fn' || node.props?.name === excludedName) continue; const binding = runnerFunctionBinding(node); if (!binding) continue; if (functions.has(binding.name)) throw new KernRunnerError(`duplicate runner function '${binding.name}'`); @@ -173,6 +173,10 @@ export function collectRunnerFunctions(root: IRNode): Map { + return collectRunnerFunctionsExcluding(root, 'main'); +} + export function assertRunnerClassAcyclic(classes: ReadonlyMap): void { for (const cls of classes.values()) { const seen = new Set(); @@ -312,3 +316,22 @@ export function buildSingleModuleRunnerRootScope(root: IRNode): RunnerModuleScop markRunnerMachineRootScope(scope); return scope; } + +/** Build the owned sibling-function scope used by one public runtime-handler entry. */ +export function buildSingleModuleRunnerFunctionScope(root: IRNode, selectedEntry: string): RunnerModuleScope { + const rawFunctions = collectRunnerFunctionsExcluding(root, selectedEntry); + const classNames = new Set( + topLevelNodes(root) + .filter((node) => node.type === 'class' && isPortableBindingName(node.props?.name)) + .map((node) => node.props?.name as string), + ); + for (const name of [selectedEntry, ...rawFunctions.keys()]) { + if (classNames.has(name)) { + throw new KernRunnerError(`runner class '${name}' conflicts with runner function '${name}'`); + } + } + const scope: RunnerModuleScope = { functions: new Map(), classes: new Map() }; + for (const [name, binding] of rawFunctions) scope.functions.set(name, { ...binding, module: scope }); + markRunnerMachineRootScope(scope); + return scope; +} diff --git a/packages/core/src/runtime-envelope/handler-entry.ts b/packages/core/src/runtime-envelope/handler-entry.ts index 2f3071391..081b29d1b 100644 --- a/packages/core/src/runtime-envelope/handler-entry.ts +++ b/packages/core/src/runtime-envelope/handler-entry.ts @@ -6,6 +6,7 @@ import { defineIntBinding, defineRecordBinding, makeEnv, + type RunnerModuleScope, type SemanticEnv, } from '../ir/semantics/semantic-env.js'; import type { IRNode } from '../types.js'; @@ -27,6 +28,7 @@ import { export interface InternalRuntimeHandlerEntry { readonly body: readonly IRNode[]; readonly parameters: readonly string[]; + readonly runnerScope?: RunnerModuleScope; } function recordArrayFieldsFromArgument(value: unknown): Set { @@ -116,6 +118,7 @@ function validatedArguments( } function handlerEnvironment( + entry: InternalRuntimeHandlerEntry, validated: ValidatedArguments, host: SemanticEnv, options: InternalRuntimeEnvelopeOptions | undefined, @@ -127,6 +130,8 @@ function handlerEnvironment( now: host.now, runnerCallCache: new Map(), runnerCallStack: [], + runnerClasses: entry.runnerScope?.classes, + runnerFunctions: entry.runnerScope?.functions, seed: host.seed, }); const interceptor = options?.capabilityInterceptor; @@ -155,7 +160,7 @@ function preparedEnvironment( const validated = validatedArguments(entry, args, options); if ('format' in validated) return validated; try { - return handlerEnvironment(validated, host, options); + return handlerEnvironment(entry, validated, host, options); } catch (error) { return normalizeInternalRuntimeFailure(error); } diff --git a/packages/core/src/runtime-envelope/source-handler.ts b/packages/core/src/runtime-envelope/source-handler.ts index 8d38b0301..3e941f694 100644 --- a/packages/core/src/runtime-envelope/source-handler.ts +++ b/packages/core/src/runtime-envelope/source-handler.ts @@ -1,6 +1,7 @@ import { isPortableBindingName } from '../ir/semantics/portable-scalar-domain.js'; -import type { SemanticEnv } from '../ir/semantics/semantic-env.js'; +import type { RunnerModuleScope, SemanticEnv } from '../ir/semantics/semantic-env.js'; import { parseDocumentWithDiagnostics } from '../parser.js'; +import { buildSingleModuleRunnerFunctionScope } from '../runner-runtime-scope.js'; import { inspectKernRuntimeHandlerSignature, type KernRuntimeHandlerSignature } from '../runtime-handler-contract.js'; import { validateSchema } from '../schema.js'; import { @@ -25,6 +26,7 @@ export interface InternalRuntimeSourceHandlerIdentity { export interface InternalRuntimeLinkedHandlerEntry extends InternalRuntimeHandlerEntry { readonly identity: InternalRuntimeSourceHandlerIdentity; + readonly runnerScope: RunnerModuleScope; readonly signature: KernRuntimeHandlerSignature; } @@ -104,6 +106,7 @@ function linkedEntry( body: handler.children ?? [], identity, parameters: signature.parameters.map(({ name }) => name), + runnerScope: buildSingleModuleRunnerFunctionScope(parsed.root, identity.handlerName), signature, }; } diff --git a/packages/core/tests/runtime-envelope-source-handler.test.ts b/packages/core/tests/runtime-envelope-source-handler.test.ts index 734775e80..39460461f 100644 --- a/packages/core/tests/runtime-envelope-source-handler.test.ts +++ b/packages/core/tests/runtime-envelope-source-handler.test.ts @@ -225,4 +225,83 @@ describe('internal source handler identity and link', () => { ); expect(calls).toBe(0); }); + + test('links one exact owned function-only scope and excludes the selected entry', () => { + const source = [ + 'class name=Box', + 'fn name=helper returns=string', + ' handler lang="kern"', + ' return value="\\"ready\\""', + 'fn name=answer returns=string', + ' handler lang="kern"', + ' return value="helper()"', + ].join('\n'); + const linked = resolveInternalRuntimeSourceHandler(source, identity, enabled); + if ('format' in linked) throw new Error(`expected linked entry, got ${linked.diagnostics[0]?.code}`); + + expect(linked.runnerScope.classes.size).toBe(0); + expect([...linked.runnerScope.functions.keys()]).toEqual(['helper']); + expect(linked.runnerScope.functions.has('answer')).toBe(false); + expect(linked.runnerScope.functions.get('helper')?.module).toBe(linked.runnerScope); + expect(makeEnv({ runnerFunctions: linked.runnerScope.functions }).runnerFunctions).toBe( + linked.runnerScope.functions, + ); + }); + + test('rejects duplicate callable helpers and class/function collisions during link', () => { + const answer = ['fn name=answer returns=string', ' handler lang="kern"', ' return value="helper()"']; + const helper = ['fn name=helper returns=string', ' handler lang="kern"', ' return value="\\"ok\\""']; + const duplicate = [...helper, ...helper, ...answer].join('\n'); + const collision = ['class name=helper', ...helper, ...answer].join('\n'); + for (const source of [duplicate, collision]) { + expect(executeInternalRuntimeSourceHandlerSync(source, identity, [], makeEnv(), enabled)).toMatchObject({ + diagnostics: [{ code: 'handler-link-error', phase: 'link' }], + events: [], + outcome: 'failure', + }); + } + }); + + test('keeps malformed helpers non-callable and non-shadowing in either source order', () => { + const answer = ['fn name=answer returns=string', ' handler lang="kern"', ' return value="helper()"']; + const valid = ['fn name=helper returns=string', ' handler lang="kern"', ' return value="\\"ok\\""']; + const ignored = [ + 'fn name=helper returns=string async=true', + ' handler lang="kern"', + ' return value="\\"ignored\\""', + ]; + for (const roots of [ + [...ignored, ...valid, ...answer], + [...valid, ...ignored, ...answer], + ]) { + expect(executeInternalRuntimeSourceHandlerSync(roots.join('\n'), identity, [], makeEnv(), enabled)).toMatchObject( + { + diagnostics: [], + outcome: 'success', + result: { presence: 'value', value: { tag: 'text', value: 'ok' } }, + }, + ); + } + }); + + test('does not promote sibling classes into the public helper scope', () => { + const source = [ + 'class name=Box', + ' method name=read returns=string', + ' handler lang="kern"', + ' return value="\\"class\\""', + 'fn name=helper returns=string', + ' handler lang="kern"', + ' let name=box value="new Box()"', + ' return value="box.read()"', + 'fn name=answer returns=string', + ' handler lang="kern"', + ' return value="helper()"', + ].join('\n'); + expect(executeInternalRuntimeSourceHandlerSync(source, identity, [], makeEnv(), enabled)).toMatchObject({ + diagnostics: [{ code: 'unsupported-runtime-input', phase: 'execution' }], + events: [], + outcome: 'failure', + }); + }); }); diff --git a/packages/core/tests/runtime-handler-helper-link.test.ts b/packages/core/tests/runtime-handler-helper-link.test.ts new file mode 100644 index 000000000..259cd49a8 --- /dev/null +++ b/packages/core/tests/runtime-handler-helper-link.test.ts @@ -0,0 +1,236 @@ +import { + executeKernRuntimeHandlerAsync, + executeKernRuntimeHandlerSync, + KERN_RUNTIME_HANDLER_ABI, + type KernRuntimeHandlerLimits, + type KernRuntimeHandlerRequest, +} from '../src/runtime-handler.js'; + +const limits: KernRuntimeHandlerLimits = { + maxBytes: 65_536, + maxCollectionLength: 64, + maxDepth: 16, + maxDiagnostics: 8, + maxEvents: 64, + maxStringBytes: 4_096, +}; +const syncOptions = { enabled: true, limits } as const; +const asyncOptions = { capabilityTimeoutMs: 100, enabled: true, limits } as const; +const identity = { handlerName: 'answer', sourcePath: 'app/helper-link.kern' } as const; + +function request(source: string, args: readonly unknown[] = []): KernRuntimeHandlerRequest { + return { abi: KERN_RUNTIME_HANDLER_ABI, arguments: args, identity, source }; +} + +function entry(parameters: readonly string[], returns: string, body: readonly string[]): string[] { + return [ + `fn name=answer returns=${returns}`, + ...parameters.map((parameter) => ` param ${parameter}`), + ' handler lang="kern"', + ...body.map((line) => ` ${line}`), + ]; +} + +async function expectParity(invocation: KernRuntimeHandlerRequest) { + const sync = executeKernRuntimeHandlerSync(invocation, syncOptions); + const asyncEnvelope = await executeKernRuntimeHandlerAsync(invocation, asyncOptions); + expect(asyncEnvelope).toEqual(sync); + expect(JSON.stringify(asyncEnvelope)).toBe(JSON.stringify(sync)); + return sync; +} + +describe('M3.31d public runtime-handler sibling helper link', () => { + test('preserves authored arguments through transitive scalar helpers', async () => { + const source = [ + 'fn name=join returns=string', + ' param name=left type=string', + ' param name=right type=string', + ' handler lang="kern"', + String.raw` return value="left + \":\" + right"`, + 'fn name=relay returns=string', + ' param name=first type=string', + ' param name=second type=string', + ' handler lang="kern"', + ' return value="join(second, first)"', + ...entry(['name=first type=string', 'name=second type=string'], 'string', [ + 'return value="relay(first, second)"', + ]), + ].join('\n'); + + const envelope = await expectParity(request(source, ['left', 'right'])); + expect(envelope).toMatchObject({ + diagnostics: [], + outcome: 'success', + result: { presence: 'value', value: { tag: 'text', value: 'right:left' } }, + }); + }); + + test('keeps list arguments inside the linked machine domain', async () => { + const source = [ + 'fn name=first returns=string', + ' param name=values type="string[]"', + ' handler lang="kern"', + ' return value="values[0]"', + ...entry(['name=values type="string[]"'], 'string', ['return value="first(values)"']), + ].join('\n'); + const envelope = await expectParity(request(source, [['zero', 'one']])); + expect(envelope).toMatchObject({ + diagnostics: [], + outcome: 'success', + result: { presence: 'value', value: { tag: 'text', value: 'zero' } }, + }); + }); + + test('does not make the selected entry callable as its own sibling helper', async () => { + const source = entry(['name=value type=string'], 'string', ['return value="answer(value)"']).join('\n'); + const envelope = await expectParity(request(source, ['ready'])); + expect(envelope).toMatchObject({ + diagnostics: [{ code: 'unsupported-runtime-input' }], + events: [], + outcome: 'failure', + }); + }); + + test('links main as an ordinary sibling when another public entry is selected', async () => { + const source = [ + 'fn name=main returns=string', + ' handler lang="kern"', + String.raw` return value="\"from-main\""`, + ...entry([], 'string', ['return value="main()"']), + ].join('\n'); + const envelope = await expectParity(request(source)); + expect(envelope).toMatchObject({ + diagnostics: [], + outcome: 'success', + result: { presence: 'value', value: { tag: 'text', value: 'from-main' } }, + }); + }); + + test('rejects a class colliding with the selected public entry before execution', async () => { + const source = ['class name=answer', ...entry([], 'string', [String.raw`return value="\"unreachable\""`])].join( + '\n', + ); + const envelope = await expectParity(request(source)); + expect(envelope).toMatchObject({ + diagnostics: [{ code: 'handler-link-error', phase: 'link' }], + events: [], + outcome: 'failure', + }); + }); + + test('rejects wrong helper arity before effects', async () => { + const source = [ + 'fn name=identity returns=string', + ' param name=value type=string', + ' handler lang="kern"', + ' return value="value"', + ...entry([], 'string', ['return value="identity()"']), + ].join('\n'); + const envelope = await expectParity(request(source)); + expect(envelope).toMatchObject({ + diagnostics: [{ code: 'unsupported-runtime-input' }], + events: [], + outcome: 'failure', + }); + }); + + test('retains bounded helper loops and recursion with sync/async parity', async () => { + const loops = [ + 'fn name=sum returns=number', + ' param name=count type=number', + ' handler lang="kern"', + ' let name=total value="0"', + ' for name=i from=0 to=count', + ' assign target=total op="+=" value="i"', + ' return value="total"', + ...entry(['name=count type=number'], 'number', ['return value="sum(count)"']), + ].join('\n'); + const loopSuccess = await expectParity(request(loops, [3])); + expect(loopSuccess).toMatchObject({ + diagnostics: [], + result: { presence: 'value', value: { tag: 'integer', value: '3' } }, + }); + const loopFailure = await expectParity(request(loops, [65])); + expect(loopFailure).toMatchObject({ + diagnostics: [{ code: 'unsupported-runtime-input' }], + events: [], + outcome: 'failure', + }); + + const recursion = [ + 'fn name=countdown returns=number', + ' param name=count type=number', + ' handler lang="kern"', + ' if cond="count <= 0"', + ' return value="0"', + ' return value="1 + countdown(count - 1)"', + ...entry(['name=count type=number'], 'number', ['return value="countdown(count)"']), + ].join('\n'); + const recursionSuccess = await expectParity(request(recursion, [5])); + expect(recursionSuccess).toMatchObject({ + diagnostics: [], + result: { presence: 'value', value: { tag: 'integer', value: '5' } }, + }); + const recursionFailure = await expectParity(request(recursion, [513])); + expect(recursionFailure).toMatchObject({ + diagnostics: [{ code: 'unsupported-runtime-input' }], + events: [], + outcome: 'failure', + }); + }); + + test('ignores unreachable effectful helpers but rejects reached helper effects before providers', async () => { + const helper = [ + 'fn name=effectful returns=string', + ' handler lang="kern"', + ' capability namespace=storage operation=get name=value', + ' print value="value"', + ' return value="value"', + ]; + const unreachable = [...helper, ...entry([], 'string', [String.raw`return value="\"safe\""`])].join('\n'); + const unreachableEnvelope = await expectParity(request(unreachable)); + expect(unreachableEnvelope).toMatchObject({ + diagnostics: [], + events: [], + outcome: 'success', + result: { presence: 'value', value: { tag: 'text', value: 'safe' } }, + }); + + let calls = 0; + const reached = [...helper, ...entry([], 'string', ['return value="effectful()"'])].join('\n'); + const invocation = request(reached); + const capabilities = { storage: { get: () => `${++calls}` } } as const; + const sync = executeKernRuntimeHandlerSync(invocation, { ...syncOptions, capabilities }); + const asyncEnvelope = await executeKernRuntimeHandlerAsync(invocation, { + ...asyncOptions, + asyncCapabilities: { storage: { get: async () => `${++calls}` } }, + }); + expect(asyncEnvelope).toEqual(sync); + expect(sync).toMatchObject({ + diagnostics: [{ code: 'unsupported-runtime-input' }], + events: [], + outcome: 'failure', + }); + expect(calls).toBe(0); + }); + + test('isolates linked scopes, arrays, and results across overlapping invocations', async () => { + const source = [ + 'fn name=pick returns=string', + ' param name=values type="string[]"', + ' handler lang="kern"', + ' return value="values[0]"', + ...entry(['name=values type="string[]"'], 'string', ['return value="pick(values)"']), + ].join('\n'); + const left = ['left']; + const right = ['right']; + const [leftEnvelope, rightEnvelope] = await Promise.all([ + executeKernRuntimeHandlerAsync(request(source, [left]), asyncOptions), + executeKernRuntimeHandlerAsync(request(source, [right]), asyncOptions), + ]); + expect(leftEnvelope.result).toEqual({ presence: 'value', value: { tag: 'text', value: 'left' } }); + expect(rightEnvelope.result).toEqual({ presence: 'value', value: { tag: 'text', value: 'right' } }); + expect(left).toEqual(['left']); + expect(right).toEqual(['right']); + }); +}); diff --git a/packages/core/tests/runtime-handler-public-api.test.ts b/packages/core/tests/runtime-handler-public-api.test.ts index 454a38643..d1bbb4943 100644 --- a/packages/core/tests/runtime-handler-public-api.test.ts +++ b/packages/core/tests/runtime-handler-public-api.test.ts @@ -35,6 +35,28 @@ const syncOptions = { enabled: true, limits } as const; const asyncOptions = { capabilityTimeoutMs: 100, enabled: true, limits } as const; describe('@kernlang/core/runtime/handler public ABI', () => { + test('links a selected handler to same-source sibling KERN helpers', async () => { + const helperSource = [ + 'fn name=decorate returns=string export=true', + ' param name=value type=string', + ' handler lang="kern"', + ' return value="value"', + '', + ...source(['name=value type=string'], 'string', ['return value="decorate(value)"']).split('\n'), + ].join('\n'); + const invocation = request(helperSource, ['ready']); + const sync = executeKernRuntimeHandlerSync(invocation, syncOptions); + const asyncEnvelope = await executeKernRuntimeHandlerAsync(invocation, asyncOptions); + expect(sync).toMatchObject({ + completion: { kind: 'return' }, + diagnostics: [], + events: [], + outcome: 'success', + result: { presence: 'value', value: { tag: 'text', value: 'ready' } }, + }); + expect(asyncEnvelope).toEqual(sync); + }); + test('puts the exact ABI on ingress and egress with sync/async byte parity', async () => { const invocation = request(source(['name=value type=string'], 'string', ['return value="value"']), ['ready']); const sync = executeKernRuntimeHandlerSync(invocation, syncOptions); From d35c38cfad3f4cd2fec5274e3a4b97dfb25598df Mon Sep 17 00:00:00 2001 From: "agon (KERN)" <292465531+KERN-Agon@users.noreply.github.com> Date: Sat, 18 Jul 2026 00:00:21 +0200 Subject: [PATCH 2/2] feat(kir): add KERN canonicalizer oracle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ⚔️ Forged by [Agon](https://github.com/KERNlang/agon) Co-Authored-By: agon (KERN) <292465531+KERN-Agon@users.noreply.github.com> --- .../kern-5-r2-m4-1-kir-canonicalizer/spec.md | 375 +++++++++++++ docs/kern-5-release-train.md | 39 ++ docs/kern-5-support-matrix.md | 2 + .../kern-canonicalizer/canonicalizer.kern | 438 ++++++++++++++++ package.json | 5 +- scripts/check-kern-canonicalizer.mjs | 164 ++++++ scripts/kern-5-fitness-policy.json | 16 +- scripts/kern-5-fitness.test.mjs | 23 +- .../kern-canonicalizer/canonicalizer.test.mjs | 95 ++++ scripts/kern-canonicalizer/fixtures.mjs | 492 ++++++++++++++++++ scripts/kern-canonicalizer/flatten.mjs | 177 +++++++ scripts/kern-canonicalizer/flatten.test.mjs | 223 ++++++++ scripts/kern-canonicalizer/policy.json | 31 ++ scripts/kern-canonicalizer/policy.mjs | 55 ++ .../profile-limit-fixtures.mjs | 61 +++ scripts/kern-canonicalizer/rehydrate.mjs | 284 ++++++++++ .../review-boundary-fixtures.mjs | 52 ++ .../semantic-boundary-fixtures.mjs | 76 +++ 18 files changed, 2604 insertions(+), 4 deletions(-) create mode 100644 .Codex/specs/kern-5-r2-m4-1-kir-canonicalizer/spec.md create mode 100644 examples/kern-canonicalizer/canonicalizer.kern create mode 100644 scripts/check-kern-canonicalizer.mjs create mode 100644 scripts/kern-canonicalizer/canonicalizer.test.mjs create mode 100644 scripts/kern-canonicalizer/fixtures.mjs create mode 100644 scripts/kern-canonicalizer/flatten.mjs create mode 100644 scripts/kern-canonicalizer/flatten.test.mjs create mode 100644 scripts/kern-canonicalizer/policy.json create mode 100644 scripts/kern-canonicalizer/policy.mjs create mode 100644 scripts/kern-canonicalizer/profile-limit-fixtures.mjs create mode 100644 scripts/kern-canonicalizer/rehydrate.mjs create mode 100644 scripts/kern-canonicalizer/review-boundary-fixtures.mjs create mode 100644 scripts/kern-canonicalizer/semantic-boundary-fixtures.mjs diff --git a/.Codex/specs/kern-5-r2-m4-1-kir-canonicalizer/spec.md b/.Codex/specs/kern-5-r2-m4-1-kir-canonicalizer/spec.md new file mode 100644 index 000000000..ff1311ace --- /dev/null +++ b/.Codex/specs/kern-5-r2-m4-1-kir-canonicalizer/spec.md @@ -0,0 +1,375 @@ +# KERN 5 R2 M4.1 — KERN-Authored KIR Canonicalizer Profile + +**Status:** IMPLEMENTED AND VERIFIED +**Date:** 2026-07-17 +**Confidence:** 0.99 after implementation, adversarial hardening, complete fitness proof, and six-engine review +**Parent objective:** begin R2 M4 formatter/frontend ownership without overstating a production formatter or KIR v1 freeze + +## Executive Summary + +M4.1 adds the first release-blocking KERN-authored canonicalizer over the current +typed structural KIR. A generic host adapter mechanically flattens decoded KIR +records into parallel primitive arrays. A KERN handler—not JavaScript—owns the +semantic decisions that turn the admitted profile into deterministic source: +node spelling, property ordering, type spelling, expression spelling, quoting, +indentation, and child order. + +The slice is deliberately an internal oracle. It accepts a bounded function +profile, emits trivia-free canonical source, proves byte-stable idempotence and +exact KIR round-trip equivalence, and fails closed for every unsupported shape. +It does not expose `kern format`, preserve comments, own tokenization/parsing, +freeze public KIR v1, or claim the full R2 formatter/frontend exit. + +## Current State + +- **VERIFIED:** the release train still marks formatter/canonicalizer, + KERN-authored frontend, compiler, fixed point, interpreter shadow, and packed + release incomplete. KERN 5.0 therefore is not achieved. +- **VERIFIED:** the structural KIR and module codecs are internal alpha formats, + with typed semantic records now carried for every portable runtime-handler + type admitted by R1.5e.1. +- **VERIFIED:** decoded structural nodes expose `kind`, canonical property + records, and authored `children`; expression values are canonical tagged + records rather than raw parser AST objects. +- **VERIFIED:** the public typed runtime-handler boundary accepts the scalar and + one-dimensional primitive array arguments required by a flattened KIR table. +- **VERIFIED:** KERN source can iterate arrays, recurse through helpers, inspect + text character-by-character, concatenate text, and throw before returning. +- **VERIFIED:** before this slice, no current root script or ownership receipt + proved a KERN-authored formatter/canonicalizer. +- **VERIFIED:** this bounded internal-oracle profile is the smallest honest step + that moves semantic source emission into KERN before source frontend work. + +## Claim Ledger + +| Claim | Status | Evidence or oracle | +| --- | --- | --- | +| KERN 5.0 is complete before M4.1 | REJECTED | release-train binary exit and remaining `not-shipped` ownership rows | +| Host code may decode KIR and mechanically flatten tagged records | VERIFIED | adapter contract plus cheat-kill mutations | +| Host code may choose source syntax, ordering, escaping, or indentation | REJECTED | semantic decisions belong to the KERN handler | +| The admitted profile can be emitted deterministically by KERN | VERIFIED | golden, shuffle, idempotence, and exact KIR byte oracles | +| The slice is a public production formatter | REJECTED | no CLI/API, trivia, recovery, or full grammar | +| The slice owns source parsing/tokenization | REJECTED | bootstrap TS parser remains the comparison oracle | +| The slice freezes public KIR v1 | REJECTED | current formats remain internal alpha | +| The slice earns a bounded canonicalizer-profile `internal-oracle` receipt | VERIFIED | distinct ownership row, terminal gate, and Agon review | +| The broad `kern-formatter` ownership row may be promoted by M4.1 | REJECTED | full-roster tribunal; production formatter exit remains open | + +## Admitted Profile + +The input is exactly one decoded module's ordered root list. M4.1 admits: + +- one or more top-level `fn` roots; +- function properties `name`, `returns`, and `export` only; +- zero or more direct `param` children, exactly one `handler lang=kern` + child, and a direct `return` inside that handler; +- exact handler types `boolean`, `integer`, `text`, `void`, and one-dimensional + lists of the three non-void scalar types; +- expression values `identifier`, `null`, `boolean`, non-negative `integer`, + `text`, and `list` only; +- text expression payloads containing Unicode scalar values except the + non-source-stable U+2028 line separator, U+2029 paragraph separator, and + U+FEFF byte-order mark, plus newline, carriage return, and tab; the other C0 + controls, DEL, and C1 controls remain outside this bounded profile; +- authored root, parameter, and list-item order. + +The checked-in canonicalizer policy further bounds one invocation to at most +16 node rows, 30 property rows, and 72 value rows. These caller-owned ceilings +are passed into and enforced by the KERN handler before its quadratic table +validation begins. The exact-boundary valid oracle reaches all 16/30/72 rows and must +complete within the separately configured 65,536-step runtime budget; each +table also has an explicit over-limit rejection oracle. Changing either the +profile ceilings or runtime budget requires changing the policy and renewing +the focused and aggregate fitness evidence. + +The same policy owns a 4x maximum KIR-to-source expansion factor and a 2x +runtime-envelope factor. Validation binds `runtimeLimits.maxStringBytes` to the +configured KIR byte ceiling and `runtimeLimits.maxBytes` to the expanded string +ceiling before any fixture runs. A source-derived fixture containing two legal +8,192-byte backslash strings proves escaping cannot exhaust the admitted +runtime envelope. + +Everything else—including an empty root list, unknown properties, duplicate +required properties, duplicate function or parameter names, unknown tags, +malformed table references, nested type +lists, records, maps, errors, decimals, expression-reserved literal/prefix +tokens presented as identifiers, unsupported C0 text controls, binary +expressions, calls, lambdas, +member/index access, conditionals, unary expressions, extra handler children, +negative integer expressions, or non-KERN handlers—fails before a result is +returned. + +The tribunal explicitly removed decimals and binaries so M4.1 does not import a +numeric spelling, precedence, associativity, or parenthesization contract by +reference. The profile may not be broadened during implementation without a new +claim-tagged spec and RED oracles. + +## Flattened Input Contract + +The runtime handler receives twelve typed primitive arrays plus three positive +integer policy arguments (`maxNodeRows`, `maxPropertyRows`, and +`maxValueRows`): + +| Array | Meaning | +| --- | --- | +| `nodeKind: string[]` | structural node kind by 1-based node id | +| `nodeParent: integer[]` | zero for roots, otherwise parent node id | +| `nodeOrder: integer[]` | authored sibling order | +| `propNode: integer[]` | owning structural node id | +| `propKey: string[]` | property key without syntax spelling | +| `propValue: integer[]` | canonical value id | +| `valueTag: string[]` | canonical value tag | +| `valueParent: integer[]` | zero for property roots, otherwise parent value id | +| `valueRole: string[]` | record key/list item/map key/map value/error-field role | +| `valueOrder: integer[]` | authored order within a composite value | +| `valueText: string[]` | text/integer/decimal payload; empty for composites | +| `valueBool: integer[]` | `1` or `0` for boolean; zero otherwise | + +All related arrays must have equal lengths within their table. IDs are dense, +one-based, and allocated in preorder so every non-root parent id precedes its +child id. The adapter preserves the decoded structure verbatim; it may allocate +IDs and copy tag/payload/parent/order facts but may not rename types, sort +properties, omit unknown facts, escape source text, or recognize function, +handler, parameter, return, type, or expression semantics. + +The value table stays lossless and self-describing for canonical value tags even +when the M4.1 KERN handler rejects a tag. Unsupported data is never silently +discarded by the adapter. + +The host proof rehydrates every flattened valid and hostile-but-structurally +representable KIR graph and requires deep equality with the decoded graph before +the KERN handler runs. A cycle/alias-detecting generic traversal rejects cyclic +or shared object graphs before id allocation; decoded canonical KIR is an owned +tree, so host alias identity is not an admissible transport fact. Exact record +and array inspection rejects symbol keys, accessors, non-enumerable facts, +sparse indexes, and extra keys. Static tests forbid admitted KERN node, property, +type, expression, indentation, quoting, or source-keyword literals in the +adapter. `valueRole` may encode only generic container relationships already +present in the canonical-value schema; it may not carry profile-specific hints. + +This is structural transport only. M4.1 performs no borrow, lifetime, +drop-order, or execution-scope analysis; preserving the lossless parent/child +graph is the entire scope obligation of this slice. + +## Output Contract + +The handler returns `string[]`, one canonical source line per item. Canonical +source has: + +- two spaces per structural depth; +- one ASCII space between a node kind and its properties; +- deterministic semantic property order selected by KERN; +- double-quoted escaped admitted text when a value is not an unquoted grammar + atom; +- non-negative integer spelling derived from the canonical KIR payload; +- no comments, blank lines, or trailing whitespace; +- a final newline supplied mechanically by the harness when joining lines. + +The handler constructs the complete result only after validating the full input +graph. Any invalid or unsupported fact throws; no partial line array is exposed. + +## Selected Design + +### 1. Generic host flattener + +`scripts/kern-canonicalizer/flatten.mjs` recursively assigns stable ids to the +decoded structural and canonical value records, detects object cycles and +shared references, and can +be reversed by `rehydrate.mjs` to the same generic KIR graph. It validates only +generic table integrity and KIR codec shape, including canonical scalar text +and record/map key ordering. It contains no admitted KERN +node/property/type/expression names and no source formatting strings. + +### 2. KERN semantic canonicalizer + +`examples/kern-canonicalizer/canonicalizer.kern` validates the complete table, +rejects rows beyond the three caller-owned profile ceilings before quadratic +validation, indexes node/property/value relations, checks the exact admitted +profile, and emits canonical source. Source syntax constants and all +ordering/escaping rules live in this KERN file. The hand-written file remains +below 500 lines; focused KERN helpers are split only if necessary. + +### 3. Differential release oracle + +`scripts/check-kern-canonicalizer.mjs` runs the following pipeline per fixture: + +1. parse source with the bootstrap frontend; +2. encode then decode module KIR; +3. mechanically flatten the decoded roots; +4. execute the KERN canonicalizer through the public typed runtime handler; +5. join returned lines into canonical source; +6. parse and encode that output and require the exact semantic structural module + artifact emitted by `encodeModuleKir` to equal the original artifact byte for + byte; source spans, file paths, diagnostics, evidence hashes, and trivia are + not members of that artifact; +7. canonicalize the output again and require byte-identical canonical source; +8. require exact source-golden equality. + +Fixtures include shuffled nonsemantic property order, multiple roots, multiple +parameters whose order must not be sorted, every admitted exact type, +expression-valid keyword-shaped identifiers, escaped text, nested lists, and +every admitted literal. Hostile fixtures are an audited matrix: empty roots, +table-length mismatch, non-dense ids, invalid parent ids, node cycles, value +cycles, duplicate sibling order, duplicate required properties, orphan values, +noncanonical scalar/map shapes, expression-reserved identifiers, retained map +and error values, every non-admitted C0/DEL/C1 text control, unknown retained +tags/properties, malformed role/order, unsupported child kind, and a valid +prefix followed by an invalid suffix. Each must reject without returning +partial source. + +Harness failures use stable categories rather than incidental exception text: +`adapter rejection`, `profile rejection`, `parse rejection`, `KIR mismatch`, +`golden mismatch`, and `idempotence mismatch`. These are internal test +categories, not a new public diagnostics API. The fixed-point assertion runs for +every valid fixture, not a representative subset. + +### 4. Release-policy receipt + +Add `test:kern-canonicalizer` as a current release gate. The gate builds core, +runs `@kernlang/check` acceptance for the new KERN source, runs focused table +tests, and executes the differential oracle. Add a distinct ownership row +`kern-kir-canonicalizer-profile` with status `internal-oracle` and evidence +`pnpm test:kern-canonicalizer`. Terminal review evaluates the prospective exact +tree before publication; review-discovered defects require RED regressions, +renewed gates, and a new exact-tree review. Committing the final reviewed +candidate publishes the earned receipt. Keep the existing broad +`kern-formatter` row unchanged at `not-shipped` / `R2 planned`. + +This receipt means only that KERN-authored semantic canonicalization logic exists +inside a release-blocking differential harness. `kern-frontend`, compiler, +fixed-point, interpreter-shadow, and packed-release rows remain `not-shipped`. + +## Cheat-Kill Matrix + +| Cheat | Oracle that kills it | +| --- | --- | +| return input source unchanged | handler never receives source text | +| host pretty-printer owns syntax | static adapter scan plus semantic mutation/golden tests | +| adapter smuggles normalized semantics | flatten→rehydrate deep equality and forbidden-literal scan | +| compare formatted source only | exact encoded KIR byte equality | +| normalize by sorting every child | two-parameter, two-root, and nested-list order fixtures | +| ignore unsupported nodes/properties | hostile unknown-fact fixtures must reject | +| silently omit malformed references | dense-id/parent/table-integrity rejection fixtures | +| crash accidentally instead of fail closed | stable adapter/profile error-category assertions | +| formatter works once but drifts | second-pass byte-identical source oracle | +| delegate to another runtime command | source/convergence scan and runtime-handler-only invocation | +| claim public KIR or frontend ownership | policy/matrix/release-train assertions keep those rows incomplete | + +## Alternatives + +### A. Build a production source formatter first + +Rejected for M4.1. Comment/trivia preservation, recovery, full grammar coverage, +and public CLI/API design would combine formatter, frontend, and product surface +in one unverifiable slice. + +### B. Let JavaScript recursively render decoded KIR + +Rejected. That would test a host formatter while KERN merely returns or joins +preselected strings; it does not move semantic ownership. + +### C. Pass JSON text to KERN + +Rejected. It adds a JSON frontend dependency and permits host-selected schema +spelling at the ownership boundary. Primitive parallel tables use the already +proven runtime ABI. + +### D. Promote the existing `kern-formatter` row + +Rejected by the full-roster tribunal. Even though its label says "formatter or +canonicalizer," changing the broad row would make a bounded KIR-backend profile +look like release formatter readiness. M4.1 adds a distinct internal-oracle row +and leaves the binary-exit row untouched. + +## Blast Radius + +| Path | Action | Purpose | +| --- | --- | --- | +| `examples/kern-canonicalizer/canonicalizer.kern` | add | KERN-owned validation and source emission | +| `scripts/kern-canonicalizer/flatten.mjs` | add | generic lossless KIR-to-table adapter | +| `scripts/kern-canonicalizer/rehydrate.mjs` | add | independent table-to-KIR losslessness oracle | +| `scripts/kern-canonicalizer/fixtures.mjs` | add | source/golden/hostile corpus | +| `scripts/kern-canonicalizer/*.test.mjs` | add | adapter and cheat-kill mutations | +| `scripts/check-kern-canonicalizer.mjs` | add | differential/idempotence release oracle | +| `package.json` | modify | root test script and infra wall | +| fitness policy/test and support matrix | modify | exact gate and bounded ownership receipt | +| release train | modify after proof | record M4.1 without closing broad M4/R2 | + +No public package export, CLI surface, parser, tokenizer, compatibility runtime, +or KIR format identifier changes in this slice. + +## Acceptance Criteria + +- [x] Claim-tagged spec and full-roster Agon tribunal complete before code; run + `tribunal-1784289697339-9t9aes-kern-5-r2-m4-1-prebuild` succeeded 3/3 and + its NO-GO risks were incorporated before implementation. +- [x] RED oracle fails on the current branch because no KERN canonicalizer or + `test:kern-canonicalizer` gate exists. +- [x] Adapter is generic and lossless; it contains no admitted-profile semantic + node/property/type/expression decisions, and flatten→rehydrate deep + equality holds before handler invocation. +- [x] KERN owns validation, semantic ordering, syntax spelling, escaping, + indentation, expression rendering, and complete-result construction. +- [x] Every valid fixture produces exact golden canonical source. +- [x] `format(format(x))` is byte-identical for every valid fixture. +- [x] Encoded module KIR before and after formatting is byte-identical. +- [x] Shuffled property input converges while root/child/list order is + preserved. +- [x] Every enumerated unsupported or malformed graph fixture rejects with no + partial source result and the expected stable error category. +- [x] Config-owned 16/30/72 row ceilings reach the KERN handler, an exact + 16/30/72 valid fixture completes inside the configured runtime budget, + and each table rejects above its ceiling without partial output. +- [x] `@kernlang/check` accepts the KERN source. +- [x] `pnpm test:kern-canonicalizer` is a current fitness gate and the support + matrix exactly matches the policy. +- [x] New `kern-kir-canonicalizer-profile` ownership language remains + `internal-oracle`; `kern-formatter`, public frontend, KIR v1, compiler, + fixed point, interpreter, and packed release stay open. +- [x] Focused gate and full `pnpm fitness:kern-5` pass on Node 22 with 11 valid + fixtures, 3 otherwise-valid profile-limit fixtures, 105 hostile fixtures, + and the complete repository wall, including the prefix-complete + runtime-ABI gate. The renewed complete wall passed on 2026-07-17; the + policy-derived escaped-output fixture then passed the focused gate + byte-identically. +- [x] Terminal `agon review` with the current six-engine usable roster has no unresolved material + findings. Exact-tree review + `review-1784301738713-whskgx-kern-5-r2-m4-1-terminal-final` found the + `$` structural-name round-trip blocker; its RED regression and KERN-owned + conditional quoting fix passed the renewed complete wall. Expanded-roster + review `review-1784308684184-pq6a0r-kern-5-r2-m4-1-dollar-fix-termin` + found the missing non-string parameter coverage and the runtime-ABI + public-entry omission; both passed the renewed complete wall. Six-engine + exact-tree review + `review-1784318402831-2a1ipx-kern-5-r2-m4-1-terminal-sealed-v` then found + signed-integer expression drift, spoofable array density, lossy root + order, and missing direct-handler table-integrity proof. Their RED + regressions and narrow fixes passed renewed aggregate proof. Six-engine + exact-tree review + `review-1784321138406-a6tlpu-kern-5-r2-m4-1-terminal-sealed-v` then found + escaped-output amplification, symbol-record omission, and shared-array + alias acceptance. Their RED regressions and policy/adapter fixes passed + renewed aggregate proof. Final exact-tree review + `review-1784323647882-9ylf9g-kern-5-r2-m4-1-terminal-sealed-v` + completed four engines; Codex hit a transport safety false positive and + Kimi timed out. Both were rerun against the identical exact range in + `review-1784324869674-axw52z-kern-5-r2-m4-1-terminal-sealed-v` and + completed. Across all six usable engines there were no blockers. The + only needs-check code claim was disproved because the named helper-link + test exists in the reviewed base tree; the other two merely observed + this intentionally unchecked close gate before it was recorded. +- [x] The publication contract requires Agon-authored/signed granular commits, + a rebase onto fresh `origin/main`, and one push to the still-open stacked + branch unless its commits land on main first. + +## Confidence Dependencies + +- **0.95:** typed handler table transport—the live twelve-array probe passed. +- **0.92:** generic flattening can stay semantics-free—depends on static and + mutation oracles forbidding KERN-specific adapter decisions. +- **0.96:** exact KIR preservation for the narrowed expression subset—decimal, + binary, precedence, associativity, and numeric normalization are excluded. +- **0.98:** ownership receipt is honest—the tribunal requires a new bounded row + and leaves the broad formatter row unchanged. +- **Overall: 0.99** after the 11/3/105 focused corpus, prefix-complete + runtime-ABI gate, renewed complete wall, and completed six-engine exact-tree + review with no unresolved material finding. diff --git a/docs/kern-5-release-train.md b/docs/kern-5-release-train.md index ee9c2e76d..aef681cf4 100644 --- a/docs/kern-5-release-train.md +++ b/docs/kern-5-release-train.md @@ -809,6 +809,45 @@ trusted-publishing/provenance configuration is inspected. stale or contradicted the exact-reachability contract, Codex reported no findings, and Claude reported no high-confidence blocker (`review-1784279863324-vvtril`). + - [x] M3.31d public runtime-handler sibling helper link: the selected typed + source entry now retains a fresh authenticated function-only scope from the + same parsed module. The public entry is excluded from that helper scope; + classes and imports remain unavailable; duplicate callable helpers and + class/function collisions fail during link. Public oracles cover transitive + scalar helpers, list arguments, wrong arity, loop and recursion bounds, + unreachable and reached effects, exact map ownership, sync/async byte + parity, and overlapping-call isolation. The machine-only public import + closure remains green after narrowing the linker dependency to the portable + scalar domain. Focused `test:runtime-abi` and the complete Node 22 + `fitness:kern-5` wall are green. The terminal 3/3 full-roster review found + zero issues + (`review-1784294556508-m2mp6g-kern-5-r2-m3-31d-terminal-v2`). +- [ ] R2 M4 toolchain ownership. + - [x] M4.1 bounded KERN-authored KIR canonicalizer profile: a generic, + lossless host adapter transports decoded structural KIR through twelve + primitive tables, while `canonicalizer.kern` owns profile validation, + property/type/expression spelling, quoting, indentation, child order, and + complete-result construction. Eleven valid fixtures prove exact goldens, + byte-identical module KIR, second-pass idempotence, and every admitted + return/parameter type. The config-owned 16/30/72 row ceilings are enforced + inside KERN; one exact-boundary valid fixture completes while three + otherwise-valid over-limit fixtures and 105 hostile table/profile fixtures + reject without partial source. `kern check + --with-semantics` reports zero diagnostics and the new current gate earns + only `kern-kir-canonicalizer-profile: internal-oracle`; the broad formatter, + frontend, compiler, fixed point, interpreter, and packed-release rows stay + open. Exact-tree reviews found and drove the `$` structural-name round-trip + fix, the missing non-string parameter/void-parameter coverage, duplicate + required-property variants, and a runtime-ABI public-entry gate omission. + Focused gates are green after the row-bound, runtime-ABI, name-uniqueness, + Unicode, policy, signed-integer, array-density, root-order, and direct-ABI + table-integrity, escaped-output, symbol-record, and shared-array hardening. + The complete Node 22 wall passed on 2026-07-17. The terminal six-engine + exact-tree review completed across an initial dispatch and exact-range + retry for two transport failures, with no unresolved material finding + (`review-1784323647882-9ylf9g-kern-5-r2-m4-1-terminal-sealed-v`, + `review-1784324869674-axw52z-kern-5-r2-m4-1-terminal-sealed-v`). M4.1 is + closed; the broader M4 toolchain exit remains open. 1. Correct the support matrix and make `fitness:kern-5` the planned aggregate, without pretending missing commands already exist. diff --git a/docs/kern-5-support-matrix.md b/docs/kern-5-support-matrix.md index 0ecad4c35..1cfe69653 100644 --- a/docs/kern-5-support-matrix.md +++ b/docs/kern-5-support-matrix.md @@ -48,6 +48,7 @@ before partial output, result, diagnostic, or implicit host effect escapes. | runtime-handler-abi | Default-off public typed runtime handler ABI | current | `pnpm test:runtime-abi` | | core-runtime-internalization | CoreRuntime public-ABI quarantine and internalization | current | `pnpm test:core-runtime-internalization` | | source-runner-convergence | Source runner convergence, call-site isolation, and blocker non-growth | current | `pnpm test:source-runner-convergence` | +| kern-kir-canonicalizer | KERN-authored bounded KIR canonicalizer profile | current | `pnpm test:kern-canonicalizer` | | kern-frontend | KERN-authored frontend | planned | `pnpm test:kern-frontend` | | kern-compiler | KERN-authored compiler | planned | `pnpm test:kern-compiler` | | selfhost-fixed-point | Stage 1 equals Stage 2 | planned | `pnpm test:selfhost-fixed-point` | @@ -100,6 +101,7 @@ wall and must remain absent until promoted. | typed-runtime-handler-abi | Default-off public typed runtime handler ABI | internal-oracle | `pnpm test:runtime-abi` | | core-runtime-internalization | CoreRuntime public-ABI quarantine and internalization | internal-oracle | `pnpm test:core-runtime-internalization` | | source-runner-convergence | Sync/async source-runner convergence and pre-execution selector | internal-oracle | `pnpm test:source-runner-convergence` | +| kern-kir-canonicalizer-profile | Bounded KERN-authored KIR canonicalizer profile | internal-oracle | `pnpm test:kern-canonicalizer` | | kern-formatter | KERN formatter or canonicalizer | not-shipped | R2 planned | | kern-frontend | KERN-authored source frontend | not-shipped | R2 planned | | kern-compiler | KERN-authored compiler | not-shipped | R2 planned | diff --git a/examples/kern-canonicalizer/canonicalizer.kern b/examples/kern-canonicalizer/canonicalizer.kern new file mode 100644 index 000000000..9e9a1f70a --- /dev/null +++ b/examples/kern-canonicalizer/canonicalizer.kern @@ -0,0 +1,438 @@ +fn name=validfirst params="c:string" returns=boolean export=true + handler lang="kern" + if cond="c >= \"A\" && c <= \"Z\"" + return value="true" + if cond="c >= \"a\" && c <= \"z\"" + return value="true" + return value="c == \"_\" || c == \"$\"" + +fn name=validnext params="c:string" returns=boolean export=true + handler lang="kern" + if cond="validfirst(c)" + return value="true" + return value="c >= \"0\" && c <= \"9\"" + +fn name=valididentifier params="value:string" returns=boolean export=true + handler lang="kern" + if cond="Text.length(value) == 0 || !validfirst(Text.charAt(value, 0))" + return value="false" + let name=n value="Text.length(value)" + for name=i from=1 to="n" + if cond="!validnext(Text.charAt(value, i))" + return value="false" + return value="true" + +fn name=validexpressionidentifier params="value:string" returns=boolean export=true + handler lang="kern" + if cond="value == \"true\" || value == \"false\" || value == \"null\" || value == \"none\" || value == \"undefined\"" + return value="false" + if cond="value == \"await\" || value == \"new\" || value == \"typeof\"" + return value="false" + return value="valididentifier(value)" + +fn name=validinteger params="value:string" returns=boolean export=true + handler lang="kern" + let name=start value="0" + if cond="Text.length(value) == 0" + return value="false" + if cond="Text.charAt(value, 0) == \"-\"" + assign target=start value="1" + if cond="start == Text.length(value)" + return value="false" + if cond="Text.charAt(value, start) == \"0\" && Text.length(value) - start > 1" + return value="false" + let name=n value="Text.length(value)" + for name=i from=0 to="n" + if cond="i >= start" + let name=c value="Text.charAt(value, i)" + if cond="c < \"0\" || c > \"9\"" + return value="false" + return value="value != \"-0\"" + +fn name=quotesource params="value:string" returns=string export=true + handler lang="kern" + let name=out value="\"\\\"\"" + let name=n value="Text.length(value)" + for name=i from=0 to="n" + let name=c value="Text.charAt(value, i)" + if cond="c == \"\\\\\"" + assign target=out value="out + \"\\\\\\\\\"" + else + if cond="c == \"\\\"\"" + assign target=out value="out + \"\\\\\\\"\"" + else + if cond="c == \"\\n\"" + assign target=out value="out + \"\\\\n\"" + else + if cond="c == \"\\r\"" + assign target=out value="out + \"\\\\r\"" + else + if cond="c == \"\\t\"" + assign target=out value="out + \"\\\\t\"" + else + if cond="c < \" \" || c == \"\\u007f\" || (c >= \"\\u0080\" && c <= \"\\u009f\") || c == \"\\u2028\" || c == \"\\u2029\" || c == \"\\ufeff\"" + return value="\"\"" + assign target=out value="out + c" + return value="out + \"\\\"\"" + +fn name=structuralname params="value:string" returns=string export=true + handler lang="kern" + if cond="!valididentifier(value)" + return value="\"\"" + let name=n value="Text.length(value)" + for name=i from=0 to="n" + if cond="Text.charAt(value, i) == \"$\"" + return value="quotesource(value)" + return value="value" + +fn name=propcount params="node:number,propNode:number[]" returns=number export=true + handler lang="kern" + let name=count value="0" + for name=i from=0 to="propNode.length" + if cond="propNode[i] == node" + assign target=count value="count + 1" + return value="count" + +fn name=stringat params="id:number,values:string[]" returns=string export=true + handler lang="kern" + for name=i from=0 to="values.length" + if cond="i + 1 == id" + return value="values[i]" + return value="\"\"" + +fn name=numberat params="id:number,values:number[]" returns=number export=true + handler lang="kern" + for name=i from=0 to="values.length" + if cond="i + 1 == id" + return value="values[i]" + return value="-1" + +fn name=propid params="node:number,key:string,propNode:number[],propKey:string[],propValue:number[]" returns=number export=true + handler lang="kern" + let name=result value="0" + for name=i from=0 to="propNode.length" + if cond="propNode[i] == node && propKey[i] == key" + if cond="result != 0" + return value="-1" + assign target=result value="propValue[i]" + return value="result" + +fn name=childcount params="parent:number,nodeParent:number[]" returns=number export=true + handler lang="kern" + let name=count value="0" + for name=i from=0 to="nodeParent.length" + if cond="nodeParent[i] == parent" + assign target=count value="count + 1" + return value="count" + +fn name=childat params="parent:number,order:number,nodeParent:number[],nodeOrder:number[]" returns=number export=true + handler lang="kern" + let name=result value="0" + for name=i from=0 to="nodeParent.length" + if cond="nodeParent[i] == parent && nodeOrder[i] == order" + if cond="result != 0" + return value="-1" + assign target=result value="i + 1" + return value="result" + +fn name=valuechildcount params="parent:number,valueParent:number[]" returns=number export=true + handler lang="kern" + let name=count value="0" + for name=i from=0 to="valueParent.length" + if cond="valueParent[i] == parent" + assign target=count value="count + 1" + return value="count" + +fn name=valuechildat params="parent:number,order:number,valueParent:number[],valueOrder:number[]" returns=number export=true + handler lang="kern" + let name=result value="0" + for name=i from=0 to="valueParent.length" + if cond="valueParent[i] == parent && valueOrder[i] == order" + if cond="result != 0" + return value="-1" + assign target=result value="i + 1" + return value="result" + +fn name=recordfield params="parent:number,key:string,valueParent:number[],valueRole:string[]" returns=number export=true + handler lang="kern" + let name=result value="0" + let name=role value="\"record:\" + key" + for name=i from=0 to="valueParent.length" + if cond="valueParent[i] == parent && valueRole[i] == role" + if cond="result != 0" + return value="-1" + assign target=result value="i + 1" + return value="result" + +fn name=typesource params="id:number,allowVoid:boolean,valueTag:string[],valueParent:number[],valueRole:string[],valueText:string[]" returns=string export=true + handler lang="kern" + if cond="id <= 0 || id > valueTag.length || stringat(id, valueTag) != \"record\"" + return value="\"\"" + let name=count value="valuechildcount(id, valueParent)" + let name=kindId value="recordfield(id, \"kind\", valueParent, valueRole)" + if cond="kindId <= 0 || stringat(kindId, valueTag) != \"text\"" + return value="\"\"" + let name=kind value="stringat(kindId, valueText)" + if cond="count == 1" + if cond="kind == \"boolean\"" + return value="\"boolean\"" + if cond="kind == \"integer\"" + return value="\"number\"" + if cond="kind == \"text\"" + return value="\"string\"" + if cond="allowVoid && kind == \"void\"" + return value="\"void\"" + return value="\"\"" + if cond="count != 2 || kind != \"list\"" + return value="\"\"" + let name=elementId value="recordfield(id, \"element\", valueParent, valueRole)" + if cond="elementId <= 0 || stringat(elementId, valueTag) != \"text\"" + return value="\"\"" + let name=element value="stringat(elementId, valueText)" + if cond="element == \"boolean\"" + return value="\"boolean[]\"" + if cond="element == \"integer\"" + return value="\"number[]\"" + if cond="element == \"text\"" + return value="\"string[]\"" + return value="\"\"" + +fn name=exprsource params="id:number,valueTag:string[],valueParent:number[],valueRole:string[],valueOrder:number[],valueText:string[],valueBool:number[]" returns=string export=true + handler lang="kern" + if cond="id <= 0 || id > valueTag.length || stringat(id, valueTag) != \"record\"" + return value="\"\"" + if cond="valuechildcount(id, valueParent) != 2" + return value="\"\"" + let name=fieldsId value="recordfield(id, \"fields\", valueParent, valueRole)" + let name=kindId value="recordfield(id, \"kind\", valueParent, valueRole)" + if cond="fieldsId <= 0 || kindId <= 0 || stringat(fieldsId, valueTag) != \"record\" || stringat(kindId, valueTag) != \"text\"" + return value="\"\"" + let name=kind value="stringat(kindId, valueText)" + if cond="kind == \"null\"" + if cond="valuechildcount(fieldsId, valueParent) == 0" + return value="\"null\"" + return value="\"\"" + if cond="kind == \"identifier\"" + let name=nameId value="recordfield(fieldsId, \"name\", valueParent, valueRole)" + if cond="valuechildcount(fieldsId, valueParent) == 1 && nameId > 0 && stringat(nameId, valueTag) == \"text\" && validexpressionidentifier(stringat(nameId, valueText))" + return value="stringat(nameId, valueText)" + return value="\"\"" + let name=valueId value="recordfield(fieldsId, \"value\", valueParent, valueRole)" + if cond="kind == \"boolean\"" + if cond="valuechildcount(fieldsId, valueParent) == 1 && valueId > 0 && stringat(valueId, valueTag) == \"bool\"" + if cond="numberat(valueId, valueBool) == 1" + return value="\"true\"" + return value="\"false\"" + return value="\"\"" + if cond="kind == \"integer\"" + if cond="valuechildcount(fieldsId, valueParent) == 1 && valueId > 0 && stringat(valueId, valueTag) == \"int\" && validinteger(stringat(valueId, valueText)) && !Text.startsWith(stringat(valueId, valueText), \"-\")" + return value="stringat(valueId, valueText)" + return value="\"\"" + if cond="kind == \"text\"" + if cond="valuechildcount(fieldsId, valueParent) == 1 && valueId > 0 && stringat(valueId, valueTag) == \"text\"" + return value="quotesource(stringat(valueId, valueText))" + return value="\"\"" + if cond="kind != \"list\"" + return value="\"\"" + let name=itemsId value="recordfield(fieldsId, \"items\", valueParent, valueRole)" + if cond="valuechildcount(fieldsId, valueParent) != 1 || itemsId <= 0 || stringat(itemsId, valueTag) != \"list\"" + return value="\"\"" + let name=out value="\"[\"" + let name=count value="valuechildcount(itemsId, valueParent)" + for name=i from=0 to="count" + let name=itemId value="valuechildat(itemsId, i, valueParent, valueOrder)" + if cond="itemId <= 0" + return value="\"\"" + let name=item value="exprsource(itemId, valueTag, valueParent, valueRole, valueOrder, valueText, valueBool)" + if cond="item == \"\"" + return value="\"\"" + if cond="i > 0" + assign target=out value="out + \", \"" + assign target=out value="out + item" + return value="out + \"]\"" + +fn name=tablesok params="nodeKind:string[],nodeParent:number[],nodeOrder:number[],propNode:number[],propKey:string[],propValue:number[],valueTag:string[],valueParent:number[],valueRole:string[],valueOrder:number[],valueText:string[],valueBool:number[]" returns=boolean export=true + handler lang="kern" + if cond="nodeKind.length == 0 || nodeKind.length != nodeParent.length || nodeKind.length != nodeOrder.length" + return value="false" + if cond="propNode.length != propKey.length || propNode.length != propValue.length" + return value="false" + if cond="valueTag.length != valueParent.length || valueTag.length != valueRole.length || valueTag.length != valueOrder.length || valueTag.length != valueText.length || valueTag.length != valueBool.length" + return value="false" + for name=i from=0 to="nodeKind.length" + if cond="nodeParent[i] < 0 || nodeParent[i] > nodeKind.length || nodeOrder[i] < 0" + return value="false" + if cond="nodeParent[i] != 0 && nodeParent[i] >= i + 1" + return value="false" + let name=siblings value="0" + for name=j from=0 to="nodeKind.length" + if cond="nodeParent[j] == nodeParent[i]" + assign target=siblings value="siblings + 1" + if cond="j > i && nodeOrder[j] == nodeOrder[i]" + return value="false" + if cond="nodeOrder[i] >= siblings" + return value="false" + for name=propertyIndex from=0 to="propNode.length" + if cond="propNode[propertyIndex] < 1 || propNode[propertyIndex] > nodeKind.length || propValue[propertyIndex] < 1 || propValue[propertyIndex] > valueTag.length" + return value="false" + if cond="numberat(propValue[propertyIndex], valueParent) != 0 || stringat(propValue[propertyIndex], valueRole) != \"\"" + return value="false" + for name=propertyCompare from=0 to="propNode.length" + if cond="propertyCompare > propertyIndex && propNode[propertyCompare] == propNode[propertyIndex] && propKey[propertyCompare] == propKey[propertyIndex]" + return value="false" + for name=valueIndex from=0 to="valueTag.length" + if cond="valueParent[valueIndex] < 0 || valueParent[valueIndex] > valueTag.length || valueOrder[valueIndex] < 0" + return value="false" + if cond="valueParent[valueIndex] != 0 && valueParent[valueIndex] >= valueIndex + 1" + return value="false" + let name=tag value="valueTag[valueIndex]" + if cond="tag != \"null\" && tag != \"bool\" && tag != \"text\" && tag != \"int\" && tag != \"list\" && tag != \"record\"" + return value="false" + let name=children value="0" + for name=valueChild from=0 to="valueTag.length" + if cond="valueParent[valueChild] == valueIndex + 1" + assign target=children value="children + 1" + if cond="valueParent[valueIndex] > 0" + let name=valueSiblings value="0" + for name=valueCompare from=0 to="valueTag.length" + if cond="valueParent[valueCompare] == valueParent[valueIndex]" + assign target=valueSiblings value="valueSiblings + 1" + if cond="valueCompare > valueIndex && valueOrder[valueCompare] == valueOrder[valueIndex]" + return value="false" + if cond="valueOrder[valueIndex] >= valueSiblings" + return value="false" + if cond="tag == \"null\" || tag == \"bool\" || tag == \"text\" || tag == \"int\"" + if cond="children != 0" + return value="false" + if cond="tag == \"null\" && (valueText[valueIndex] != \"\" || valueBool[valueIndex] != 0)" + return value="false" + if cond="tag == \"bool\" && (valueText[valueIndex] != \"\" || (valueBool[valueIndex] != 0 && valueBool[valueIndex] != 1))" + return value="false" + if cond="(tag == \"text\" || tag == \"int\") && valueBool[valueIndex] != 0" + return value="false" + if cond="(tag == \"list\" || tag == \"record\") && (valueText[valueIndex] != \"\" || valueBool[valueIndex] != 0)" + return value="false" + if cond="tag == \"int\" && !validinteger(valueText[valueIndex])" + return value="false" + if cond="valueParent[valueIndex] == 0" + let name=owners value="0" + for name=p from=0 to="propValue.length" + if cond="propValue[p] == valueIndex + 1" + assign target=owners value="owners + 1" + if cond="owners != 1 || valueRole[valueIndex] != \"\" || valueOrder[valueIndex] != 0" + return value="false" + else + for name=p from=0 to="propValue.length" + if cond="propValue[p] == valueIndex + 1" + return value="false" + let name=parentTag value="stringat(valueParent[valueIndex], valueTag)" + if cond="parentTag == \"list\"" + if cond="valueRole[valueIndex] != \"list-item\" || valueOrder[valueIndex] >= valuechildcount(valueParent[valueIndex], valueParent)" + return value="false" + else + if cond="parentTag != \"record\" || !Text.startsWith(valueRole[valueIndex], \"record:\") || valueOrder[valueIndex] >= valuechildcount(valueParent[valueIndex], valueParent)" + return value="false" + if cond="Text.length(valueRole[valueIndex]) <= 7" + return value="false" + for name=recordCompare from=0 to="valueTag.length" + if cond="recordCompare != valueIndex && valueParent[recordCompare] == valueParent[valueIndex]" + if cond="valueRole[recordCompare] == valueRole[valueIndex]" + return value="false" + if cond="valueOrder[recordCompare] < valueOrder[valueIndex] && valueRole[recordCompare] >= valueRole[valueIndex]" + return value="false" + return value="true" + +fn name=canonicalize params="nodeKind:string[],nodeParent:number[],nodeOrder:number[],propNode:number[],propKey:string[],propValue:number[],valueTag:string[],valueParent:number[],valueRole:string[],valueOrder:number[],valueText:string[],valueBool:number[],maxNodeRows:number,maxPropertyRows:number,maxValueRows:number" returns=string[] export=true + handler lang="kern" + if cond="maxNodeRows <= 0 || maxPropertyRows <= 0 || maxValueRows <= 0 || nodeKind.length > maxNodeRows || propNode.length > maxPropertyRows || valueTag.length > maxValueRows" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + if cond="!tablesok(nodeKind, nodeParent, nodeOrder, propNode, propKey, propValue, valueTag, valueParent, valueRole, valueOrder, valueText, valueBool)" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + let name=roots value="childcount(0, nodeParent)" + if cond="roots == 0" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + for name=outputRoot from=0 to="roots" + let name=fnId value="childat(0, outputRoot, nodeParent, nodeOrder)" + if cond="fnId <= 0 || stringat(fnId, nodeKind) != \"fn\"" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + let name=nameId value="propid(fnId, \"name\", propNode, propKey, propValue)" + let name=returnsId value="propid(fnId, \"returns\", propNode, propKey, propValue)" + let name=exportId value="propid(fnId, \"export\", propNode, propKey, propValue)" + if cond="nameId <= 0 || returnsId <= 0 || exportId < 0 || propcount(fnId, propNode) < 2 || propcount(fnId, propNode) > 3" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + for name=fnProperty from=0 to="propNode.length" + if cond="propNode[fnProperty] == fnId && propKey[fnProperty] != \"name\" && propKey[fnProperty] != \"returns\" && propKey[fnProperty] != \"export\"" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + if cond="stringat(nameId, valueTag) != \"text\" || structuralname(stringat(nameId, valueText)) == \"\" || typesource(returnsId, true, valueTag, valueParent, valueRole, valueText) == \"\"" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + for name=previousRoot from=0 to="outputRoot" + let name=previousFnId value="childat(0, previousRoot, nodeParent, nodeOrder)" + let name=previousNameId value="propid(previousFnId, \"name\", propNode, propKey, propValue)" + if cond="previousNameId > 0 && stringat(previousNameId, valueText) == stringat(nameId, valueText)" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + if cond="exportId > 0 && stringat(exportId, valueTag) != \"bool\"" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + let name=fnChildren value="childcount(fnId, nodeParent)" + if cond="fnChildren == 0" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + for name=c from=0 to="fnChildren" + let name=childId value="childat(fnId, c, nodeParent, nodeOrder)" + if cond="childId <= 0" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + if cond="c < fnChildren - 1" + if cond="stringat(childId, nodeKind) != \"param\" || childcount(childId, nodeParent) != 0 || propcount(childId, propNode) != 2" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + let name=paramName value="propid(childId, \"name\", propNode, propKey, propValue)" + let name=paramType value="propid(childId, \"type\", propNode, propKey, propValue)" + if cond="paramName <= 0 || paramType <= 0 || stringat(paramName, valueTag) != \"text\" || structuralname(stringat(paramName, valueText)) == \"\" || typesource(paramType, false, valueTag, valueParent, valueRole, valueText) == \"\"" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + for name=previousParam from=0 to="c" + let name=previousParamId value="childat(fnId, previousParam, nodeParent, nodeOrder)" + let name=previousParamName value="propid(previousParamId, \"name\", propNode, propKey, propValue)" + if cond="previousParamName > 0 && stringat(previousParamName, valueText) == stringat(paramName, valueText)" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + else + if cond="stringat(childId, nodeKind) != \"handler\" || propcount(childId, propNode) != 1 || childcount(childId, nodeParent) != 1" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + let name=langId value="propid(childId, \"lang\", propNode, propKey, propValue)" + let name=returnId value="childat(childId, 0, nodeParent, nodeOrder)" + if cond="langId <= 0 || stringat(langId, valueTag) != \"text\" || stringat(langId, valueText) != \"kern\" || returnId <= 0 || stringat(returnId, nodeKind) != \"return\" || childcount(returnId, nodeParent) != 0" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + let name=returnType value="typesource(returnsId, true, valueTag, valueParent, valueRole, valueText)" + let name=returnValue value="propid(returnId, \"value\", propNode, propKey, propValue)" + if cond="returnType == \"void\"" + if cond="returnValue != 0 || propcount(returnId, propNode) != 0" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + else + if cond="returnValue <= 0 || propcount(returnId, propNode) != 1 || exprsource(returnValue, valueTag, valueParent, valueRole, valueOrder, valueText, valueBool) == \"\"" + throw value="new Error(\"KERN_CANONICALIZER_PROFILE\")" + let name=lines value="[]" + for name=r from=0 to="roots" + let name=fnId value="childat(0, r, nodeParent, nodeOrder)" + let name=nameId value="propid(fnId, \"name\", propNode, propKey, propValue)" + let name=returnsId value="propid(fnId, \"returns\", propNode, propKey, propValue)" + let name=exportId value="propid(fnId, \"export\", propNode, propKey, propValue)" + let name=line value="\"fn name=\" + structuralname(stringat(nameId, valueText)) + \" returns=\" + typesource(returnsId, true, valueTag, valueParent, valueRole, valueText)" + if cond="exportId > 0" + if cond="numberat(exportId, valueBool) == 1" + assign target=line value="line + \" export=true\"" + else + assign target=line value="line + \" export=false\"" + do value="lines.push(line)" + let name=fnChildren value="childcount(fnId, nodeParent)" + for name=c from=0 to="fnChildren" + let name=childId value="childat(fnId, c, nodeParent, nodeOrder)" + if cond="stringat(childId, nodeKind) == \"param\"" + let name=paramName value="propid(childId, \"name\", propNode, propKey, propValue)" + let name=paramType value="propid(childId, \"type\", propNode, propKey, propValue)" + do value="lines.push(\" param name=\" + structuralname(stringat(paramName, valueText)) + \" type=\" + typesource(paramType, false, valueTag, valueParent, valueRole, valueText))" + else + do value="lines.push(\" handler lang=\" + quotesource(\"kern\"))" + let name=returnId value="childat(childId, 0, nodeParent, nodeOrder)" + let name=returnValue value="propid(returnId, \"value\", propNode, propKey, propValue)" + if cond="returnValue == 0" + do value="lines.push(\" return\")" + else + let name=expression value="exprsource(returnValue, valueTag, valueParent, valueRole, valueOrder, valueText, valueBool)" + do value="lines.push(\" return value=\" + quotesource(expression))" + return value="lines" diff --git a/package.json b/package.json index 2f60341e3..b9c26d092 100644 --- a/package.json +++ b/package.json @@ -50,12 +50,13 @@ "test:kern-kir-evidence": "pnpm --filter @kernlang/core test --testPathPatterns=kir-evidence && node ./scripts/check-kir-evidence.mjs", "test:kern-alpha-receipt": "node --test scripts/kir-v1/alpha-receipt.test.mjs", "test:kern-runtime-envelope": "pnpm --filter @kernlang/core test --testPathPatterns='runtime-envelope|portable-machine-evaluator' && node --test ./scripts/runtime-envelope-import-closure.test.mjs ./scripts/runtime-envelope-handler-import-closure.test.mjs && node ./scripts/check-runtime-envelope.mjs", - "test:runtime-abi": "pnpm --filter @kernlang/core test --testPathPatterns='runtime-handler-public' && node --test ./scripts/runtime-handler-public-import-closure.test.mjs && node ./scripts/check-runtime-envelope.mjs", + "test:runtime-abi": "pnpm --filter @kernlang/core test --testPathPatterns='runtime-handler-(public|helper-link)' && node --test ./scripts/runtime-handler-public-import-closure.test.mjs && node ./scripts/check-runtime-envelope.mjs", "test:core-runtime-internalization": "pnpm --filter @kernlang/core --filter kern-lang build && node scripts/check-core-runtime-internalization.mjs && node --test scripts/core-runtime-internalization.test.mjs", "check:source-runner-convergence": "node ./scripts/check-source-runner-convergence.mjs", "test:source-runner-convergence": "pnpm --filter @kernlang/core test --testPathPatterns='runner-capability-(class-frame|plan)|source-runner-engine|runtime-envelope-effect-machine-(do|each|expression-v1|helper|lambda|non-root|module-ownership|class-)' && node --test ./scripts/source-runner*-convergence.test.mjs && node ./scripts/check-source-runner-convergence.mjs", + "test:kern-canonicalizer": "pnpm --filter @kernlang/core build && pnpm --filter @kernlang/cli build && node packages/cli/dist/cli.js check examples/kern-canonicalizer/canonicalizer.kern --with-semantics --quiet && node --test ./scripts/kern-canonicalizer/flatten.test.mjs ./scripts/kern-canonicalizer/canonicalizer.test.mjs && node ./scripts/check-kern-canonicalizer.mjs", "build:kern-alpha-receipt": "node ./scripts/kir-v1/alpha-receipt.mjs", - "test:infra": "pnpm test:prepush && pnpm test:release-policy && pnpm test:kern-5-fitness && pnpm test:kern-semantic-ownership && pnpm test:kern-ir-eligibility && pnpm test:kern-canonical-value && pnpm test:kern-kir-structural-constitution && pnpm test:kern-kir-structural-codec && pnpm test:kern-kir-module-graph && pnpm test:kern-kir-coverage-closure && pnpm test:kern-kir-evidence && pnpm test:kern-alpha-receipt && pnpm test:kern-runtime-envelope && pnpm test:runtime-abi && pnpm test:core-runtime-internalization && pnpm test:source-runner-convergence", + "test:infra": "pnpm test:prepush && pnpm test:release-policy && pnpm test:kern-5-fitness && pnpm test:kern-semantic-ownership && pnpm test:kern-ir-eligibility && pnpm test:kern-canonical-value && pnpm test:kern-kir-structural-constitution && pnpm test:kern-kir-structural-codec && pnpm test:kern-kir-module-graph && pnpm test:kern-kir-coverage-closure && pnpm test:kern-kir-evidence && pnpm test:kern-alpha-receipt && pnpm test:kern-runtime-envelope && pnpm test:runtime-abi && pnpm test:core-runtime-internalization && pnpm test:source-runner-convergence && pnpm test:kern-canonicalizer", "fitness": "pnpm fitness:kern-5", "fitness:kern-5": "node ./scripts/kern-5-fitness.mjs", "check:kern-5-contract": "node ./scripts/kern-5-fitness.mjs --check", diff --git a/scripts/check-kern-canonicalizer.mjs b/scripts/check-kern-canonicalizer.mjs new file mode 100644 index 000000000..2c7022c32 --- /dev/null +++ b/scripts/check-kern-canonicalizer.mjs @@ -0,0 +1,164 @@ +#!/usr/bin/env node +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +import { decodeModuleKir, encodeModuleKir } from '../packages/core/dist/kir-structural/module-canonical.js'; +import { parseDocumentWithDiagnostics } from '../packages/core/dist/parser.js'; +import { + executeKernRuntimeHandlerSync, + KERN_RUNTIME_HANDLER_ABI, +} from '../packages/core/dist/runtime-handler.js'; +import { flattenKirRoots, tableArguments } from './kern-canonicalizer/flatten.mjs'; +import { HOSTILE_FIXTURES, VALID_FIXTURES } from './kern-canonicalizer/fixtures.mjs'; +import { loadCanonicalizerPolicy } from './kern-canonicalizer/policy.mjs'; +import { rehydrateKirRoots } from './kern-canonicalizer/rehydrate.mjs'; +import { PROFILE_LIMIT_FIXTURES } from './kern-canonicalizer/profile-limit-fixtures.mjs'; + +const CANONICALIZER_POLICY = loadCanonicalizerPolicy(); +const { kirLimits: KIR_LIMITS, profileLimits: PROFILE_LIMITS, runtimeLimits: RUNTIME_LIMITS } = + CANONICALIZER_POLICY; +const CANONICALIZER_PATH = fileURLToPath( + new URL('../examples/kern-canonicalizer/canonicalizer.kern', import.meta.url), +); +const CANONICALIZER_SOURCE = readFileSync(CANONICALIZER_PATH, 'utf8'); +const fixtureById = new Map(VALID_FIXTURES.map((fixture) => [fixture.id, fixture])); + +function fail(category, detail) { + throw new Error(`${category}: ${detail}`); +} + +function canonicalizerArguments(tables) { + return [ + ...tableArguments(tables), + PROFILE_LIMITS.maxNodeRows, + PROFILE_LIMITS.maxPropertyRows, + PROFILE_LIMITS.maxValueRows, + ]; +} + +function rowCounts(tables) { + return { + nodes: tables.nodeKind.length, + properties: tables.propNode.length, + values: tables.valueTag.length, + }; +} + +function parsedRoots(source, label) { + const parsed = parseDocumentWithDiagnostics(source); + const errors = parsed.diagnostics.filter((diagnostic) => diagnostic.severity === 'error'); + if (parsed.partial || errors.length > 0) fail('parse rejection', `${label} produced ${errors.length} errors`); + return parsed.root.type === 'document' ? (parsed.root.children ?? []) : []; +} + +function semanticArtifact(source, moduleId, label) { + const roots = parsedRoots(source, label); + const bytes = encodeModuleKir([{ id: moduleId, roots }], KIR_LIMITS); + const decoded = decodeModuleKir(bytes, KIR_LIMITS); + const module = decoded.modules.find((candidate) => candidate.id === moduleId); + if (!module) fail('KIR mismatch', `${label} omitted ${moduleId}`); + return { bytes, roots: module.roots }; +} + +function executeTables(tables, label) { + const envelope = executeTableEnvelope(tables); + if (envelope.outcome !== 'success') { + const code = envelope.diagnostics[0]?.code ?? 'missing-diagnostic'; + fail('profile rejection', `${label} returned ${code}`); + } + if ( + envelope.completion.kind !== 'return' || + envelope.events.length !== 0 || + envelope.result.presence !== 'value' || + envelope.result.value.tag !== 'list' + ) { + fail('profile rejection', `${label} returned a malformed success envelope`); + } + const lines = envelope.result.value.value.map((value, index) => { + if (value.tag !== 'text') fail('profile rejection', `${label} line ${index} is not text`); + return value.value; + }); + return lines; +} + +function executeTableEnvelope(tables) { + return executeKernRuntimeHandlerSync( + { + abi: KERN_RUNTIME_HANDLER_ABI, + arguments: canonicalizerArguments(tables), + identity: { handlerName: 'canonicalize', sourcePath: 'examples/kern-canonicalizer/canonicalizer.kern' }, + source: CANONICALIZER_SOURCE, + }, + { enabled: true, limits: RUNTIME_LIMITS }, + ); +} + +function canonicalizeSource(source, moduleId, label) { + const artifact = semanticArtifact(source, moduleId, label); + const tables = flattenKirRoots(artifact.roots); + assert.deepEqual(rehydrateKirRoots(tables), artifact.roots, `adapter rejection: ${label} was not lossless`); + return { artifact, source: `${executeTables(tables, label).join('\n')}\n`, tables }; +} + +export function runKernCanonicalizerCheck() { + for (const fixture of VALID_FIXTURES) { + const moduleId = `${fixture.id}.kern`; + const first = canonicalizeSource(fixture.source, moduleId, fixture.id); + if (fixture.expectedRows) assert.deepEqual(rowCounts(first.tables), fixture.expectedRows, fixture.id); + if (first.source !== fixture.golden) fail('golden mismatch', fixture.id); + + const formattedArtifact = semanticArtifact(first.source, moduleId, `${fixture.id}:formatted`); + if (!Buffer.from(formattedArtifact.bytes).equals(Buffer.from(first.artifact.bytes))) { + fail('KIR mismatch', fixture.id); + } + + const second = canonicalizeSource(first.source, moduleId, `${fixture.id}:second-pass`); + if (second.source !== first.source) fail('idempotence mismatch', fixture.id); + } + + for (const fixture of PROFILE_LIMIT_FIXTURES) { + const artifact = semanticArtifact(fixture.source, `${fixture.id}.kern`, fixture.id); + const tables = flattenKirRoots(artifact.roots); + assert.deepEqual(rowCounts(tables), fixture.expectedRows, fixture.id); + const envelope = executeTableEnvelope(tables); + assert.equal(envelope.outcome, 'failure', `${fixture.id} must reject`); + assert.equal(envelope.diagnostics[0]?.code, 'uncaught-throw', `${fixture.id} must reject explicitly in KERN`); + assert.deepEqual(envelope.events, [], `${fixture.id} must not emit partial events`); + assert.deepEqual(envelope.result, { presence: 'absent' }, `${fixture.id} must not return partial source`); + } + + for (const hostile of HOSTILE_FIXTURES) { + const fixture = fixtureById.get(hostile.base); + if (!fixture) fail('adapter rejection', `${hostile.id} names missing base ${hostile.base}`); + const moduleId = `${fixture.id}.kern`; + const artifact = semanticArtifact(fixture.source, moduleId, `${hostile.id}:base`); + const tables = structuredClone(flattenKirRoots(artifact.roots)); + hostile.mutate(tables); + + if (hostile.category === 'adapter rejection') { + assert.throws(() => rehydrateKirRoots(tables), /adapter rejection:/u, hostile.id); + continue; + } + + if (hostile.category === 'direct profile rejection') { + assert.throws(() => rehydrateKirRoots(tables), /adapter rejection:/u, hostile.id); + } else { + const hostileRoots = rehydrateKirRoots(tables); + assert.notDeepEqual(hostileRoots, artifact.roots, `${hostile.id} mutation must change the graph`); + assert.deepEqual(rehydrateKirRoots(flattenKirRoots(hostileRoots)), hostileRoots, hostile.id); + } + const envelope = executeTableEnvelope(tables); + assert.equal(envelope.outcome, 'failure', `${hostile.id} must reject`); + assert.equal(envelope.diagnostics[0]?.code, 'uncaught-throw', `${hostile.id} must reject explicitly in KERN`); + assert.deepEqual(envelope.events, [], `${hostile.id} must not emit partial events`); + assert.deepEqual(envelope.result, { presence: 'absent' }, `${hostile.id} must not return partial source`); + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + runKernCanonicalizerCheck(); + process.stdout.write( + `KERN canonicalizer: ${VALID_FIXTURES.length} golden/idempotence/KIR fixtures, ${PROFILE_LIMIT_FIXTURES.length} profile-limit fixtures, and ${HOSTILE_FIXTURES.length} hostile fixtures passed.\n`, + ); +} diff --git a/scripts/kern-5-fitness-policy.json b/scripts/kern-5-fitness-policy.json index dd4942abd..00c8ce77a 100644 --- a/scripts/kern-5-fitness-policy.json +++ b/scripts/kern-5-fitness-policy.json @@ -19,8 +19,10 @@ "test:kern-runtime-envelope": "pnpm --filter @kernlang/core test --testPathPatterns='runtime-envelope|portable-machine-evaluator' && node --test ./scripts/runtime-envelope-import-closure.test.mjs ./scripts/runtime-envelope-handler-import-closure.test.mjs && node ./scripts/check-runtime-envelope.mjs", "test:kern-5-fitness": "node --test scripts/kern-5-fitness.test.mjs", "test:core-runtime-internalization": "pnpm --filter @kernlang/core --filter kern-lang build && node scripts/check-core-runtime-internalization.mjs && node --test scripts/core-runtime-internalization.test.mjs", + "test:runtime-abi": "pnpm --filter @kernlang/core test --testPathPatterns='runtime-handler-(public|helper-link)' && node --test ./scripts/runtime-handler-public-import-closure.test.mjs && node ./scripts/check-runtime-envelope.mjs", "check:source-runner-convergence": "node ./scripts/check-source-runner-convergence.mjs", - "test:source-runner-convergence": "pnpm --filter @kernlang/core test --testPathPatterns='runner-capability-(class-frame|plan)|source-runner-engine|runtime-envelope-effect-machine-(do|each|expression-v1|helper|lambda|non-root|module-ownership|class-)' && node --test ./scripts/source-runner*-convergence.test.mjs && node ./scripts/check-source-runner-convergence.mjs" + "test:source-runner-convergence": "pnpm --filter @kernlang/core test --testPathPatterns='runner-capability-(class-frame|plan)|source-runner-engine|runtime-envelope-effect-machine-(do|each|expression-v1|helper|lambda|non-root|module-ownership|class-)' && node --test ./scripts/source-runner*-convergence.test.mjs && node ./scripts/check-source-runner-convergence.mjs", + "test:kern-canonicalizer": "pnpm --filter @kernlang/core build && pnpm --filter @kernlang/cli build && node packages/cli/dist/cli.js check examples/kern-canonicalizer/canonicalizer.kern --with-semantics --quiet && node --test ./scripts/kern-canonicalizer/flatten.test.mjs ./scripts/kern-canonicalizer/canonicalizer.test.mjs && node ./scripts/check-kern-canonicalizer.mjs" }, "gates": [ { @@ -185,6 +187,12 @@ "status": "current", "argv": ["pnpm", "test:source-runner-convergence"] }, + { + "id": "kern-kir-canonicalizer", + "label": "KERN-authored bounded KIR canonicalizer profile", + "status": "current", + "argv": ["pnpm", "test:kern-canonicalizer"] + }, { "id": "kern-frontend", "label": "KERN-authored frontend", @@ -427,6 +435,12 @@ "status": "internal-oracle", "evidence": "pnpm test:source-runner-convergence" }, + { + "id": "kern-kir-canonicalizer-profile", + "label": "Bounded KERN-authored KIR canonicalizer profile", + "status": "internal-oracle", + "evidence": "pnpm test:kern-canonicalizer" + }, { "id": "kern-formatter", "label": "KERN formatter or canonicalizer", diff --git a/scripts/kern-5-fitness.test.mjs b/scripts/kern-5-fitness.test.mjs index ee5776c71..085bed55a 100644 --- a/scripts/kern-5-fitness.test.mjs +++ b/scripts/kern-5-fitness.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { readdirSync, readFileSync } from 'node:fs'; import test from 'node:test'; import { @@ -52,6 +52,7 @@ test('current KERN 5 policy, matrix, and root scripts form one exact contract', 'runtime-handler-abi', 'core-runtime-internalization', 'source-runner-convergence', + 'kern-kir-canonicalizer', ], ); }); @@ -84,6 +85,11 @@ test('matrix mutations fail with the affected gate or ownership id', () => { ), error: /checker-v2/i, }, + { + name: 'missing bounded canonicalizer receipt', + text: matrixText.replace(/^\| kern-kir-canonicalizer-profile \|.*\n/mu, ''), + error: /kern-kir-canonicalizer-profile/i, + }, { name: 'missing ownership row', text: matrixText.replace(/^\| kern-formatter \|.*\n/mu, ''), @@ -156,6 +162,21 @@ test('entrypoint scripts must exactly match policy', () => { assert.throws(() => validate({ packageJson: mutated }), /fitness:kern-5.*exactly match/i); }); +test('runtime ABI gate keeps every public handler entry and helper-link oracle', () => { + const runtimeAbi = packageJson.scripts['test:runtime-abi']; + assert.match( + runtimeAbi, + /runtime-handler-\(public\|helper-link\)/u, + 'test:runtime-abi must select the public-handler prefix and helper-link oracle', + ); + assert.ok( + readdirSync(new URL('../packages/core/tests/', import.meta.url)).some((name) => + /^runtime-handler-helper-link.*\.test\.ts$/u.test(name), + ), + 'test:runtime-abi helper-link selector must match a real test file', + ); +}); + test('aggregate uses argv without a shell, preserves order, and stops at first failure', () => { const calls = []; const currentGates = policy.gates.filter((gate) => gate.status === 'current').slice(0, 3); diff --git a/scripts/kern-canonicalizer/canonicalizer.test.mjs b/scripts/kern-canonicalizer/canonicalizer.test.mjs new file mode 100644 index 000000000..d27f37d3d --- /dev/null +++ b/scripts/kern-canonicalizer/canonicalizer.test.mjs @@ -0,0 +1,95 @@ +import assert from 'node:assert/strict'; +import { existsSync, readFileSync } from 'node:fs'; +import test from 'node:test'; + +import { parseDocumentWithDiagnostics } from '../../packages/core/dist/parser.js'; +import { VALID_FIXTURES } from './fixtures.mjs'; +import { validateCanonicalizerPolicy } from './policy.mjs'; + +const source = readFileSync(new URL('../../examples/kern-canonicalizer/canonicalizer.kern', import.meta.url), 'utf8'); +const policyUrl = new URL('./policy.json', import.meta.url); + +test('the KERN canonicalizer is parseable, bounded, and contains the semantic source decisions', () => { + const parsed = parseDocumentWithDiagnostics(source); + assert.notEqual(parsed.partial, true); + assert.deepEqual( + parsed.diagnostics.filter((diagnostic) => diagnostic.severity === 'error'), + [], + ); + assert.ok(source.split('\n').length - 1 < 500, 'hand-written KERN source must stay below 500 lines'); + for (const owned of ['fn name=', 'param name=', 'handler lang=', 'return value=', 'quotesource', 'typesource']) { + assert.ok(source.includes(owned), `missing KERN-owned source decision ${owned}`); + } +}); + +test('the admitted table profile is policy-owned and enforced by KERN', () => { + assert.equal(existsSync(policyUrl), true, 'missing canonicalizer policy'); + const policy = JSON.parse(readFileSync(policyUrl, 'utf8')); + validateCanonicalizerPolicy(policy); + assert.deepEqual(policy.profileLimits, { + maxNodeRows: 16, + maxPropertyRows: 30, + maxValueRows: 72, + }); + assert.deepEqual(policy.expansionLimits, { + kirToSourceMaxFactor: 4, + runtimeEnvelopeMaxFactor: 2, + }); + assert.equal(policy.runtimeLimits.maxStringBytes, 1_048_576); + assert.equal(policy.runtimeLimits.maxBytes, 2_097_152); + assert.equal(policy.runtimeLimits.maxCollectionLength, 65_536); + for (const limitName of Object.keys(policy.profileLimits)) { + assert.match(source, new RegExp(limitName, 'u'), `KERN omitted ${limitName}`); + } + for (const mutate of [ + (copy) => delete copy.expansionLimits.kirToSourceMaxFactor, + (copy) => delete copy.kirLimits.maxBytes, + (copy) => { + copy.runtimeLimits.futureLimit = 1; + }, + ...Object.keys(policy.profileLimits).map((key) => (copy) => delete copy.profileLimits[key]), + ]) { + const copy = structuredClone(policy); + mutate(copy); + assert.throws(() => validateCanonicalizerPolicy(copy), /must contain exactly/u); + } + for (const mutate of [ + (copy) => { copy.runtimeLimits.maxStringBytes -= 1; }, + (copy) => { copy.runtimeLimits.maxBytes -= 1; }, + ]) { + const copy = structuredClone(policy); + mutate(copy); + assert.throws(() => validateCanonicalizerPolicy(copy), /must cover the configured/u); + } +}); + +test('the canonicalizer has no host-handler, capability, import, or delegated runtime escape', () => { + for (const forbidden of ['handler lang=ts', 'capability namespace=', 'import ', 'use path=', 'handler code=', '<<<']) { + assert.equal(source.includes(forbidden), false, `forbidden canonicalizer escape ${forbidden}`); + } +}); + +test('the valid corpus covers every admitted return and parameter type', () => { + const coveredReturns = new Set(); + const coveredParameters = new Set(); + for (const fixture of VALID_FIXTURES) { + const parsed = parseDocumentWithDiagnostics(fixture.source); + for (const root of parsed.root.children ?? []) { + if (root.type !== 'fn') continue; + if (typeof root.props?.returns === 'string') coveredReturns.add(root.props.returns); + for (const child of root.children ?? []) { + if (child.type === 'param' && typeof child.props?.type === 'string') { + coveredParameters.add(child.props.type); + } + } + } + } + assert.deepEqual( + [...coveredReturns].sort(), + ['boolean', 'boolean[]', 'number', 'number[]', 'string', 'string[]', 'void'], + ); + assert.deepEqual( + [...coveredParameters].sort(), + ['boolean', 'boolean[]', 'number', 'number[]', 'string', 'string[]'], + ); +}); diff --git a/scripts/kern-canonicalizer/fixtures.mjs b/scripts/kern-canonicalizer/fixtures.mjs new file mode 100644 index 000000000..5bf3d351c --- /dev/null +++ b/scripts/kern-canonicalizer/fixtures.mjs @@ -0,0 +1,492 @@ +import { PROFILE_BOUNDARY_FIXTURE } from './profile-limit-fixtures.mjs'; +import { + ESCAPED_OUTPUT_BOUNDARY_FIXTURE, + REVIEW_BOUNDARY_FIXTURES, +} from './review-boundary-fixtures.mjs'; +import { SEMANTIC_BOUNDARY_FIXTURES } from './semantic-boundary-fixtures.mjs'; + +function lines(...items) { + return `${items.join('\n')}\n`; +} + +export const VALID_FIXTURES = [ + { + id: 'shuffled-identifier', + source: lines( + 'fn export=true returns=string name=greet', + ' param type=string name=name', + ' handler lang="kern"', + ' return value="name"', + ), + golden: lines( + 'fn name=greet returns=string export=true', + ' param name=name type=string', + ' handler lang="kern"', + ' return value="name"', + ), + }, + { + id: 'ordered-list-text', + source: lines( + 'fn returns="string[]" name=ordered', + ' param type=string name=first', + ' param name=second type=string', + ' handler lang=kern', + String.raw` return value="[second, \"tab\\tcarriage\\rline\\nquote\\\"slash\\\\\", first]"`, + ), + golden: lines( + 'fn name=ordered returns=string[]', + ' param name=first type=string', + ' param name=second type=string', + ' handler lang="kern"', + String.raw` return value="[second, \"tab\\tcarriage\\rline\\nquote\\\"slash\\\\\", first]"`, + ), + }, + { + id: 'multiple-roots', + source: lines( + 'fn name=zeta returns=boolean export=false', + ' handler lang=kern', + ' return value="false"', + 'fn returns=number name=alpha', + ' handler lang=kern', + ' return value="12"', + ), + golden: lines( + 'fn name=zeta returns=boolean export=false', + ' handler lang="kern"', + ' return value="false"', + 'fn name=alpha returns=number', + ' handler lang="kern"', + ' return value="12"', + ), + }, + { + id: 'null-and-nested-list', + source: lines( + 'fn name=nested returns="string[]"', + ' handler lang=kern', + ' return value="[null,[true,7]]"', + ), + golden: lines( + 'fn name=nested returns=string[]', + ' handler lang="kern"', + ' return value="[null, [true, 7]]"', + ), + }, + { + id: 'remaining-list-types', + source: lines( + 'fn returns="boolean[]" name=flags', + ' handler lang=kern', + ' return value="[true,false]"', + 'fn name=counts returns="number[]"', + ' handler lang=kern', + ' return value="[1,2]"', + ), + golden: lines( + 'fn name=flags returns=boolean[]', + ' handler lang="kern"', + ' return value="[true, false]"', + 'fn name=counts returns=number[]', + ' handler lang="kern"', + ' return value="[1, 2]"', + ), + }, + { + id: 'keyword-shaped-identifiers', + source: lines( + 'fn returns="string[]" name=keywords', + ' param type=string name=let', + ' param name=fn type=string', + ' handler lang=kern', + ' return value="[let,fn]"', + ), + golden: lines( + 'fn name=keywords returns=string[]', + ' param name=let type=string', + ' param name=fn type=string', + ' handler lang="kern"', + ' return value="[let, fn]"', + ), + }, + { + id: 'dollar-structural-identifiers', + source: lines( + 'fn name="$fn" returns=string', + ' param name="$value" type=string', + ' handler lang=kern', + ' return value="$value"', + ), + golden: lines( + 'fn name="$fn" returns=string', + ' param name="$value" type=string', + ' handler lang="kern"', + ' return value="$value"', + ), + }, + { + id: 'void-return', + source: lines('fn export=true name=finished returns=void', ' handler lang=kern', ' return'), + golden: lines('fn name=finished returns=void export=true', ' handler lang="kern"', ' return'), + }, + { + id: 'parameter-types', + source: lines( + 'fn returns=void name=accepts', + ' param type=string name=textValue', + ' param name=flagValue type=boolean', + ' param type=number name=countValue', + ' param name=textValues type="string[]"', + ' param type="boolean[]" name=flagValues', + ' param name=countValues type="number[]"', + ' handler lang=kern', + ' return', + ), + golden: lines( + 'fn name=accepts returns=void', + ' param name=textValue type=string', + ' param name=flagValue type=boolean', + ' param name=countValue type=number', + ' param name=textValues type=string[]', + ' param name=flagValues type=boolean[]', + ' param name=countValues type=number[]', + ' handler lang="kern"', + ' return', + ), + }, + ESCAPED_OUTPUT_BOUNDARY_FIXTURE, + PROFILE_BOUNDARY_FIXTURE, +]; + +function appendTextValue(tables, text, parent = 0, role = '', order = 0) { + tables.valueTag.push('text'); + tables.valueParent.push(parent); + tables.valueRole.push(role); + tables.valueOrder.push(order); + tables.valueText.push(text); + tables.valueBool.push(0); + return tables.valueTag.length; +} + +function appendRootTextValue(tables, text) { + return appendTextValue(tables, text); +} + +function appendRootBoolValue(tables, value) { + tables.valueTag.push('bool'); + tables.valueParent.push(0); + tables.valueRole.push(''); + tables.valueOrder.push(0); + tables.valueText.push(''); + tables.valueBool.push(value ? 1 : 0); + return tables.valueTag.length; +} + +function appendRootTypeValue(tables, kind) { + tables.valueTag.push('record'); + tables.valueParent.push(0); + tables.valueRole.push(''); + tables.valueOrder.push(0); + tables.valueText.push(''); + tables.valueBool.push(0); + const id = tables.valueTag.length; + appendTextValue(tables, kind, id, 'record:kind', 0); + return id; +} + +export const HOSTILE_FIXTURES = [ + ...SEMANTIC_BOUNDARY_FIXTURES, + { + id: 'empty-root-list', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + for (const values of Object.values(tables)) values.length = 0; + }, + }, + { + id: 'node-table-length', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.nodeParent.pop(); + }, + }, + { + id: 'property-table-length', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.propKey.pop(); + }, + }, + { + id: 'value-table-length', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.valueRole.pop(); + }, + }, + { + id: 'non-dense-value-id', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.propValue[0] = tables.valueTag.length + 1; + }, + }, + { + id: 'invalid-node-parent', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.nodeParent[0] = tables.nodeKind.length + 1; + }, + }, + { + id: 'node-cycle', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.nodeParent[0] = 2; + tables.nodeParent[1] = 1; + }, + }, + { + id: 'value-cycle', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const value = tables.propValue[0]; + tables.valueParent[value - 1] = value; + }, + }, + { + id: 'duplicate-sibling-order', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const siblings = tables.nodeParent + .map((parent, index) => ({ index, parent })) + .filter((entry) => entry.parent === 1); + tables.nodeOrder[siblings[1].index] = tables.nodeOrder[siblings[0].index]; + }, + }, + { + id: 'duplicate-required-property', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const value = appendRootTextValue(tables, 'duplicate'); + tables.propNode.push(1); + tables.propKey.push('name'); + tables.propValue.push(value); + }, + }, + { + id: 'duplicate-returns-property', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const value = appendRootTypeValue(tables, 'string'); + tables.propNode.push(1); + tables.propKey.push('returns'); + tables.propValue.push(value); + }, + }, + { + id: 'duplicate-export-property', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const value = appendRootBoolValue(tables, false); + tables.propNode.push(1); + tables.propKey.push('export'); + tables.propValue.push(value); + }, + }, + { + id: 'void-parameter-type', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const param = tables.nodeKind.indexOf('param') + 1; + const typeProperty = tables.propNode.findIndex( + (node, index) => node === param && tables.propKey[index] === 'type', + ); + const typeValue = tables.propValue[typeProperty]; + const kind = tables.valueParent.findIndex( + (parent, index) => parent === typeValue && tables.valueRole[index] === 'record:kind', + ); + tables.valueText[kind] = 'void'; + }, + }, + { + id: 'orphan-value', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + appendRootTextValue(tables, 'orphan'); + }, + }, + { + id: 'unknown-tag', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.valueTag[tables.propValue[0] - 1] = 'future-tag'; + }, + }, + { + id: 'malformed-role', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const child = tables.valueParent.findIndex((parent) => parent > 0); + tables.valueRole[child] = 'unknown-role'; + }, + }, + { + id: 'noncanonical-record-field-order-direct', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const fields = tables.valueRole.indexOf('record:fields'); + const parent = tables.valueParent[fields]; + const kind = tables.valueRole.findIndex( + (role, index) => role === 'record:kind' && tables.valueParent[index] === parent, + ); + const order = tables.valueOrder[fields]; + tables.valueOrder[fields] = tables.valueOrder[kind]; + tables.valueOrder[kind] = order; + }, + }, + { + id: 'duplicate-record-field-direct', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + const fields = tables.valueRole.indexOf('record:fields'); + const parent = tables.valueParent[fields]; + const kind = tables.valueRole.findIndex( + (role, index) => role === 'record:kind' && tables.valueParent[index] === parent, + ); + tables.valueRole[fields] = tables.valueRole[kind]; + }, + }, + { + id: 'unknown-property-with-export', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const value = appendRootTextValue(tables, 'retained'); + tables.propNode.push(1); + tables.propKey.push('future'); + tables.propValue.push(value); + }, + }, + { + id: 'unknown-property-without-export', + base: 'ordered-list-text', + category: 'profile rejection', + mutate(tables) { + const value = appendRootTextValue(tables, 'retained'); + tables.propNode.push(1); + tables.propKey.push('future'); + tables.propValue.push(value); + }, + }, + ...REVIEW_BOUNDARY_FIXTURES, + { + id: 'unsupported-expression-kind', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const kind = tables.valueRole.findIndex( + (role, index) => role === 'record:kind' && tables.valueText[index] === 'identifier', + ); + tables.valueText[kind] = 'binary'; + }, + }, + ...['true', 'false', 'null', 'none', 'undefined', 'await', 'new', 'typeof'].map((reserved) => ({ + id: `reserved-identifier-${reserved}`, + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const name = tables.valueRole.findIndex((role) => role === 'record:name'); + tables.valueText[name] = reserved; + }, + })), + ...Array.from({ length: 32 }, (_, code) => code) + .filter((code) => code !== 9 && code !== 10 && code !== 13) + .map((code) => ({ + id: `unsupported-text-control-${code.toString(16).padStart(2, '0')}`, + base: 'ordered-list-text', + category: 'profile rejection', + mutate(tables) { + const value = tables.valueRole.findIndex( + (role, index) => role === 'record:value' && tables.valueText[index].includes('line'), + ); + tables.valueText[value] = String.fromCharCode(code); + }, + })), + ...Array.from({ length: 33 }, (_, index) => index + 0x7f).map((code) => ({ + id: `unsupported-text-control-${code.toString(16)}`, + base: 'ordered-list-text', + category: 'profile rejection', + mutate(tables) { + const value = tables.valueRole.findIndex( + (role, valueIndex) => role === 'record:value' && tables.valueText[valueIndex].includes('line'), + ); + tables.valueText[value] = String.fromCharCode(code); + }, + })), + { + id: 'unsupported-map-value', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const parent = tables.propValue[tables.propKey.indexOf('returns')]; + const child = tables.valueParent.findIndex((candidate) => candidate === parent); + tables.valueTag[parent - 1] = 'map'; + tables.valueRole[child] = 'map-key'; + tables.valueOrder[child] = 0; + appendTextValue(tables, 'value', parent, 'map-value', 0); + }, + }, + { + id: 'unsupported-error-value', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const parent = tables.propValue[tables.propKey.indexOf('returns')]; + const child = tables.valueParent.findIndex((candidate) => candidate === parent); + tables.valueTag[parent - 1] = 'error'; + tables.valueRole[child] = 'error-code'; + tables.valueOrder[child] = 0; + appendTextValue(tables, 'unsupported profile value', parent, 'error-message', 1); + }, + }, + { + id: 'unsupported-child-kind', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const returned = tables.nodeKind.indexOf('return'); + tables.nodeKind[returned] = 'print'; + }, + }, + { + id: 'valid-prefix-invalid-suffix', + base: 'multiple-roots', + category: 'profile rejection', + mutate(tables) { + const roots = tables.nodeParent + .map((parent, index) => ({ index, parent })) + .filter((entry) => entry.parent === 0); + tables.nodeKind[roots.at(-1).index] = 'class'; + }, + }, +]; diff --git a/scripts/kern-canonicalizer/flatten.mjs b/scripts/kern-canonicalizer/flatten.mjs new file mode 100644 index 000000000..9ad949d80 --- /dev/null +++ b/scripts/kern-canonicalizer/flatten.mjs @@ -0,0 +1,177 @@ +import { rehydrateKirRoots } from './rehydrate.mjs'; + +const VALUE_TAGS = new Set(['null', 'bool', 'text', 'int', 'decimal', 'list', 'record', 'map', 'error']); + +function fail(message) { + throw new TypeError(`adapter rejection: ${message}`); +} + +function plainRecord(value, fields, label) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) fail(`${label} must be a plain record`); + if (Object.getPrototypeOf(value) !== Object.prototype) fail(`${label} must be a plain record`); + const ownKeys = Reflect.ownKeys(value); + const descriptors = Object.getOwnPropertyDescriptors(value); + if ( + ownKeys.some((key) => typeof key === 'symbol') || + ownKeys.some((key) => descriptors[key].get || descriptors[key].set || !descriptors[key].enumerable) + ) { + fail(`${label} must be inspectable plain data`); + } + const keys = ownKeys.sort(); + if (keys.length !== fields.length || keys.some((key, index) => key !== fields[index])) { + fail(`${label} must have exact fields ${fields.join(',')}`); + } + return value; +} + +function denseArray(value, label) { + if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype) { + fail(`${label} must be a dense plain array`); + } + const keys = Reflect.ownKeys(value); + if (keys.length !== value.length + 1 || keys.some((key) => typeof key === 'symbol')) { + fail(`${label} must be a dense plain array`); + } + for (let index = 0; index < value.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + if (!descriptor || descriptor.get || descriptor.set || !descriptor.enumerable) { + fail(`${label} must be a dense plain array`); + } + } + return value; +} + +function text(value, label) { + if (typeof value !== 'string') fail(`${label} must be text`); + return value; +} + +function mark(value, seen) { + if (seen.has(value)) fail('cyclic or shared object graph'); + seen.add(value); +} + +export function flattenKirRoots(input) { + const seen = new WeakSet(); + const roots = denseArray(input, 'roots'); + mark(roots, seen); + const tables = { + nodeKind: [], + nodeParent: [], + nodeOrder: [], + propNode: [], + propKey: [], + propValue: [], + valueTag: [], + valueParent: [], + valueRole: [], + valueOrder: [], + valueText: [], + valueBool: [], + }; + + function addValue(inputValue, parent, role, order, label) { + const value = plainRecord(inputValue, Object.keys(inputValue ?? {}).sort(), label); + mark(value, seen); + const tag = text(value.tag, `${label}.tag`); + if (!VALUE_TAGS.has(tag)) fail(`unknown canonical value tag ${tag}`); + const scalar = tag === 'null' || tag === 'bool' || tag === 'text' || tag === 'int' || tag === 'decimal'; + const expectedFields = tag === 'null' ? ['tag'] : ['tag', 'value']; + plainRecord(value, expectedFields, label); + const id = tables.valueTag.length + 1; + tables.valueTag.push(tag); + tables.valueParent.push(parent); + tables.valueRole.push(role); + tables.valueOrder.push(order); + tables.valueText.push(tag === 'text' || tag === 'int' || tag === 'decimal' ? text(value.value, `${label}.value`) : ''); + tables.valueBool.push(tag === 'bool' ? (value.value === true ? 1 : value.value === false ? 0 : fail(`${label}.value must be boolean`)) : 0); + if (scalar) return id; + + if (tag === 'list') { + const items = denseArray(value.value, `${label}.value`); + mark(items, seen); + items.forEach((item, index) => + addValue(item, id, 'list-item', index, `${label}.value[${index}]`), + ); + return id; + } + if (tag === 'record') { + const entries = denseArray(value.value, `${label}.value`); + mark(entries, seen); + entries.forEach((entryInput, index) => { + const entry = plainRecord(entryInput, ['key', 'value'], `${label}.value[${index}]`); + mark(entry, seen); + addValue(entry.value, id, `record:${text(entry.key, `${label}.value[${index}].key`)}`, index, `${label}.value[${index}].value`); + }); + return id; + } + if (tag === 'map') { + const entries = denseArray(value.value, `${label}.value`); + mark(entries, seen); + entries.forEach((entryInput, index) => { + const entry = plainRecord(entryInput, ['key', 'value'], `${label}.value[${index}]`); + mark(entry, seen); + addValue(entry.key, id, 'map-key', index, `${label}.value[${index}].key`); + addValue(entry.value, id, 'map-value', index, `${label}.value[${index}].value`); + }); + return id; + } + const error = plainRecord(value.value, ['code', 'details', 'message'], `${label}.value`); + mark(error, seen); + addValue({ tag: 'text', value: text(error.code, `${label}.value.code`) }, id, 'error-code', 0, `${label}.value.codeValue`); + addValue( + { tag: 'text', value: text(error.message, `${label}.value.message`) }, + id, + 'error-message', + 1, + `${label}.value.messageValue`, + ); + if (error.details !== null) addValue(error.details, id, 'error-details', 2, `${label}.value.details`); + return id; + } + + function addNode(inputNode, parent, order, label) { + const node = plainRecord(inputNode, ['children', 'kind', 'properties'], label); + mark(node, seen); + const id = tables.nodeKind.length + 1; + tables.nodeKind.push(text(node.kind, `${label}.kind`)); + tables.nodeParent.push(parent); + tables.nodeOrder.push(order); + const properties = denseArray(node.properties, `${label}.properties`); + mark(properties, seen); + properties.forEach((entryInput, index) => { + const entry = plainRecord(entryInput, ['key', 'value'], `${label}.properties[${index}]`); + mark(entry, seen); + tables.propNode.push(id); + tables.propKey.push(text(entry.key, `${label}.properties[${index}].key`)); + tables.propValue.push(addValue(entry.value, 0, '', 0, `${label}.properties[${index}].value`)); + }); + const children = denseArray(node.children, `${label}.children`); + mark(children, seen); + children.forEach((child, index) => + addNode(child, id, index, `${label}.children[${index}]`), + ); + } + + roots.forEach((root, index) => addNode(root, 0, index, `roots[${index}]`)); + // Mandatory: this is the adapter's canonical scalar-spelling and table-shape validation pass. + rehydrateKirRoots(tables); + return tables; +} + +export function tableArguments(tables) { + return [ + tables.nodeKind, + tables.nodeParent, + tables.nodeOrder, + tables.propNode, + tables.propKey, + tables.propValue, + tables.valueTag, + tables.valueParent, + tables.valueRole, + tables.valueOrder, + tables.valueText, + tables.valueBool, + ]; +} diff --git a/scripts/kern-canonicalizer/flatten.test.mjs b/scripts/kern-canonicalizer/flatten.test.mjs new file mode 100644 index 000000000..c67c39880 --- /dev/null +++ b/scripts/kern-canonicalizer/flatten.test.mjs @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; +import ts from 'typescript'; + +import { flattenKirRoots, tableArguments } from './flatten.mjs'; +import { rehydrateKirRoots } from './rehydrate.mjs'; + +function genericRoots() { + return [ + { + kind: 'arbitrary-node', + properties: [ + { + key: 'arbitrary-property', + value: { + tag: 'record', + value: [ + { + key: 'alpha', + value: { + tag: 'list', + value: [ + { tag: 'null' }, + { tag: 'bool', value: true }, + { tag: 'text', value: 'text' }, + { tag: 'int', value: '-12' }, + { tag: 'decimal', value: '1.25' }, + ], + }, + }, + { + key: 'map-value', + value: { + tag: 'map', + value: [ + { + key: { tag: 'text', value: 'key' }, + value: { + tag: 'error', + value: { + code: 'E_GENERIC', + message: 'generic error', + details: { tag: 'null' }, + }, + }, + }, + ], + }, + }, + ], + }, + }, + ], + children: [ + { + kind: 'child-node', + properties: [{ key: 'leaf', value: { tag: 'text', value: 'kept' } }], + children: [], + }, + ], + }, + ]; +} + +function copyTables() { + return structuredClone(flattenKirRoots(genericRoots())); +} + +test('generic flattening rehydrates every canonical value tag without semantic loss', () => { + const roots = genericRoots(); + const tables = flattenKirRoots(roots); + assert.deepEqual(rehydrateKirRoots(tables), roots); + assert.equal(tableArguments(tables).length, 12); + assert.ok(tableArguments(tables).every(Array.isArray)); +}); + +test('the adapter rejects cyclic and shared object graphs before id allocation', () => { + const root = genericRoots()[0]; + root.children.push(root); + assert.throws(() => flattenKirRoots([root]), /adapter rejection: cyclic or shared object graph/u); + + const shared = []; + const sharedArrays = [{ kind: 'shared-arrays', properties: shared, children: shared }]; + assert.throws(() => flattenKirRoots(sharedArrays), /adapter rejection: cyclic or shared object graph/u); +}); + +test('both adapter directions reject symbol-decorated records', () => { + const roots = genericRoots(); + roots[0][Symbol('unknown')] = true; + assert.throws(() => flattenKirRoots(roots), /adapter rejection: roots\[0\] must/u); + + const tables = copyTables(); + tables[Symbol('unknown')] = []; + assert.throws(() => rehydrateKirRoots(tables), /adapter rejection: tables have unknown fields/u); +}); + +test('both adapter directions reject sparse, decorated, and accessor arrays', () => { + const sparseRoots = []; + sparseRoots.length = 1; + sparseRoots.extra = genericRoots()[0]; + assert.throws(() => flattenKirRoots(sparseRoots), /adapter rejection: roots must be a dense plain array/u); + + const decoratedRoots = genericRoots(); + decoratedRoots[Symbol('extra')] = true; + assert.throws(() => flattenKirRoots(decoratedRoots), /adapter rejection: roots must be a dense plain array/u); + + const accessorRoots = []; + Object.defineProperty(accessorRoots, 0, { enumerable: true, get: () => genericRoots()[0] }); + assert.throws(() => flattenKirRoots(accessorRoots), /adapter rejection: roots must be a dense plain array/u); + + for (const mutate of [ + (tables) => { delete tables.nodeKind[0]; tables.nodeKind.extra = 'arbitrary-node'; }, + (tables) => { tables.nodeKind[Symbol('extra')] = 'arbitrary-node'; }, + (tables) => { + const first = tables.nodeKind[0]; + Object.defineProperty(tables.nodeKind, 0, { enumerable: true, get: () => first }); + }, + ]) { + const tables = copyTables(); + mutate(tables); + assert.throws(() => rehydrateKirRoots(tables), /adapter rejection: nodeKind must be a dense plain array/u); + } +}); + +test('the flattener rejects noncanonical codec shapes before tables escape', () => { + const integer = genericRoots(); + integer[0].properties[0].value.value[0].value.value[3].value = '01'; + assert.throws(() => flattenKirRoots(integer), /adapter rejection:/u, 'noncanonical integer'); + + const unsortedRecord = genericRoots(); + unsortedRecord[0].properties[0].value.value.reverse(); + assert.throws(() => flattenKirRoots(unsortedRecord), /adapter rejection:/u, 'noncanonical record order'); + + const duplicateMap = genericRoots(); + const map = duplicateMap[0].properties[0].value.value.find((entry) => entry.key === 'map-value').value; + map.value.push(structuredClone(map.value[0])); + assert.throws(() => flattenKirRoots(duplicateMap), /adapter rejection:/u, 'duplicate map key'); +}); + +test('the adapter source contains no admitted-profile or formatting literals', () => { + const source = readFileSync(new URL('./flatten.mjs', import.meta.url), 'utf8'); + const parsed = ts.createSourceFile('flatten.mjs', source, ts.ScriptTarget.Latest, true, ts.ScriptKind.JS); + const literals = new Set(); + const visit = (node) => { + if ( + ts.isStringLiteralLike(node) || + node.kind === ts.SyntaxKind.TemplateHead || + node.kind === ts.SyntaxKind.TemplateMiddle || + node.kind === ts.SyntaxKind.TemplateTail + ) { + literals.add(node.text); + } + ts.forEachChild(node, visit); + }; + visit(parsed); + for (const forbidden of [ + 'fn', 'param', 'handler', 'return', 'export', 'returns', 'lang', + 'identifier', 'integer', 'void', 'kern', + 'name', 'type', ' ', + ]) { + assert.equal(literals.has(forbidden), false, `adapter leaked semantic/formatting literal ${forbidden}`); + } + assert.equal([...literals].some((literal) => literal.includes('value=')), false, 'adapter leaked source syntax'); +}); + +test('rehydration rejects malformed table lengths, ids, cycles, orders, and orphans', () => { + const cases = [ + ['node table length', (tables) => tables.nodeParent.pop()], + ['property table length', (tables) => tables.propKey.pop()], + ['value table length', (tables) => tables.valueRole.pop()], + ['non-dense value id', (tables) => { tables.propValue[0] = tables.valueTag.length + 1; }], + ['invalid node parent', (tables) => { tables.nodeParent[0] = tables.nodeKind.length + 1; }], + ['node cycle', (tables) => { tables.nodeParent[0] = 2; tables.nodeParent[1] = 1; }], + ['value cycle', (tables) => { tables.valueParent[0] = 1; }], + ['duplicate sibling order', (tables) => { tables.nodeOrder[1] = tables.nodeOrder[0]; tables.nodeParent[1] = tables.nodeParent[0]; }], + ['orphan root value', (tables) => { tables.propValue[0] = 2; }], + ['malformed role', (tables) => { tables.valueRole[1] = 'unknown-role'; }], + ['nonzero root order', (tables) => { tables.valueOrder[tables.propValue[0] - 1] = 7; }], + ['malformed error order', (tables) => { + const message = tables.valueRole.indexOf('error-message'); + tables.valueOrder[message] = 0; + }], + ]; + for (const [name, mutate] of cases) { + const tables = copyTables(); + mutate(tables); + assert.throws(() => rehydrateKirRoots(tables), /adapter rejection:/u, name); + } +}); + +test('rehydration rejects noncanonical scalar, record, and map codec shapes', () => { + const integer = copyTables(); + integer.valueText[integer.valueTag.indexOf('int')] = '01'; + assert.throws(() => rehydrateKirRoots(integer), /adapter rejection:/u, 'noncanonical integer'); + + const decimal = copyTables(); + decimal.valueText[decimal.valueTag.indexOf('decimal')] = '01.25'; + assert.throws(() => rehydrateKirRoots(decimal), /adapter rejection:/u, 'noncanonical decimal'); + + const unsortedRecord = genericRoots(); + unsortedRecord[0].properties[0].value.value.reverse(); + assert.throws( + () => rehydrateKirRoots(flattenKirRoots(unsortedRecord)), + /adapter rejection:/u, + 'record keys must be canonical-code-point sorted', + ); + + const duplicateMap = genericRoots(); + const map = duplicateMap[0].properties[0].value.value.find((entry) => entry.key === 'map-value').value; + map.value.push(structuredClone(map.value[0])); + assert.throws( + () => rehydrateKirRoots(flattenKirRoots(duplicateMap)), + /adapter rejection:/u, + 'map scalar keys must be unique', + ); +}); + +test('unknown canonical tags are rejected instead of omitted', () => { + const roots = genericRoots(); + roots[0].properties[0].value.tag = 'future-tag'; + assert.throws(() => flattenKirRoots(roots), /adapter rejection: unknown canonical value tag/u); +}); diff --git a/scripts/kern-canonicalizer/policy.json b/scripts/kern-canonicalizer/policy.json new file mode 100644 index 000000000..848b395b0 --- /dev/null +++ b/scripts/kern-canonicalizer/policy.json @@ -0,0 +1,31 @@ +{ + "expansionLimits": { + "kirToSourceMaxFactor": 4, + "runtimeEnvelopeMaxFactor": 2 + }, + "kirLimits": { + "maxBytes": 262144, + "maxDepth": 64, + "maxNodes": 4096, + "maxStringBytes": 8192, + "maxCollectionLength": 1024, + "maxRecordFields": 512, + "maxMapEntries": 64, + "maxIntegerDigits": 256, + "maxFractionDigits": 256, + "maxDecimalChars": 520 + }, + "runtimeLimits": { + "maxBytes": 2097152, + "maxCollectionLength": 65536, + "maxDepth": 64, + "maxDiagnostics": 8, + "maxEvents": 64, + "maxStringBytes": 1048576 + }, + "profileLimits": { + "maxNodeRows": 16, + "maxPropertyRows": 30, + "maxValueRows": 72 + } +} diff --git a/scripts/kern-canonicalizer/policy.mjs b/scripts/kern-canonicalizer/policy.mjs new file mode 100644 index 000000000..061ae9d90 --- /dev/null +++ b/scripts/kern-canonicalizer/policy.mjs @@ -0,0 +1,55 @@ +import { readFileSync } from 'node:fs'; + +const POLICY_KEYS = { + expansionLimits: ['kirToSourceMaxFactor', 'runtimeEnvelopeMaxFactor'], + kirLimits: [ + 'maxBytes', 'maxCollectionLength', 'maxDecimalChars', 'maxDepth', + 'maxFractionDigits', 'maxIntegerDigits', 'maxMapEntries', 'maxNodes', + 'maxRecordFields', 'maxStringBytes', + ], + profileLimits: ['maxNodeRows', 'maxPropertyRows', 'maxValueRows'], + runtimeLimits: [ + 'maxBytes', 'maxCollectionLength', 'maxDepth', 'maxDiagnostics', 'maxEvents', + 'maxStringBytes', + ], +}; + +function fail(message) { + throw new TypeError(`canonicalizer policy rejection: ${message}`); +} + +function assertExactKeys(value, expected, label) { + if (value === null || typeof value !== 'object' || Array.isArray(value)) fail(`${label} must be a record`); + const actual = Object.keys(value).sort(); + const sortedExpected = [...expected].sort(); + if (actual.length !== sortedExpected.length || actual.some((key, index) => key !== sortedExpected[index])) { + fail(`${label} must contain exactly ${sortedExpected.join(',')}`); + } +} + +export function validateCanonicalizerPolicy(policy) { + assertExactKeys(policy, Object.keys(POLICY_KEYS), 'policy'); + for (const [label, keys] of Object.entries(POLICY_KEYS)) { + const group = policy[label]; + assertExactKeys(group, keys, label); + for (const [key, value] of Object.entries(group)) { + if (!Number.isSafeInteger(value) || value <= 0) fail(`${label}.${key} must be a positive safe integer`); + } + } + const requiredStringBytes = policy.kirLimits.maxBytes * policy.expansionLimits.kirToSourceMaxFactor; + if (!Number.isSafeInteger(requiredStringBytes) || policy.runtimeLimits.maxStringBytes < requiredStringBytes) { + fail('runtimeLimits.maxStringBytes must cover the configured KIR-to-source expansion'); + } + const requiredEnvelopeBytes = + policy.runtimeLimits.maxStringBytes * policy.expansionLimits.runtimeEnvelopeMaxFactor; + if (!Number.isSafeInteger(requiredEnvelopeBytes) || policy.runtimeLimits.maxBytes < requiredEnvelopeBytes) { + fail('runtimeLimits.maxBytes must cover the configured runtime envelope expansion'); + } + return policy; +} + +export function loadCanonicalizerPolicy() { + return validateCanonicalizerPolicy( + JSON.parse(readFileSync(new URL('./policy.json', import.meta.url), 'utf8')), + ); +} diff --git a/scripts/kern-canonicalizer/profile-limit-fixtures.mjs b/scripts/kern-canonicalizer/profile-limit-fixtures.mjs new file mode 100644 index 000000000..be82915df --- /dev/null +++ b/scripts/kern-canonicalizer/profile-limit-fixtures.mjs @@ -0,0 +1,61 @@ +function lines(...items) { + return `${items.join('\n')}\n`; +} + +const BOUNDARY_PARAMETERS = Array.from({ length: 13 }, (_, index) => ` param name=p${index} type=number`); + +export const PROFILE_BOUNDARY_FIXTURE = { + id: 'profile-row-boundary', + expectedRows: { nodes: 16, properties: 30, values: 72 }, + source: lines( + 'fn name=bounded returns="number[]"', + ...BOUNDARY_PARAMETERS, + ' handler lang=kern', + ' return value="[0,1,2,3,4,5]"', + ), + golden: lines( + 'fn name=bounded returns=number[]', + ...BOUNDARY_PARAMETERS, + ' handler lang="kern"', + ' return value="[0, 1, 2, 3, 4, 5]"', + ), +}; + +export const PROFILE_LIMIT_FIXTURES = [ + { + id: 'over-node-row-limit', + expectedRows: { nodes: 17, properties: 19, values: 26 }, + source: lines( + 'fn name=f0 returns=void', + ' param name=a type=number', + ' param name=b type=number', + ' handler lang=kern', + ' return', + ...Array.from({ length: 4 }, (_, index) => [ + `fn name=f${index + 1} returns=void`, + ' handler lang=kern', + ' return', + ]).flat(), + ), + }, + { + id: 'over-property-row-limit', + expectedRows: { nodes: 16, properties: 31, values: 69 }, + source: lines( + 'fn name=properties returns="number[]" export=true', + ...BOUNDARY_PARAMETERS, + ' handler lang=kern', + ' return value="[0,1,2,3,4]"', + ), + }, + { + id: 'over-value-row-limit', + expectedRows: { nodes: 16, properties: 30, values: 76 }, + source: lines( + 'fn name=values returns="number[]"', + ...BOUNDARY_PARAMETERS, + ' handler lang=kern', + ' return value="[0,1,2,3,4,5,6]"', + ), + }, +]; diff --git a/scripts/kern-canonicalizer/rehydrate.mjs b/scripts/kern-canonicalizer/rehydrate.mjs new file mode 100644 index 000000000..7f95bf35b --- /dev/null +++ b/scripts/kern-canonicalizer/rehydrate.mjs @@ -0,0 +1,284 @@ +const TABLE_KEYS = [ + 'nodeKind', 'nodeParent', 'nodeOrder', 'propNode', 'propKey', 'propValue', + 'valueTag', 'valueParent', 'valueRole', 'valueOrder', 'valueText', 'valueBool', +]; +const VALUE_TAGS = new Set(['null', 'bool', 'text', 'int', 'decimal', 'list', 'record', 'map', 'error']); +const SCALAR_TAGS = new Set(['null', 'bool', 'text', 'int', 'decimal']); +const textEncoder = new TextEncoder(); + +function fail(message) { + throw new TypeError(`adapter rejection: ${message}`); +} + +function denseArray(value, label) { + if (!Array.isArray(value) || Object.getPrototypeOf(value) !== Array.prototype) { + fail(`${label} must be a dense plain array`); + } + const keys = Reflect.ownKeys(value); + if (keys.length !== value.length + 1 || keys.some((key) => typeof key === 'symbol')) { + fail(`${label} must be a dense plain array`); + } + for (let index = 0; index < value.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + if (!descriptor || descriptor.get || descriptor.set || !descriptor.enumerable) { + fail(`${label} must be a dense plain array`); + } + } + return value; +} + +function safeInt(value, label, minimum = 0) { + if (!Number.isSafeInteger(value) || value < minimum) fail(`${label} must be an integer >= ${minimum}`); + return value; +} + +function text(value, label) { + if (typeof value !== 'string') fail(`${label} must be text`); + return value; +} + +function compareCodePoints(left, right) { + const leftPoints = [...left]; + const rightPoints = [...right]; + const length = Math.min(leftPoints.length, rightPoints.length); + for (let index = 0; index < length; index += 1) { + const difference = leftPoints[index].codePointAt(0) - rightPoints[index].codePointAt(0); + if (difference !== 0) return difference; + } + return leftPoints.length - rightPoints.length; +} + +function compareBytes(left, right) { + const length = Math.min(left.length, right.length); + for (let index = 0; index < length; index += 1) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return left.length - right.length; +} + +function scalarBytes(value) { + const normalized = value.tag === 'null' ? { tag: value.tag } : { tag: value.tag, value: value.value }; + return textEncoder.encode(JSON.stringify(normalized)); +} + +function exactTables(input) { + if (input === null || typeof input !== 'object' || Array.isArray(input) || Object.getPrototypeOf(input) !== Object.prototype) { + fail('tables must be a plain record'); + } + const keys = Reflect.ownKeys(input); + const expected = [...TABLE_KEYS].sort(); + if (keys.some((key) => typeof key === 'symbol')) fail('tables have unknown fields'); + const descriptors = Object.getOwnPropertyDescriptors(input); + if (keys.some((key) => descriptors[key].get || descriptors[key].set || !descriptors[key].enumerable)) { + fail('tables must be inspectable plain data'); + } + keys.sort(); + if (keys.length !== expected.length || keys.some((key, index) => key !== expected[index])) { + fail('tables have unknown fields'); + } + for (const key of TABLE_KEYS) denseArray(input[key], key); + if (input.nodeKind.length !== input.nodeParent.length || input.nodeKind.length !== input.nodeOrder.length) { + fail('node table lengths differ'); + } + if (input.propNode.length !== input.propKey.length || input.propNode.length !== input.propValue.length) { + fail('property table lengths differ'); + } + const valueLength = input.valueTag.length; + for (const key of ['valueParent', 'valueRole', 'valueOrder', 'valueText', 'valueBool']) { + if (input[key].length !== valueLength) fail('value table lengths differ'); + } + return input; +} + +function assertParentGraph(parents, label) { + for (let index = 0; index < parents.length; index += 1) { + let current = index + 1; + for (let depth = 0; depth <= parents.length; depth += 1) { + const parent = safeInt(parents[current - 1], `${label}[${current - 1}]`); + if (parent === 0) break; + if (parent > parents.length) fail(`${label}[${current - 1}] points outside the table`); + if (parent >= current) fail(`${label}[${current - 1}] must reference an earlier row`); + current = parent; + if (depth === parents.length) fail(`${label} contains a cycle`); + } + } +} + +function assertSiblingOrders(parents, orders, label) { + const byParent = new Map(); + for (let index = 0; index < parents.length; index += 1) { + const parent = parents[index]; + const order = safeInt(orders[index], `${label}Order[${index}]`); + const group = byParent.get(parent) ?? []; + group.push(order); + byParent.set(parent, group); + } + for (const [parent, group] of byParent) { + const sorted = [...group].sort((left, right) => left - right); + if (sorted.some((order, index) => order !== index)) fail(`${label} siblings of ${parent} must have dense unique order`); + } +} + +export function rehydrateKirRoots(input) { + const tables = exactTables(input); + const nodeCount = tables.nodeKind.length; + const valueCount = tables.valueTag.length; + assertParentGraph(tables.nodeParent, 'nodeParent'); + assertParentGraph(tables.valueParent, 'valueParent'); + assertSiblingOrders(tables.nodeParent, tables.nodeOrder, 'node'); + + const valuesByParent = Array.from({ length: valueCount + 1 }, () => []); + for (let index = 0; index < valueCount; index += 1) { + const tag = text(tables.valueTag[index], `valueTag[${index}]`); + if (!VALUE_TAGS.has(tag)) fail(`unknown canonical value tag ${tag}`); + const parent = safeInt(tables.valueParent[index], `valueParent[${index}]`); + if (parent > valueCount) fail(`valueParent[${index}] points outside the table`); + text(tables.valueRole[index], `valueRole[${index}]`); + const order = safeInt(tables.valueOrder[index], `valueOrder[${index}]`); + if (parent === 0 && order !== 0) fail(`root value ${index + 1} must have order 0`); + text(tables.valueText[index], `valueText[${index}]`); + if (tables.valueBool[index] !== 0 && tables.valueBool[index] !== 1) fail(`valueBool[${index}] must be 0 or 1`); + if (parent > 0) valuesByParent[parent].push(index + 1); + } + + const rootReferences = new Array(valueCount + 1).fill(0); + const propsByNode = Array.from({ length: nodeCount + 1 }, () => []); + for (let index = 0; index < tables.propNode.length; index += 1) { + const node = safeInt(tables.propNode[index], `propNode[${index}]`, 1); + const value = safeInt(tables.propValue[index], `propValue[${index}]`, 1); + if (node > nodeCount || value > valueCount) fail(`property row ${index} points outside a table`); + if (tables.valueParent[value - 1] !== 0 || tables.valueRole[value - 1] !== '') fail(`property row ${index} must point to a root value`); + rootReferences[value] += 1; + propsByNode[node].push({ key: text(tables.propKey[index], `propKey[${index}]`), value }); + } + for (let id = 1; id <= valueCount; id += 1) { + if (tables.valueParent[id - 1] === 0 && rootReferences[id] !== 1) fail(`root value ${id} must have one property owner`); + if (tables.valueParent[id - 1] !== 0 && rootReferences[id] !== 0) fail(`child value ${id} cannot be a property root`); + } + for (let id = 1; id <= nodeCount; id += 1) { + const keys = propsByNode[id].map((entry) => entry.key); + if (new Set(keys).size !== keys.length) fail(`node ${id} has duplicate properties`); + } + + const cache = new Map(); + function valueAt(id) { + if (cache.has(id)) return cache.get(id); + const index = id - 1; + const tag = tables.valueTag[index]; + const children = valuesByParent[id]; + const textValue = tables.valueText[index]; + const boolValue = tables.valueBool[index]; + if (SCALAR_TAGS.has(tag)) { + if (children.length !== 0) fail(`scalar value ${id} cannot have children`); + if (tag === 'null') { + if (textValue !== '' || boolValue !== 0) fail(`null value ${id} has payload`); + return { tag: 'null' }; + } + if (tag === 'bool') { + if (textValue !== '') fail(`boolean value ${id} has text payload`); + return { tag: 'bool', value: boolValue === 1 }; + } + if (boolValue !== 0) fail(`${tag} value ${id} has boolean payload`); + if (tag === 'int' && !/^(?:0|-?[1-9][0-9]*)$/u.test(textValue)) { + fail(`integer value ${id} is not canonical`); + } + if (tag === 'decimal') { + const match = /^-?(0|[1-9][0-9]*)\.([0-9]+)$/u.exec(textValue); + if (!match || (textValue.startsWith('-') && match[1] === '0' && /^0+$/u.test(match[2]))) { + fail(`decimal value ${id} is not canonical`); + } + } + return { tag, value: textValue }; + } + if (textValue !== '' || boolValue !== 0) fail(`composite value ${id} has scalar payload`); + const ordered = [...children].sort((left, right) => tables.valueOrder[left - 1] - tables.valueOrder[right - 1]); + if (tag === 'list') { + if (ordered.some((child, order) => tables.valueRole[child - 1] !== 'list-item' || tables.valueOrder[child - 1] !== order)) { + fail(`list value ${id} has malformed role or order`); + } + return { tag: 'list', value: ordered.map(valueAt) }; + } + if (tag === 'record') { + const entries = ordered.map((child, order) => { + const role = tables.valueRole[child - 1]; + if (!role.startsWith('record:') || tables.valueOrder[child - 1] !== order) fail(`record value ${id} has malformed role or order`); + return { key: role.slice('record:'.length), value: valueAt(child) }; + }); + for (let index = 1; index < entries.length; index += 1) { + const comparison = compareCodePoints(entries[index - 1].key, entries[index].key); + if (comparison >= 0) fail(`record value ${id} has duplicate or noncanonical key order`); + } + return { tag: 'record', value: entries }; + } + if (tag === 'map') { + const byOrder = new Map(); + for (const child of ordered) { + const order = tables.valueOrder[child - 1]; + const role = tables.valueRole[child - 1]; + if (role !== 'map-key' && role !== 'map-value') fail(`map value ${id} has malformed role`); + const entry = byOrder.get(order) ?? {}; + if (entry[role] !== undefined) fail(`map value ${id} has duplicate ${role}`); + entry[role] = child; + byOrder.set(order, entry); + } + const orders = [...byOrder.keys()].sort((left, right) => left - right); + if (orders.some((order, index) => order !== index)) fail(`map value ${id} has sparse order`); + const entries = orders.map((order) => { + const entry = byOrder.get(order); + if (entry['map-key'] === undefined || entry['map-value'] === undefined) fail(`map value ${id} has incomplete entry`); + const key = valueAt(entry['map-key']); + if (!SCALAR_TAGS.has(key.tag)) fail(`map value ${id} has non-scalar key`); + return { key, value: valueAt(entry['map-value']) }; + }); + let previous; + for (const entry of entries) { + const encoded = scalarBytes(entry.key); + if (previous !== undefined && compareBytes(previous, encoded) >= 0) { + fail(`map value ${id} has duplicate or noncanonical key order`); + } + previous = encoded; + } + return { tag: 'map', value: entries }; + } + const roles = new Map(children.map((child) => [tables.valueRole[child - 1], child])); + if (roles.size !== children.length || !roles.has('error-code') || !roles.has('error-message')) fail(`error value ${id} has malformed fields`); + if ([...roles.keys()].some((role) => !['error-code', 'error-message', 'error-details'].includes(role))) fail(`error value ${id} has unknown field`); + for (const [role, order] of [['error-code', 0], ['error-message', 1], ['error-details', 2]]) { + const child = roles.get(role); + if (child !== undefined && tables.valueOrder[child - 1] !== order) fail(`error value ${id} has malformed order`); + } + const code = valueAt(roles.get('error-code')); + const message = valueAt(roles.get('error-message')); + if (code.tag !== 'text' || message.tag !== 'text') fail(`error value ${id} code and message must be text`); + if (!/^[A-Za-z_][A-Za-z0-9_.-]*$/u.test(code.value)) fail(`error value ${id} code is not canonical`); + return { + tag: 'error', + value: { + code: code.value, + message: message.value, + details: roles.has('error-details') ? valueAt(roles.get('error-details')) : null, + }, + }; + } + + const childrenByNode = Array.from({ length: nodeCount + 1 }, () => []); + for (let index = 0; index < nodeCount; index += 1) { + text(tables.nodeKind[index], `nodeKind[${index}]`); + const parent = safeInt(tables.nodeParent[index], `nodeParent[${index}]`); + if (parent > nodeCount) fail(`nodeParent[${index}] points outside the table`); + if (parent > 0) childrenByNode[parent].push(index + 1); + } + function nodeAt(id) { + const children = [...childrenByNode[id]].sort((left, right) => tables.nodeOrder[left - 1] - tables.nodeOrder[right - 1]); + return { + kind: tables.nodeKind[id - 1], + properties: propsByNode[id].map((entry) => ({ key: entry.key, value: valueAt(entry.value) })), + children: children.map(nodeAt), + }; + } + return tables.nodeParent + .map((parent, index) => ({ parent, id: index + 1 })) + .filter((entry) => entry.parent === 0) + .sort((left, right) => tables.nodeOrder[left.id - 1] - tables.nodeOrder[right.id - 1]) + .map((entry) => nodeAt(entry.id)); +} diff --git a/scripts/kern-canonicalizer/review-boundary-fixtures.mjs b/scripts/kern-canonicalizer/review-boundary-fixtures.mjs new file mode 100644 index 000000000..ee3a289ae --- /dev/null +++ b/scripts/kern-canonicalizer/review-boundary-fixtures.mjs @@ -0,0 +1,52 @@ +import { loadCanonicalizerPolicy } from './policy.mjs'; + +export const REVIEW_BOUNDARY_FIXTURES = [ + { + id: 'unsupported-decimal-value', + base: 'multiple-roots', + category: 'profile rejection', + mutate(tables) { + const integer = tables.valueTag.indexOf('int'); + tables.valueTag[integer] = 'decimal'; + tables.valueText[integer] = '1.5'; + }, + }, + { + id: 'unsupported-negative-integer-expression', + base: 'multiple-roots', + category: 'profile rejection', + mutate(tables) { + const integer = tables.valueRole.findIndex( + (role, index) => role === 'record:value' && tables.valueTag[index] === 'int', + ); + tables.valueText[integer] = '-12'; + }, + }, + { + id: 'nonzero-root-value-order', + base: 'shuffled-identifier', + category: 'direct profile rejection', + mutate(tables) { + tables.valueOrder[tables.propValue[0] - 1] = 7; + }, + }, +]; + +const boundaryText = '\\'.repeat(loadCanonicalizerPolicy().kirLimits.maxStringBytes); +const boundaryExpression = `[${JSON.stringify(boundaryText)}, ${JSON.stringify(boundaryText)}]`; + +export const ESCAPED_OUTPUT_BOUNDARY_FIXTURE = { + id: 'escaped-output-boundary', + source: [ + 'fn name=escapedBoundary returns="string[]"', + ' handler lang=kern', + ` return value=${JSON.stringify(boundaryExpression)}`, + '', + ].join('\n'), + golden: [ + 'fn name=escapedBoundary returns=string[]', + ' handler lang="kern"', + ` return value=${JSON.stringify(boundaryExpression)}`, + '', + ].join('\n'), +}; diff --git a/scripts/kern-canonicalizer/semantic-boundary-fixtures.mjs b/scripts/kern-canonicalizer/semantic-boundary-fixtures.mjs new file mode 100644 index 000000000..17b9c18dd --- /dev/null +++ b/scripts/kern-canonicalizer/semantic-boundary-fixtures.mjs @@ -0,0 +1,76 @@ +function appendTextValue(tables, text, parent, role, order) { + tables.valueTag.push('text'); + tables.valueParent.push(parent); + tables.valueRole.push(role); + tables.valueOrder.push(order); + tables.valueText.push(text); + tables.valueBool.push(0); +} + +export const SEMANTIC_BOUNDARY_FIXTURES = [ + { + id: 'duplicate-function-name', + base: 'multiple-roots', + category: 'profile rejection', + mutate(tables) { + const roots = tables.nodeParent + .map((parent, index) => ({ id: index + 1, parent })) + .filter((entry) => entry.parent === 0); + const names = roots.map((root) => { + const property = tables.propNode.findIndex( + (node, index) => node === root.id && tables.propKey[index] === 'name', + ); + return tables.propValue[property] - 1; + }); + tables.valueText[names[1]] = tables.valueText[names[0]]; + }, + }, + { + id: 'duplicate-parameter-name', + base: 'ordered-list-text', + category: 'profile rejection', + mutate(tables) { + const params = tables.nodeKind + .map((kind, index) => ({ id: index + 1, kind })) + .filter((entry) => entry.kind === 'param'); + const names = params.map((param) => { + const property = tables.propNode.findIndex( + (node, index) => node === param.id && tables.propKey[index] === 'name', + ); + return tables.propValue[property] - 1; + }); + tables.valueText[names[1]] = tables.valueText[names[0]]; + }, + }, + { + id: 'nested-list-type', + base: 'remaining-list-types', + category: 'profile rejection', + mutate(tables) { + const element = tables.valueRole.findIndex((role) => role === 'record:element'); + tables.valueTag[element] = 'record'; + tables.valueText[element] = ''; + appendTextValue(tables, 'text', element + 1, 'record:kind', 0); + }, + }, + { + id: 'non-kern-handler', + base: 'shuffled-identifier', + category: 'profile rejection', + mutate(tables) { + const langProperty = tables.propKey.indexOf('lang'); + tables.valueText[tables.propValue[langProperty] - 1] = 'ts'; + }, + }, + ...[0x2028, 0x2029, 0xfeff].map((code) => ({ + id: `unsupported-text-separator-${code.toString(16)}`, + base: 'ordered-list-text', + category: 'profile rejection', + mutate(tables) { + const value = tables.valueRole.findIndex( + (role, valueIndex) => role === 'record:value' && tables.valueText[valueIndex].includes('line'), + ); + tables.valueText[value] = String.fromCodePoint(code); + }, + })), +];