diff --git a/docs/rfc-detailed-flag-evaluation.md b/docs/rfc-detailed-flag-evaluation.md new file mode 100644 index 00000000..1ac1e22b --- /dev/null +++ b/docs/rfc-detailed-flag-evaluation.md @@ -0,0 +1,830 @@ +# RFC: Detailed Flag Evaluation Trace for DD Flags Node SDK + +**Status:** Draft +**Date:** 2026-04-14 +**Author:** Tyler Potter + +--- + +## Summary + +Add a `DDFlagEvaluationDetails` type to the DD Flags Node SDK that surfaces the full +allocation waterfall trace — which allocations were evaluated and why each one was +skipped or matched. This data lives in the DD-internal evaluation layer and is +serialized to a JSON string at the OpenFeature provider boundary, where it is stored +in `flagMetadata`. + +--- + +## Motivation + +The current `evaluateForSubject()` function iterates through a flag's `allocations` +array in order (the "waterfall") and returns as soon as one matches. The resolved +`ResolutionDetails` tells callers: + +- Which value was returned +- A coarse reason (`TARGETING_MATCH`, `DEFAULT`, `DISABLED`, `ERROR`) +- Which allocation key matched (`flagMetadata.allocationKey`) + +It does **not** tell callers: + +- Which allocations were evaluated before the match (or before falling through) +- Why each skipped allocation did not match (date range? targeting rules? traffic split?) +- Which targeting rule triggered the match +- Which allocations were never reached because the waterfall terminated earlier + +This gap makes several use cases impractical or impossible: + +- **Debugging mismatches** — a user is unexpectedly in (or out of) a treatment group; + there is no way to see which allocations they were evaluated against. +- **Observability / telemetry** — there is a `targetingRuleKey` field in + `FlagEvaluationEvent` and `FlagEvaluationAggregationKey` that is never populated + because the evaluator discards the matched rule before returning. This RFC captures + the matched rule index, which partially closes this gap; fully populating + `targetingRuleKey` as a string requires a config format change (see Implementation + Prerequisites). +- **Testing** — asserting which branch of the waterfall was exercised requires + inspecting debug logs, which are not machine-readable. + +### Prior Art: EPPO + +The EPPO JS SDK solves this with a `FlagEvaluationDetailsBuilder` pattern +(`js-sdk-common/src/flag-evaluation-details-builder.ts`). The evaluator calls +`builder.addUnmatchedAllocation()` for each allocation that is skipped, records the +reason code, then calls `builder.setMatch()` when a match is found. Allocations after +the match point are automatically classified as `UNEVALUATED`. + +EPPO's `AllocationEvaluationCode` enum: + +``` +UNEVALUATED — never reached +MATCH — selected +BEFORE_START_TIME — skipped: too early +AFTER_END_TIME — skipped: too late +FAILING_RULE — skipped: targeting rules did not match +TRAFFIC_EXPOSURE_MISS — skipped: rules matched, subject outside split ranges +``` + +The DD design follows this structure with naming adjustments to match DD conventions +and additional cases required by the DD evaluator. + +--- + +## Background: How the Evaluator Works Today + +There are two evaluation entry points with different responsibilities: + +``` +evaluate() [evaluation.ts] + ├─ config loaded? → ERROR (PROVIDER_NOT_READY) + ├─ targetingKey present? → ERROR (TARGETING_KEY_MISSING) + ├─ flag exists? → ERROR (FLAG_NOT_FOUND) + └─ evaluateForSubject() [evaluateForSubject.ts] + ├─ flag.enabled? → DISABLED + ├─ type matches? → TYPE_MISMATCH error + └─ for each allocation: + ├─ now < startAt? → continue (silent) + ├─ now >= endAt? → continue (silent) + ├─ containsMatchingRule()? + │ → false → continue (silent) + └─ selectSplitUsingSharding()? + → null → continue (silent) + → split → variant = flag.variations[split.variationKey] + ├─ variant exists → return TARGETING_MATCH ✓ + └─ variant missing → continue (silent, config corrupt) + └─ (fell through) → return DEFAULT +``` + +Every `continue` discards evaluation state that is only surfaced in debug logs. +`ResolutionDetails` carries none of it. Notably, the `variant-not-found` fall-through +(split selected but `flag.variations[variationKey]` is undefined) is a silent failure +that is not currently distinguishable from a normal TRAFFIC_MISS. + +`config.createdAt` and `config.environment.name` are already present on +`UniversalFlagConfigurationV1` and available inside `evaluate()`, but are never +threaded into `evaluateForSubject()`. + +--- + +## Proposed Design + +### Layer Separation + +``` +┌──────────────────────────────────────────────────────────┐ +│ evaluate() [evaluation.ts] │ +│ Handles: config-not-loaded, missing targetingKey, │ +│ flag-not-found, unhandled exceptions │ +│ Returns: DDFlagEvaluationDetails (pre-waterfall cases) │ +└──────────────────┬───────────────────────────────────────┘ + │ +┌──────────────────▼───────────────────────────────────────┐ +│ evaluateForSubject() [evaluateForSubject.ts] │ +│ Handles: disabled, type mismatch, waterfall │ +│ Returns: DDFlagEvaluationDetails (rich structured trace)│ +└──────────────────┬───────────────────────────────────────┘ + │ provider maps to ResolutionDetails +┌──────────────────▼───────────────────────────────────────┐ +│ OpenFeature provider boundary │ +│ ResolutionDetails.flagMetadata │ +│ { ddEvaluationTrace: string } (opt-in serialization) │ +└──────────────────────────────────────────────────────────┘ +``` + +Both `evaluate()` and `evaluateForSubject()` return `DDFlagEvaluationDetails`. The +provider calls `toResolutionDetails()` on the result of `evaluate()`, making all +paths — including pre-waterfall errors — observable through the same type. + +--- + +### New Types + +**File:** `packages/node-server/src/configuration/flagEvaluationDetails.ts` + +```typescript +/** + * Why an individual allocation was skipped or selected during waterfall evaluation. + */ +export enum AllocationOutcomeCode { + /** This allocation matched and its variant was returned. */ + MATCH = 'MATCH', + /** Skipped: current time is before allocation.startAt. */ + BEFORE_START_TIME = 'BEFORE_START_TIME', + /** Skipped: current time is at or after allocation.endAt. */ + AFTER_END_TIME = 'AFTER_END_TIME', + /** Skipped: allocation has targeting rules and none of them matched. */ + RULES_MISMATCH = 'RULES_MISMATCH', + /** Skipped: targeting rules matched but the subject fell outside all split ranges. */ + TRAFFIC_MISS = 'TRAFFIC_MISS', + /** + * Skipped: split was selected but the variationKey it references does not exist in + * flag.variations. This indicates a corrupt or inconsistent flag configuration. + * The waterfall continues to the next allocation. + */ + MISSING_VARIATION = 'MISSING_VARIATION', + /** Never evaluated: the waterfall terminated (matched or errored) before reaching this allocation. */ + UNEVALUATED = 'UNEVALUATED', +} + +/** + * Trace record for a single allocation in the waterfall. + */ +export interface AllocationOutcome { + /** Matches Allocation.key from the flag configuration. */ + key: string + /** + * 1-indexed position of this allocation in flag.allocations. + * Matches the ordering shown in the flag management UI. + */ + orderPosition: number + outcomeCode: AllocationOutcomeCode + /** + * Whether this allocation defined targeting rules. + * Use this to distinguish "no rules (implicit match-all)" from + * "rules present but matchedRuleIndex is not set" — a condition that + * should not occur but is otherwise ambiguous. + */ + rulesPresent: boolean + /** + * 0-indexed position of the rule that matched within allocation.rules. + * Defined only when rulesPresent is true AND a rule matched + * (i.e. outcomeCode is MATCH, TRAFFIC_MISS, or MISSING_VARIATION). + * Undefined when rulesPresent is false (implicit match-all) or when + * outcomeCode is RULES_MISMATCH. + */ + matchedRuleIndex?: number +} + +/** + * Top-level outcome of a flag evaluation, covering both pre-waterfall + * and waterfall outcomes. + */ +export enum FlagEvaluationOutcomeCode { + /** A matching allocation was found and a variant was returned. */ + MATCH = 'MATCH', + /** Flag is disabled; default value was returned. */ + DISABLED = 'DISABLED', + /** Config is not loaded; default value was returned. */ + PROVIDER_NOT_READY = 'PROVIDER_NOT_READY', + /** targetingKey was not present in evaluation context. */ + TARGETING_KEY_MISSING = 'TARGETING_KEY_MISSING', + /** Flag key not present in configuration; default value was returned. */ + FLAG_NOT_FOUND = 'FLAG_NOT_FOUND', + /** Expected type does not match the flag's variationType; default value was returned. */ + TYPE_MISMATCH = 'TYPE_MISMATCH', + /** All allocations were evaluated and none matched; default value was returned. */ + DEFAULT = 'DEFAULT', + /** An unexpected error occurred during evaluation; default value was returned. */ + ERROR = 'ERROR', +} + +/** + * Detailed result of a flag evaluation. This is the DD-internal type. + * It is NOT directly exposed at the OpenFeature boundary — use toResolutionDetails() + * to convert it there. + * + * unmatchedAllocations and unevaluatedAllocations are empty for all outcome codes + * in PRE_WATERFALL_CODES (PROVIDER_NOT_READY, TARGETING_KEY_MISSING, FLAG_NOT_FOUND, + * DISABLED, TYPE_MISMATCH, ERROR). They are only populated for MATCH and DEFAULT, + * where the waterfall loop ran and allocation-level outcomes were recorded. + */ +export interface DDFlagEvaluationDetails { + flagKey: string + /** The value returned to the caller (may be the supplied default). */ + value: T + /** The key of the selected variation. Null when no allocation matched. */ + variationKey: string | null + /** The key of the matched allocation. Null when no allocation matched. */ + allocationKey: string | null + /** The variationType declared in the flag configuration. Null for pre-waterfall errors. */ + variationType: VariantType | null + /** + * Whether the matched allocation's doLog field was true. + * False when no allocation matched or for pre-waterfall errors. + */ + doLog: boolean + + outcomeCode: FlagEvaluationOutcomeCode + /** Human-readable description of the outcome, suitable for debug logging. */ + outcomeDescription: string + + /** + * The allocation that was selected, or null if none matched. + * Non-null only when outcomeCode is MATCH. + */ + matchedAllocation: AllocationOutcome | null + + /** + * Allocations evaluated but not matched, in evaluation order. + * Empty for pre-waterfall error outcomes (PROVIDER_NOT_READY, TARGETING_KEY_MISSING, + * FLAG_NOT_FOUND, ERROR). + */ + unmatchedAllocations: AllocationOutcome[] + + /** + * Allocations never evaluated because the waterfall terminated first. + * Empty for pre-waterfall error outcomes. + */ + unevaluatedAllocations: AllocationOutcome[] + + /** ISO 8601 timestamp from config.createdAt. Null for pre-waterfall errors. */ + configFetchedAt: string | null + /** Environment name from config.environment.name. Null for pre-waterfall errors. */ + environmentName: string | null +} +``` + +--- + +### Builder + +**File:** `packages/node-server/src/configuration/flagEvaluationDetails.ts` (continued) + +```typescript +/** + * Outcome codes where the waterfall never ran; unevaluated/unmatched lists are + * always empty for these outcomes. + * + * DISABLED and TYPE_MISMATCH are included even though evaluateForSubject has access + * to flag.allocations at that point. The builder is constructed before the enabled/type + * checks but recordUnmatched is never called for them, so evaluatedCount === 0 and + * calculateUnevaluated() would return all allocations as UNEVALUATED — which is + * semantically wrong (the allocations were never candidates for evaluation, the flag + * itself was gated). Including them here suppresses that misleading output. + */ +const PRE_WATERFALL_CODES = new Set([ + FlagEvaluationOutcomeCode.PROVIDER_NOT_READY, + FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING, + FlagEvaluationOutcomeCode.FLAG_NOT_FOUND, + FlagEvaluationOutcomeCode.DISABLED, + FlagEvaluationOutcomeCode.TYPE_MISMATCH, + FlagEvaluationOutcomeCode.ERROR, +]) + +export class DDFlagEvaluationDetailsBuilder { + private variationKey: string | null = null + private allocationKey: string | null = null + private doLog = false + private matchedAllocation: AllocationOutcome | null = null + private readonly unmatchedAllocations: AllocationOutcome[] = [] + + constructor( + private readonly flagKey: string, + private readonly allocations: Allocation[], + private readonly configFetchedAt: string | null, + private readonly environmentName: string | null, + ) {} + + recordUnmatched( + allocation: Allocation, + position: number, + outcomeCode: AllocationOutcomeCode, + matchedRuleIndex?: number, + ): this { + this.unmatchedAllocations.push({ + key: allocation.key, + orderPosition: position, + outcomeCode, + rulesPresent: !!allocation.rules?.length, + matchedRuleIndex, + }) + return this + } + + recordMatch( + allocation: Allocation, + position: number, + variationKey: string, + matchedRuleIndex?: number, + ): this { + // Guard against double-call: recordMatch being called twice would silently overwrite + // the first match and produce a corrupted trace with no record of the first match. + if (this.matchedAllocation !== null) { + throw new Error( + `recordMatch called twice on flag '${this.flagKey}': ` + + `first match was allocation '${this.matchedAllocation.key}', ` + + `second was '${allocation.key}'` + ) + } + this.variationKey = variationKey + this.allocationKey = allocation.key + this.doLog = !!allocation.doLog + this.matchedAllocation = { + key: allocation.key, + orderPosition: position, + outcomeCode: AllocationOutcomeCode.MATCH, + rulesPresent: !!allocation.rules?.length, + matchedRuleIndex, + } + return this + } + + build( + value: T, + outcomeCode: FlagEvaluationOutcomeCode, + outcomeDescription: string, + variationType: VariantType | null, + ): DDFlagEvaluationDetails { + const isPreWaterfall = PRE_WATERFALL_CODES.has(outcomeCode) + return { + flagKey: this.flagKey, + value, + variationKey: this.variationKey, + allocationKey: this.allocationKey, + variationType, + doLog: this.doLog, + outcomeCode, + outcomeDescription, + matchedAllocation: this.matchedAllocation, + // Pre-waterfall outcomes never entered the allocation loop, so these lists + // are meaningless and should not be populated. + unmatchedAllocations: isPreWaterfall ? [] : this.unmatchedAllocations, + unevaluatedAllocations: isPreWaterfall ? [] : this.calculateUnevaluated(), + configFetchedAt: this.configFetchedAt, + environmentName: this.environmentName, + } + } + + private calculateUnevaluated(): AllocationOutcome[] { + const evaluatedCount = + this.unmatchedAllocations.length + (this.matchedAllocation ? 1 : 0) + // INVARIANT: recordUnmatched must be called exactly once for each allocation + // in this.allocations order before build() is called. calculateUnevaluated + // uses evaluatedCount as a slice index, which is only correct if the first + // evaluatedCount allocations in this.allocations are exactly those for which + // recordUnmatched/recordMatch was called, in order. The waterfall loop in + // evaluateForSubject satisfies this by calling recordUnmatched on every + // allocation it processes before continuing. Future callers of this builder + // must preserve this invariant. + return this.allocations.slice(evaluatedCount).map((allocation, i) => ({ + key: allocation.key, + orderPosition: evaluatedCount + i + 1, // 1-indexed + outcomeCode: AllocationOutcomeCode.UNEVALUATED, + rulesPresent: !!allocation.rules?.length, + })) + } +} +``` + +--- + +### Changes to `evaluate()` — Pre-waterfall Error Paths + +`evaluate()` currently returns `ResolutionDetails` directly for pre-waterfall errors. +It must be updated to return `DDFlagEvaluationDetails` and pass `config.createdAt` / +`config.environment.name` into the builder (or directly into `evaluateForSubject`). + +For pre-waterfall outcomes the builder is constructed with an empty `allocations` array +since the flag config is either unavailable or we never reached the point of looking up +the flag: + +```typescript +// evaluate.ts — pre-waterfall error paths now return DDFlagEvaluationDetails + +export function evaluate( + config: UniversalFlagConfigurationV1 | undefined, + type: T, + flagKey: string, + defaultValue: FlagTypeToValue, + context: EvaluationContext, + logger: Logger, +): DDFlagEvaluationDetails> { + + const configMeta = config + ? { configFetchedAt: config.createdAt, environmentName: config.environment.name } + : { configFetchedAt: null, environmentName: null } + + const noAllocBuilder = () => + new DDFlagEvaluationDetailsBuilder(flagKey, [], configMeta.configFetchedAt, configMeta.environmentName) + + if (!config) { + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.PROVIDER_NOT_READY, + 'Configuration is not loaded', null) + } + + const { targetingKey: subjectKey, ...remainingContext } = context + if (subjectKey == null) { + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING, + 'targetingKey is required but was not provided', null) + } + + const flag = config.flags[flagKey] + if (!flag) { + logger.debug('returning default value because flag is not found', { flagKey, subjectKey }) + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.FLAG_NOT_FOUND, + `Flag '${flagKey}' not found in configuration`, null) + } + + const subjectAttributes = { id: subjectKey, ...remainingContext } + try { + return evaluateForSubject( + flag, type, subjectKey, subjectAttributes, defaultValue, logger, + configMeta.configFetchedAt, configMeta.environmentName, + ) + } catch (error) { + // evaluateForSubject is an in-process pure function; exceptions here are programming + // errors, not expected failure modes. We catch to preserve the OpenFeature contract + // (evaluation must never throw), but the allocation trace from any mid-waterfall + // progress is lost — the catch has no access to the builder inside evaluateForSubject. + // The error message is threaded into outcomeDescription so it survives in the return + // value; callers must not rely on this for production logic. + logger.error('Error evaluating flag', { error }) + const description = error instanceof Error ? error.message : 'Unexpected error during evaluation' + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.ERROR, + description, flag.variationType) + } +} +``` + +--- + +### Changes to `evaluateForSubject` + +Signature change: accepts `configFetchedAt` and `environmentName` and returns +`DDFlagEvaluationDetails` instead of `ResolutionDetails`. Waterfall loop calls +`builder.recordUnmatched()` / `builder.recordMatch()` for each allocation: + +```typescript +// Key changes — full function shown for clarity + +export function evaluateForSubject( + flag: Flag, + type: T, + subjectKey: string, + subjectAttributes: EvaluationContext, + defaultValue: FlagTypeToValue, + logger: Logger, + configFetchedAt: string | null, + environmentName: string | null, +): DDFlagEvaluationDetails> { + + const builder = new DDFlagEvaluationDetailsBuilder( + flag.key, flag.allocations, configFetchedAt, environmentName, + ) + + if (!flag.enabled) { + return builder.build(defaultValue, FlagEvaluationOutcomeCode.DISABLED, + 'Flag is disabled', flag.variationType) + } + + if (!validateTypeMatch(type, flag.variationType)) { + return builder.build(defaultValue, FlagEvaluationOutcomeCode.TYPE_MISMATCH, + `Expected type '${type}' does not match flag variationType '${flag.variationType}'`, + flag.variationType) + } + // DISABLED and TYPE_MISMATCH are in PRE_WATERFALL_CODES, so build() suppresses + // unmatchedAllocations and unevaluatedAllocations for both outcomes. This is correct: + // recordUnmatched is never called for either path (the allocations loop never ran), + // so without PRE_WATERFALL_CODES the builder would incorrectly classify every allocation + // in the flag as UNEVALUATED. + + const now = new Date() + for (const [index, allocation] of flag.allocations.entries()) { + const position = index + 1 // 1-indexed + + // allocation.startAt / endAt are typed as Date in ufc-v1.ts but runtime values are + // ISO strings (the type is a known mismatch; wrapping with new Date() is required). + if (allocation.startAt && now < new Date(allocation.startAt as unknown as string)) { + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.BEFORE_START_TIME) + continue + } + + if (allocation.endAt && now >= new Date(allocation.endAt as unknown as string)) { + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.AFTER_END_TIME) + continue + } + + const ruleMatchIndex = findMatchingRuleIndex(allocation.rules, subjectAttributes, logger) + if (ruleMatchIndex === null) { + // rules present but none matched + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.RULES_MISMATCH) + continue + } + // ruleMatchIndex is a number (matched rule) or undefined (no rules, implicit match-all) + + const selectedSplit = selectSplitUsingSharding(allocation.splits, subjectKey, flag.key, logger) + if (!selectedSplit) { + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.TRAFFIC_MISS, ruleMatchIndex) + continue + } + + const variant = flag.variations[selectedSplit.variationKey] + if (!variant) { + // Corrupt config: split references a variationKey that doesn't exist. + // Log and continue to next allocation rather than short-circuiting. + logger.warn('Split references unknown variationKey', { + flagKey: flag.key, allocationKey: allocation.key, + variationKey: selectedSplit.variationKey, + }) + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.MISSING_VARIATION, ruleMatchIndex) + continue + } + + builder.recordMatch(allocation, position, variant.key, ruleMatchIndex) + return builder.build( + variant.value as FlagTypeToValue, + FlagEvaluationOutcomeCode.MATCH, + `Matched allocation '${allocation.key}'`, + flag.variationType, + ) + } + + return builder.build(defaultValue, FlagEvaluationOutcomeCode.DEFAULT, + 'No allocation matched; returning default value', flag.variationType) +} +``` + +--- + +### Changes to Rule Matching + +`containsMatchingRule` currently returns `boolean`. It is split into two functions: +one returning the matched rule index (for the trace), and a wrapper preserving the +existing boolean API for any other call sites: + +```typescript +/** + * Returns the 0-indexed position of the first rule that matched, + * undefined if no rules are defined (implicit match-all), or + * null if rules are present and none matched. + */ +export function findMatchingRuleIndex( + rules: Rule[] | undefined, + subjectAttributes: EvaluationContext, + logger: Logger, +): number | null | undefined { + if (!rules?.length) return undefined // no rules → implicit match-all + logger.debug('evaluating rules', { rules: JSON.stringify(rules), subjectAttributes }) + const idx = rules.findIndex((rule) => matchesRule(rule, subjectAttributes)) + return idx === -1 ? null : idx +} + +/** + * Thin wrapper for call sites that need only a boolean result. + * - undefined (no rules) → true (implicit match-all) + * - number (matched) → true + * - null (no match) → false + * + * Implementation note: the existing containsMatchingRule export in + * evaluateForSubject.ts must be deleted as part of this change. It is currently + * exported from that file and any call sites importing it from there must be + * updated to import from rules.ts instead. Leaving both in place risks silent + * divergence if either is changed independently. + */ +export function containsMatchingRule( + rules: Rule[] | undefined, + subjectAttributes: EvaluationContext, + logger: Logger, +): boolean { + return findMatchingRuleIndex(rules, subjectAttributes, logger) !== null +} +``` + +--- + +### OpenFeature Boundary Serialization + +The provider converts `DDFlagEvaluationDetails` to `ResolutionDetails`. The +`ddEvaluationTrace` key in `flagMetadata` is opt-in and defaults to off to avoid +JSON serialization cost on every evaluation in high-throughput contexts: + +```typescript +function toResolutionDetails( + details: DDFlagEvaluationDetails>, + requestedType: T, + includeTrace: boolean, +): ResolutionDetails> { + // For pre-waterfall errors, variationType is null (no flag config available). + // Fall back to the requested type T rather than hardcoding 'boolean', so downstream + // consumers of flagMetadata.variationType get an accurate signal about the flag type + // the caller was asking for. + const flagMetadata: PrecomputedFlagMetadata & { ddEvaluationTrace?: string } = { + allocationKey: details.allocationKey ?? '', + variationType: details.variationType + ? variantTypeToFlagValueType(details.variationType) + : requestedType, + doLog: details.doLog, + } + + if (includeTrace) { + flagMetadata.ddEvaluationTrace = JSON.stringify({ + flagKey: details.flagKey, + variationKey: details.variationKey, + allocationKey: details.allocationKey, + outcomeCode: details.outcomeCode, + outcomeDescription: details.outcomeDescription, // preserves error.message for ERROR outcomes + matchedAllocation: details.matchedAllocation, + unmatchedAllocations: details.unmatchedAllocations, + unevaluatedAllocations: details.unevaluatedAllocations, + environmentName: details.environmentName, + configFetchedAt: details.configFetchedAt, + }) + } + + return { + value: details.value as FlagTypeToValue, + reason: outcomeCodeToReason(details.outcomeCode), + variant: details.variationKey ?? undefined, + errorCode: outcomeCodeToErrorCode(details.outcomeCode), + flagMetadata, + } +} + +function outcomeCodeToReason(code: FlagEvaluationOutcomeCode): string { + switch (code) { + case FlagEvaluationOutcomeCode.MATCH: return StandardResolutionReasons.TARGETING_MATCH + case FlagEvaluationOutcomeCode.DISABLED: return StandardResolutionReasons.DISABLED + case FlagEvaluationOutcomeCode.DEFAULT: return StandardResolutionReasons.DEFAULT + default: return StandardResolutionReasons.ERROR + } +} + +function outcomeCodeToErrorCode(code: FlagEvaluationOutcomeCode): ErrorCode | undefined { + switch (code) { + case FlagEvaluationOutcomeCode.PROVIDER_NOT_READY: return ErrorCode.PROVIDER_NOT_READY + case FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING: return ErrorCode.TARGETING_KEY_MISSING + case FlagEvaluationOutcomeCode.FLAG_NOT_FOUND: return ErrorCode.FLAG_NOT_FOUND + case FlagEvaluationOutcomeCode.TYPE_MISMATCH: return ErrorCode.TYPE_MISMATCH + case FlagEvaluationOutcomeCode.ERROR: return ErrorCode.GENERAL + default: return undefined + } +} +``` + +--- + +## What Is Not Included (Intentionally) + +### Per-condition failure detail + +We expose `matchedRuleIndex` (which rule triggered the match) but not which +*condition* within a failing rule caused the mismatch. + +This is left out for two reasons: + +1. **Privacy** — condition evaluation includes the subject's attribute values. + Surfacing `{ attribute: 'email', operator: ONE_OF, value: [...], subjectValue: 'user@example.com' }` + at the telemetry layer requires a deliberate PII review before it can be included + in any external payload. +2. **Scope** — `matchedRuleIndex` is sufficient for the primary use cases (debugging + which branch of the waterfall fired). Per-condition tracing can be a follow-on. + +### Shard assignment detail + +The hash value and computed shard number for a `TRAFFIC_MISS` are not included. +They are deterministic from `(salt, subjectKey, totalShards)` and can be recalculated +offline. Including raw hash output adds noise without clear benefit. + +--- + +## Implementation Prerequisites + +### `targetingRuleKey` in `flagMetadata` + +The existing `FlagEvaluationAggregator` reads `details.flagMetadata?.targetingRuleKey` to +populate `FlagEvaluationEvent.targeting_rule` — one of the stated motivations for this RFC +("Observability / telemetry" in the Motivation section). However, `toResolutionDetails` +cannot populate `flagMetadata.targetingRuleKey` with a string key because the current +`Rule` type has no `key` field: + +```typescript +// ufc-v1.ts / rules.ts — current Rule type +export interface Rule { + conditions: Condition[] // no key field +} +``` + +This RFC captures `matchedRuleIndex` (integer position in `allocation.rules`) but cannot +derive a string key from it. Until the flag configuration format adds a stable `key` field +to `Rule`, `FlagEvaluationAggregationKey.targetingRuleKey` will remain unpopulated. + +Options: +1. **Use index-as-string as a temporary key** — set `targetingRuleKey` to + `String(matchedRuleIndex)` in `toResolutionDetails`. Functional but fragile: rule order + changes in the flag config break historical comparisons. +2. **Add `key` to the `Rule` type and config format** — the proper fix, but requires a + config format change and a coordinated backend deploy. This is a prerequisite for the + observability use case to be fully realized. + +This RFC does not resolve this gap. The trace (`ddEvaluationTrace`) includes +`matchedAllocation.matchedRuleIndex`, which is sufficient for ad-hoc debugging. The +telemetry aggregation path (`targeting_rule` in events) requires the config format change. + +--- + +## Alternatives Considered + +### Put everything in `flagMetadata` directly (no new internal type) + +Rejected: imposes serialization cost on all callers unconditionally, collapses the +type system to untyped strings inside the DD layer, and makes the trace unavailable +to non-OpenFeature consumers of the SDK. + +### Use `FlagEvaluationOutcomeCode` values that mirror `StandardResolutionReasons` directly + +The `FlagEvaluationOutcomeCode` enum could be replaced by reusing +`StandardResolutionReasons` plus an `ErrorCode` field, matching the shape of +`ResolutionDetails`. This would eliminate the `outcomeCodeToReason()` mapping step. + +Rejected because: `FlagEvaluationOutcomeCode` distinguishes cases that +`StandardResolutionReasons` collapses — specifically, `PROVIDER_NOT_READY`, +`TARGETING_KEY_MISSING`, `FLAG_NOT_FOUND`, and `TYPE_MISMATCH` all map to +`StandardResolutionReasons.ERROR` but are meaningfully different outcomes for +debugging and telemetry. Keeping a richer internal enum makes the DD layer +self-describing without requiring callers to cross-reference the `errorCode` field. + +### Extend `ResolutionDetails` with a non-standard field + +OpenFeature SDKs that wrap `ResolutionDetails` may strip unknown fields. Fragile at +the OF boundary and not forward-compatible with a future spec extension. + +### Place the builder in `packages/core` + +The builder pattern is currently only used by the node-server evaluator. If a +browser/client-side package is added that shares the same waterfall evaluation logic, +the builder should be promoted to `packages/core`. For now it belongs in +`packages/node-server` to avoid premature abstraction. + +### Replicate the EPPO builder exactly + +EPPO's builder takes the full `Allocation[]` at construction time and derives +`unevaluatedAllocations` by slicing, which is reused here. The main divergences: +- EPPO exposes the full matched `Rule` object; we expose only the index (privacy) +- EPPO has no equivalent of `MISSING_VARIATION` (not needed in their evaluator) +- Naming adjusted to DD conventions (`FAILING_RULE` → `RULES_MISMATCH`, + `TRAFFIC_EXPOSURE_MISS` → `TRAFFIC_MISS`) + +--- + +## Open Questions + +1. **Should `outcomeDescription` be a constrained format?** + Currently proposed as human-readable freeform. If it is to be indexed or alerted on, + a structured format or enum may be preferable. + +2. **Should `findMatchingRuleIndex` also record which conditions failed, behind a + verbose flag?** Useful for deep debugging but requires a PII review before inclusion + in any telemetry payload. Out of scope for this RFC. + +3. **`includeTrace` as a provider config flag vs. always-on?** Serialization overhead + is small per evaluation but accumulates at high throughput. Defaulting off is + conservative; if the trace is always useful, remove the flag and always serialize. + +4. **`MISSING_VARIATION` continue behavior — should it short-circuit instead?** + The RFC proposes logging a warning and continuing the waterfall when a split + selects a `variationKey` absent from `flag.variations`. The risk: the subject has + already passed targeting rules and traffic sharding for that allocation. Continuing + means they may match a later allocation and receive a `TARGETING_MATCH` result that + appears legitimate but was only reached because of a corrupt earlier allocation. + + Two options: + - **Hard fail with `ERROR`**: return immediately from `evaluateForSubject` with + `FlagEvaluationOutcomeCode.ERROR` and `defaultValue`. Config corruption surfaces + immediately; no misleading downstream match is possible. + - **Continue with new top-level code**: keep the waterfall running but if the final + outcome is reached through a `MISSING_VARIATION` allocation, mark the top-level + `outcomeCode` as a new value (e.g. `CORRUPT_CONFIG`) rather than `MATCH` or `DEFAULT`. + + The current RFC implements "continue" because it mirrors how the existing evaluator + behaves (silent fall-through). The team should decide whether surfacing corrupt config + as a hard error is worth the behavior change. diff --git a/packages/node-server/src/configuration/evaluateForSubject.ts b/packages/node-server/src/configuration/evaluateForSubject.ts index b4d4ccb0..e6062e26 100644 --- a/packages/node-server/src/configuration/evaluateForSubject.ts +++ b/packages/node-server/src/configuration/evaluateForSubject.ts @@ -1,16 +1,19 @@ -import type { FlagTypeToValue, PrecomputedFlagMetadata } from '@datadog/flagging-core' +import type { FlagTypeToValue } from '@datadog/flagging-core' import { - ErrorCode, type EvaluationContext, type FlagValueType, type Logger, - type ResolutionDetails, - StandardResolutionReasons, TargetingKeyMissingError, } from '@openfeature/server-sdk' -import { matchesRule, type Rule } from '../rules/rules' +import { findMatchingRuleIndex } from '../rules/rules' import { matchesShard } from '../shards/matchesShard' import { type Flag, type Split, type VariantType, variantTypeToFlagValueType } from './ufc-v1' +import { + AllocationOutcomeCode, + DDFlagEvaluationDetailsBuilder, + type DDFlagEvaluationDetails, + FlagEvaluationOutcomeCode, +} from './flagEvaluationDetails' export function evaluateForSubject( flag: Flag, @@ -18,17 +21,21 @@ export function evaluateForSubject( subjectKey: string | null | undefined, subjectAttributes: EvaluationContext, defaultValue: FlagTypeToValue, - logger: Logger -): ResolutionDetails> { + logger: Logger, + configFetchedAt: string | null, + environmentName: string | null, +): DDFlagEvaluationDetails> { + const builder = new DDFlagEvaluationDetailsBuilder( + flag.key, flag.allocations, configFetchedAt, environmentName, + ) + if (!flag.enabled) { logger.debug(`returning default assignment because flag is disabled`, { flagKey: flag.key, subjectKey, }) - return { - value: defaultValue, - reason: StandardResolutionReasons.DISABLED, - } + return builder.build(defaultValue, FlagEvaluationOutcomeCode.DISABLED, + 'Flag is disabled', flag.variationType) } const isValid = validateTypeMatch(type, flag.variationType) @@ -39,111 +46,108 @@ export function evaluateForSubject( expectedType: type, variantType: flag.variationType, }) - return { - value: defaultValue, - reason: StandardResolutionReasons.ERROR, - errorCode: ErrorCode.TYPE_MISMATCH, - } + return builder.build( + defaultValue, + FlagEvaluationOutcomeCode.TYPE_MISMATCH, + `Expected type '${type}' does not match flag variationType '${flag.variationType}'`, + flag.variationType, + ) } const now = new Date() - for (const allocation of flag.allocations) { - if (allocation.startAt && now < new Date(allocation.startAt)) { + for (const [index, allocation] of flag.allocations.entries()) { + const position = index + 1 // 1-indexed + + if (allocation.startAt && now < new Date(allocation.startAt as unknown as string)) { logger.debug(`allocation before start date`, { flagKey: flag.key, subjectKey, allocationKey: allocation.key, startAt: allocation.startAt, }) + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.BEFORE_START_TIME) continue } - if (allocation.endAt && now >= new Date(allocation.endAt)) { + if (allocation.endAt && now >= new Date(allocation.endAt as unknown as string)) { logger.debug(`allocation after end date`, { flagKey: flag.key, subjectKey, allocationKey: allocation.key, endAt: allocation.endAt, }) + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.AFTER_END_TIME) continue } - const matched = containsMatchingRule(allocation.rules, subjectAttributes, logger) - if (!matched) { + const ruleMatchIndex = findMatchingRuleIndex(allocation.rules, subjectAttributes) + if (ruleMatchIndex === null) { + // rules present but none matched + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.RULES_MISMATCH) continue } + // ruleMatchIndex is a number (matched rule) or undefined (no rules, implicit match-all) const selectedSplit = selectSplitUsingSharding(allocation.splits, subjectKey, flag.key, logger) - if (selectedSplit) { - const variant = flag.variations[selectedSplit.variationKey] - if (variant) { - logger.debug(`evaluated a flag`, { - flagKey: flag.key, - subjectKey, - assignment: variant.value, - }) - - return { - value: variant.value as FlagTypeToValue, - reason: StandardResolutionReasons.TARGETING_MATCH, - variant: variant.key, - flagMetadata: { - allocationKey: allocation.key, - variationType: variantTypeToFlagValueType(flag.variationType), - doLog: !!allocation.doLog, - } as PrecomputedFlagMetadata, - } - } - } else { + if (!selectedSplit) { logger.debug(`no matching split found for subject`, { flagKey: flag.key, subjectKey, allocationKey: allocation.key, }) + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.TRAFFIC_MISS, ruleMatchIndex) + continue } + + const variant = flag.variations[selectedSplit.variationKey] + if (!variant) { + // Intentional continue: we mirror the pre-RFC behavior of falling through on a + // corrupt allocation rather than hard-failing. The subject has already passed rules + // and sharding for this allocation, so a later allocation may produce a different + // MATCH — MISSING_VARIATION in unmatchedAllocations signals that corruption occurred. + // See RFC open question #4 for the trade-offs of hard-failing vs. continuing. + logger.warn('Split references unknown variationKey', { + flagKey: flag.key, allocationKey: allocation.key, + variationKey: selectedSplit.variationKey, + }) + builder.recordUnmatched(allocation, position, AllocationOutcomeCode.MISSING_VARIATION, ruleMatchIndex) + continue + } + + logger.debug(`evaluated a flag`, { + flagKey: flag.key, + subjectKey, + assignment: variant.value, + }) + + // variant.key is the variation's own .key field, which should equal selectedSplit.variationKey + // (the key used to look it up in flag.variations). In well-formed config these are identical. + // Using variant.key mirrors the pre-RFC behavior (ResolutionDetails.variant = variant.key). + builder.recordMatch(allocation, position, variant.key, ruleMatchIndex) + return builder.build( + variant.value as FlagTypeToValue, + FlagEvaluationOutcomeCode.MATCH, + `Matched allocation '${allocation.key}'`, + flag.variationType, + ) } - // This shouldn't happen since a default allocation is generated by the server logger.debug(`returning default assignment because no allocation matched`, { flagKey: flag.key, subjectKey, }) - return { - value: defaultValue, - reason: StandardResolutionReasons.DEFAULT, - } + return builder.build(defaultValue, FlagEvaluationOutcomeCode.DEFAULT, + 'No allocation matched; returning default value', flag.variationType) } function validateTypeMatch(expectedType: FlagValueType, variantType: VariantType): boolean { - if (expectedType === 'boolean') { - return variantType === 'BOOLEAN' - } - if (expectedType === 'string') { - return variantType === 'STRING' - } - if (expectedType === 'number') { - return variantType === 'INTEGER' || variantType === 'NUMERIC' - } - if (expectedType === 'object') { - return variantType === 'JSON' - } - throw new Error(`Invalid expected type: ${expectedType}`) -} - -export function containsMatchingRule( - rules: Rule[] | undefined, - subjectAttributes: EvaluationContext, - logger: Logger -): boolean { - if (!rules?.length) { - return true - } - logger.debug(`evaluating rules`, { - rules: JSON.stringify(rules), - subjectAttributes, - }) - return rules.some((rule) => matchesRule(rule, subjectAttributes)) + // variantTypeToFlagValueType encodes the full VariantType→FlagValueType mapping; reusing + // it here keeps the two in sync if new variant types are added later. + // Unknown variantType values throw from variantTypeToFlagValueType — we let that propagate + // to evaluate()'s catch block so it is logged at error level as FlagEvaluationOutcomeCode.ERROR, + // rather than silently surfacing as a TYPE_MISMATCH that implies the caller used the wrong type. + return variantTypeToFlagValueType(variantType) === expectedType } function selectSplitUsingSharding( @@ -164,6 +168,10 @@ function selectSplitUsingSharding( shards: split.shards, }) + // Note: when split.shards is empty, Array.every() returns true vacuously — the split + // matches without evaluating the subjectKey null guard below. This is pre-existing + // behavior: an empty shards array acts as a catch-all. A null subjectKey will not + // throw TargetingKeyMissingError in that case. const matches = split.shards.every((shard) => { if (subjectKey == null) { throw new TargetingKeyMissingError() diff --git a/packages/node-server/src/configuration/evaluation.ts b/packages/node-server/src/configuration/evaluation.ts index be6ece51..3de484fa 100644 --- a/packages/node-server/src/configuration/evaluation.ts +++ b/packages/node-server/src/configuration/evaluation.ts @@ -1,15 +1,17 @@ import type { FlagTypeToValue } from '@datadog/flagging-core' import { - ErrorCode, type EvaluationContext, type FlagValueType, type Logger, - type ResolutionDetails, - StandardResolutionReasons, TargetingKeyMissingError, } from '@openfeature/server-sdk' import { evaluateForSubject } from './evaluateForSubject' import type { UniversalFlagConfigurationV1 } from './ufc-v1' +import { + DDFlagEvaluationDetailsBuilder, + type DDFlagEvaluationDetails, + FlagEvaluationOutcomeCode, +} from './flagEvaluationDetails' export function evaluate( config: UniversalFlagConfigurationV1 | undefined, @@ -17,49 +19,57 @@ export function evaluate( flagKey: string, defaultValue: FlagTypeToValue, context: EvaluationContext, - logger: Logger -): ResolutionDetails> { + logger: Logger, +): DDFlagEvaluationDetails> { + const configMeta = config + ? { configFetchedAt: config.createdAt, environmentName: config.environment.name } + : { configFetchedAt: null, environmentName: null } + + const noAllocBuilder = () => + new DDFlagEvaluationDetailsBuilder(flagKey, [], configMeta.configFetchedAt, configMeta.environmentName) + if (!config) { - return { - value: defaultValue, - reason: 'ERROR', - errorCode: ErrorCode.PROVIDER_NOT_READY, - } + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.PROVIDER_NOT_READY, + 'Configuration is not loaded', null) } const { targetingKey: subjectKey, ...remainingContext } = context + const flag = config.flags[flagKey] + if (!flag) { + logger.debug('returning default value because flag is not found', { flagKey, subjectKey }) + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.FLAG_NOT_FOUND, + `Flag '${flagKey}' not found in configuration`, null) + } + // Include the subjectKey as an "id" attribute for rule matching only when present const subjectAttributes = { ...(subjectKey != null ? { id: subjectKey } : {}), ...remainingContext, } - const flag = config.flags[flagKey] - if (!flag) { - logger.debug('returning default value because flag is not found', { flagKey, subjectKey }) - return { - value: defaultValue, - reason: StandardResolutionReasons.ERROR, - errorCode: ErrorCode.FLAG_NOT_FOUND, - } - } try { - const resultWithDetails = evaluateForSubject(flag, type, subjectKey, subjectAttributes, defaultValue, logger) - return resultWithDetails + return evaluateForSubject( + flag, type, subjectKey, subjectAttributes, defaultValue, logger, + configMeta.configFetchedAt, configMeta.environmentName, + ) } catch (error) { if (error instanceof TargetingKeyMissingError) { - return { - value: defaultValue, - reason: StandardResolutionReasons.ERROR, - errorCode: ErrorCode.TARGETING_KEY_MISSING, - } + // TargetingKeyMissingError is thrown inside selectSplitUsingSharding when subjectKey is + // null and sharding is required. Any allocation-level progress made before the throw + // (recorded inside the builder in evaluateForSubject) is discarded here — this catch + // has no access to that builder. TARGETING_KEY_MISSING is in PRE_WATERFALL_CODES, so + // the empty allocation lists in the result are correct per the contract. + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING, + 'targetingKey is required but was not provided', flag.variationType) } + // evaluateForSubject is an in-process pure function; exceptions here are programming + // errors, not expected failure modes. We catch to preserve the OpenFeature contract + // (evaluation must never throw). The allocation trace from any mid-waterfall progress + // is lost — the catch has no access to the builder inside evaluateForSubject. logger.error('Error evaluating flag', { error }) - return { - value: defaultValue, - reason: StandardResolutionReasons.ERROR, - errorCode: ErrorCode.GENERAL, - } + const description = error instanceof Error ? error.message : String(error) + return noAllocBuilder().build(defaultValue, FlagEvaluationOutcomeCode.ERROR, + description, flag.variationType) } } diff --git a/packages/node-server/src/configuration/flagEvaluationDetails.ts b/packages/node-server/src/configuration/flagEvaluationDetails.ts new file mode 100644 index 00000000..1eb16db8 --- /dev/null +++ b/packages/node-server/src/configuration/flagEvaluationDetails.ts @@ -0,0 +1,364 @@ +import type { FlagTypeToValue, PrecomputedFlagMetadata } from '@datadog/flagging-core' +import { + ErrorCode, + type FlagValueType, + type ResolutionDetails, + StandardResolutionReasons, +} from '@openfeature/server-sdk' +import type { Allocation } from './ufc-v1' +import { type VariantType, variantTypeToFlagValueType } from './ufc-v1' + +/** + * Why an individual allocation was skipped or selected during waterfall evaluation. + */ +export enum AllocationOutcomeCode { + /** This allocation matched and its variant was returned. */ + MATCH = 'MATCH', + /** Skipped: current time is before allocation.startAt. */ + BEFORE_START_TIME = 'BEFORE_START_TIME', + /** Skipped: current time is at or after allocation.endAt. */ + AFTER_END_TIME = 'AFTER_END_TIME', + /** Skipped: allocation has targeting rules and none of them matched. */ + RULES_MISMATCH = 'RULES_MISMATCH', + /** Skipped: targeting rules matched but the subject fell outside all split ranges. */ + TRAFFIC_MISS = 'TRAFFIC_MISS', + /** + * Skipped: split was selected but the variationKey it references does not exist in + * flag.variations. This indicates a corrupt or inconsistent flag configuration. + * The waterfall continues to the next allocation. + */ + MISSING_VARIATION = 'MISSING_VARIATION', + /** Never evaluated: the waterfall terminated (matched or errored) before reaching this allocation. */ + UNEVALUATED = 'UNEVALUATED', +} + +/** + * Trace record for a single allocation in the waterfall. + */ +export interface AllocationOutcome { + /** Matches Allocation.key from the flag configuration. */ + key: string + /** + * 1-indexed position of this allocation in flag.allocations. + * Matches the ordering shown in the flag management UI. + */ + orderPosition: number + outcomeCode: AllocationOutcomeCode + /** + * Whether this allocation defined targeting rules. + * Use this to distinguish "no rules (implicit match-all)" from + * "rules present but matchedRuleIndex is not set" — a condition that + * should not occur but is otherwise ambiguous. + */ + rulesPresent: boolean + /** + * 0-indexed position of the rule that matched within allocation.rules. + * Defined only when rulesPresent is true AND a rule matched + * (i.e. outcomeCode is MATCH, TRAFFIC_MISS, or MISSING_VARIATION). + * Undefined when rulesPresent is false (implicit match-all) or when + * outcomeCode is RULES_MISMATCH. + */ + matchedRuleIndex?: number +} + +/** + * Top-level outcome of a flag evaluation, covering both pre-waterfall + * and waterfall outcomes. + */ +export enum FlagEvaluationOutcomeCode { + /** A matching allocation was found and a variant was returned. */ + MATCH = 'MATCH', + /** Flag is disabled; default value was returned. */ + DISABLED = 'DISABLED', + /** Config is not loaded; default value was returned. */ + PROVIDER_NOT_READY = 'PROVIDER_NOT_READY', + /** targetingKey was not present in evaluation context. */ + TARGETING_KEY_MISSING = 'TARGETING_KEY_MISSING', + /** Flag key not present in configuration; default value was returned. */ + FLAG_NOT_FOUND = 'FLAG_NOT_FOUND', + /** Expected type does not match the flag's variationType; default value was returned. */ + TYPE_MISMATCH = 'TYPE_MISMATCH', + /** All allocations were evaluated and none matched; default value was returned. */ + DEFAULT = 'DEFAULT', + /** An unexpected error occurred during evaluation; default value was returned. */ + ERROR = 'ERROR', +} + +/** + * Detailed result of a flag evaluation. This is the DD-internal type. + * It is NOT directly exposed at the OpenFeature boundary — use toResolutionDetails() + * to convert it there. + * + * unmatchedAllocations and unevaluatedAllocations are empty for all outcome codes + * in PRE_WATERFALL_CODES (PROVIDER_NOT_READY, TARGETING_KEY_MISSING, FLAG_NOT_FOUND, + * DISABLED, TYPE_MISMATCH, ERROR). They are only populated for MATCH and DEFAULT, + * where the waterfall loop ran and allocation-level outcomes were recorded. + */ +export interface DDFlagEvaluationDetails { + flagKey: string + /** The value returned to the caller (may be the supplied default). */ + value: T + /** The key of the selected variation. Null when no allocation matched. */ + variationKey: string | null + /** The key of the matched allocation. Null when no allocation matched. */ + allocationKey: string | null + /** The variationType declared in the flag configuration. Null for pre-waterfall errors. */ + variationType: VariantType | null + /** + * Whether the matched allocation's doLog field was true. + * False when no allocation matched or for pre-waterfall errors. + */ + doLog: boolean + + outcomeCode: FlagEvaluationOutcomeCode + /** Human-readable description of the outcome, suitable for debug logging. */ + outcomeDescription: string + + /** + * The allocation that was selected, or null if none matched. + * Non-null only when outcomeCode is MATCH. + */ + matchedAllocation: AllocationOutcome | null + + /** + * Allocations evaluated but not matched, in evaluation order. + * Empty for pre-waterfall error outcomes (PROVIDER_NOT_READY, TARGETING_KEY_MISSING, + * FLAG_NOT_FOUND, ERROR). + */ + unmatchedAllocations: AllocationOutcome[] + + /** + * Allocations never evaluated because the waterfall terminated first. + * Empty for pre-waterfall error outcomes. + */ + unevaluatedAllocations: AllocationOutcome[] + + /** ISO 8601 timestamp from config.createdAt. Null for pre-waterfall errors. */ + configFetchedAt: string | null + /** Environment name from config.environment.name. Null for pre-waterfall errors. */ + environmentName: string | null +} + +/** + * Outcome codes where the waterfall never ran; unevaluated/unmatched lists are + * always empty for these outcomes. + * + * DISABLED and TYPE_MISMATCH are included even though evaluateForSubject has access + * to flag.allocations at that point. The builder is constructed before the enabled/type + * checks but recordUnmatched is never called for them, so evaluatedCount === 0 and + * calculateUnevaluated() would return all allocations as UNEVALUATED — which is + * semantically wrong (the allocations were never candidates for evaluation, the flag + * itself was gated). Including them here suppresses that misleading output. + */ +const PRE_WATERFALL_CODES = new Set([ + FlagEvaluationOutcomeCode.PROVIDER_NOT_READY, + FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING, + FlagEvaluationOutcomeCode.FLAG_NOT_FOUND, + FlagEvaluationOutcomeCode.DISABLED, + FlagEvaluationOutcomeCode.TYPE_MISMATCH, + FlagEvaluationOutcomeCode.ERROR, +]) + +export class DDFlagEvaluationDetailsBuilder { + private variationKey: string | null = null + private allocationKey: string | null = null + private doLog = false + private matchedAllocation: AllocationOutcome | null = null + private readonly unmatchedAllocations: AllocationOutcome[] = [] + + constructor( + private readonly flagKey: string, + private readonly allocations: Allocation[], + private readonly configFetchedAt: string | null, + private readonly environmentName: string | null, + ) {} + + recordUnmatched( + allocation: Allocation, + position: number, + outcomeCode: AllocationOutcomeCode, + matchedRuleIndex?: number, + ): this { + this.unmatchedAllocations.push({ + key: allocation.key, + orderPosition: position, + outcomeCode, + rulesPresent: !!allocation.rules?.length, + matchedRuleIndex, + }) + return this + } + + recordMatch( + allocation: Allocation, + position: number, + variationKey: string, + matchedRuleIndex?: number, + ): this { + if (this.matchedAllocation !== null) { + throw new Error( + `recordMatch called twice on flag '${this.flagKey}': ` + + `first match was allocation '${this.matchedAllocation.key}', ` + + `second was '${allocation.key}'` + ) + } + this.variationKey = variationKey + this.allocationKey = allocation.key + this.doLog = !!allocation.doLog + this.matchedAllocation = { + key: allocation.key, + orderPosition: position, + outcomeCode: AllocationOutcomeCode.MATCH, + rulesPresent: !!allocation.rules?.length, + matchedRuleIndex, + } + return this + } + + /** + * Produce the final DDFlagEvaluationDetails. + * + * Each builder instance is intended for single use — `build()` should be called exactly once + * after all `recordUnmatched`/`recordMatch` calls are complete. The method is generic on the + * value type (`T`) rather than the class because the value type is not known at construction + * time. As a consequence TypeScript does not prevent calling `build` twice on the same instance + * with different `T` values; callers must not do this. + */ + build( + value: T, + outcomeCode: FlagEvaluationOutcomeCode, + outcomeDescription: string, + variationType: VariantType | null, + ): DDFlagEvaluationDetails { + const isPreWaterfall = PRE_WATERFALL_CODES.has(outcomeCode) + return { + flagKey: this.flagKey, + value, + variationKey: this.variationKey, + allocationKey: this.allocationKey, + variationType, + doLog: this.doLog, + outcomeCode, + outcomeDescription, + matchedAllocation: this.matchedAllocation, + unmatchedAllocations: isPreWaterfall ? [] : this.unmatchedAllocations, + unevaluatedAllocations: isPreWaterfall ? [] : this.calculateUnevaluated(), + configFetchedAt: this.configFetchedAt, + environmentName: this.environmentName, + } + } + + private calculateUnevaluated(): AllocationOutcome[] { + const evaluatedCount = + this.unmatchedAllocations.length + (this.matchedAllocation ? 1 : 0) + return this.allocations.slice(evaluatedCount).map((allocation, i) => ({ + key: allocation.key, + orderPosition: evaluatedCount + i + 1, // 1-indexed + outcomeCode: AllocationOutcomeCode.UNEVALUATED, + rulesPresent: !!allocation.rules?.length, + })) + } +} + +function outcomeCodeToReason(code: FlagEvaluationOutcomeCode): string { + switch (code) { + case FlagEvaluationOutcomeCode.MATCH: + return StandardResolutionReasons.TARGETING_MATCH + case FlagEvaluationOutcomeCode.DISABLED: + return StandardResolutionReasons.DISABLED + case FlagEvaluationOutcomeCode.DEFAULT: + return StandardResolutionReasons.DEFAULT + case FlagEvaluationOutcomeCode.PROVIDER_NOT_READY: + case FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING: + case FlagEvaluationOutcomeCode.FLAG_NOT_FOUND: + case FlagEvaluationOutcomeCode.TYPE_MISMATCH: + case FlagEvaluationOutcomeCode.ERROR: + return StandardResolutionReasons.ERROR + default: { + // Exhaustiveness guard: TypeScript flags this if a new enum member is added without + // updating this switch. The default is unreachable at runtime because FlagEvaluationOutcomeCode + // is a closed string enum defined in this package — no external code produces new values. + const _exhaustive: never = code + return StandardResolutionReasons.ERROR + } + } +} + +function outcomeCodeToErrorCode(code: FlagEvaluationOutcomeCode): ErrorCode | undefined { + switch (code) { + case FlagEvaluationOutcomeCode.PROVIDER_NOT_READY: + return ErrorCode.PROVIDER_NOT_READY + case FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING: + return ErrorCode.TARGETING_KEY_MISSING + case FlagEvaluationOutcomeCode.FLAG_NOT_FOUND: + return ErrorCode.FLAG_NOT_FOUND + case FlagEvaluationOutcomeCode.TYPE_MISMATCH: + return ErrorCode.TYPE_MISMATCH + case FlagEvaluationOutcomeCode.ERROR: + return ErrorCode.GENERAL + case FlagEvaluationOutcomeCode.MATCH: + case FlagEvaluationOutcomeCode.DISABLED: + case FlagEvaluationOutcomeCode.DEFAULT: + return undefined + default: { + // Exhaustiveness guard: TypeScript flags this if a new enum member is added without + // updating this switch. The default is unreachable at runtime (closed enum). + const _exhaustive: never = code + return undefined + } + } +} + +export function toResolutionDetails( + details: DDFlagEvaluationDetails>, + requestedType: T, + includeTrace: boolean, +): ResolutionDetails> { + const flagMetadata: PrecomputedFlagMetadata & { ddEvaluationTrace?: string } = { + // PrecomputedFlagMetadata.allocationKey is typed as string (non-nullable) upstream. + // For pre-waterfall outcomes (no allocation was ever matched), this produces '' rather + // than null. Consumers should treat '' as "no allocation matched" for these cases. + allocationKey: details.allocationKey ?? '', + // variantTypeToFlagValueType throws for unknown VariantType values (e.g., if a new type + // is deployed server-side before this SDK is updated). Fall back to requestedType rather + // than throwing post-evaluation, which would violate the OpenFeature "never throw" contract. + variationType: (() => { + if (!details.variationType) return requestedType + try { + return variantTypeToFlagValueType(details.variationType) + } catch { + return requestedType + } + })(), + doLog: details.doLog, + } + + if (includeTrace) { + // JSON.stringify can throw on circular references or non-serializable values (e.g. BigInt). + // Guard to preserve the "evaluation must never throw" contract at the OpenFeature boundary. + try { + flagMetadata.ddEvaluationTrace = JSON.stringify({ + flagKey: details.flagKey, + variationKey: details.variationKey, + allocationKey: details.allocationKey, + outcomeCode: details.outcomeCode, + outcomeDescription: details.outcomeDescription, + matchedAllocation: details.matchedAllocation, + unmatchedAllocations: details.unmatchedAllocations, + unevaluatedAllocations: details.unevaluatedAllocations, + environmentName: details.environmentName, + configFetchedAt: details.configFetchedAt, + }) + } catch { + flagMetadata.ddEvaluationTrace = '[trace serialization failed]' + } + } + + return { + value: details.value, + reason: outcomeCodeToReason(details.outcomeCode), + variant: details.variationKey ?? undefined, + errorCode: outcomeCodeToErrorCode(details.outcomeCode), + flagMetadata, + } +} diff --git a/packages/node-server/src/provider.ts b/packages/node-server/src/provider.ts index fda23a62..062b0a3c 100644 --- a/packages/node-server/src/provider.ts +++ b/packages/node-server/src/provider.ts @@ -20,6 +20,7 @@ import type { } from '@openfeature/server-sdk' import { OpenFeatureEventEmitter, ProviderEvents } from '@openfeature/server-sdk' import { evaluate } from './configuration/evaluation' +import { toResolutionDetails } from './configuration/flagEvaluationDetails' import type { UniversalFlagConfigurationV1 } from './configuration/ufc-v1' import { InitializationController } from './initialization-controller' @@ -39,6 +40,12 @@ export interface DatadogNodeServerProviderOptions { * @default DEFAULT_INITIALIZATION_TIMEOUT_MS (30000ms / 30 seconds) */ initializationTimeoutMs?: number + /** + * When true, serializes the full allocation waterfall trace into + * flagMetadata.ddEvaluationTrace on every evaluation. + * Defaults to false to avoid JSON serialization overhead in high-throughput contexts. + */ + includeEvaluationTrace?: boolean } export class DatadogNodeServerProvider implements Provider { @@ -75,6 +82,10 @@ export class DatadogNodeServerProvider implements Provider { const hadConfiguration = !!this.configuration if (hadConfiguration && this.configuration !== configuration) { + // Note: ConfigurationChanged is emitted before this.configuration is updated. Any + // synchronous handler that calls getConfiguration() or triggers evaluations will still + // see the previous configuration. Fixing this ordering is tracked separately and is + // pre-existing behavior not introduced by this change. this.events.emit(ProviderEvents.ConfigurationChanged) const newCreatedAt = configuration?.createdAt if (prevCreatedAt !== newCreatedAt) { @@ -137,7 +148,8 @@ export class DatadogNodeServerProvider implements Provider { context: EvaluationContext, _logger: Logger ): Promise> { - const resolutionDetails = evaluate(this.configuration, 'boolean', flagKey, defaultValue, context, _logger) + const details = evaluate(this.configuration, 'boolean', flagKey, defaultValue, context, _logger) + const resolutionDetails = toResolutionDetails(details, 'boolean', !!this.options.includeEvaluationTrace) this.handleExposure(flagKey, context, resolutionDetails) return resolutionDetails } @@ -148,7 +160,8 @@ export class DatadogNodeServerProvider implements Provider { context: EvaluationContext, _logger: Logger ): Promise> { - const resolutionDetails = evaluate(this.configuration, 'string', flagKey, defaultValue, context, _logger) + const details = evaluate(this.configuration, 'string', flagKey, defaultValue, context, _logger) + const resolutionDetails = toResolutionDetails(details, 'string', !!this.options.includeEvaluationTrace) this.handleExposure(flagKey, context, resolutionDetails) return resolutionDetails } @@ -159,7 +172,8 @@ export class DatadogNodeServerProvider implements Provider { context: EvaluationContext, _logger: Logger ): Promise> { - const resolutionDetails = evaluate(this.configuration, 'number', flagKey, defaultValue, context, _logger) + const details = evaluate(this.configuration, 'number', flagKey, defaultValue, context, _logger) + const resolutionDetails = toResolutionDetails(details, 'number', !!this.options.includeEvaluationTrace) this.handleExposure(flagKey, context, resolutionDetails) return resolutionDetails } @@ -176,14 +190,8 @@ export class DatadogNodeServerProvider implements Provider { // type-sound way because there's no runtime information passed to // learn what type the user expects. So it's up to the user to // make sure they pass the appropriate type. - const resolutionDetails = evaluate( - this.configuration, - 'object', - flagKey, - defaultValue, - context, - _logger - ) as ResolutionDetails + const details = evaluate(this.configuration, 'object', flagKey, defaultValue, context, _logger) + const resolutionDetails = toResolutionDetails(details, 'object', !!this.options.includeEvaluationTrace) as ResolutionDetails this.handleExposure(flagKey, context, resolutionDetails) return resolutionDetails } diff --git a/packages/node-server/src/rules/rules.ts b/packages/node-server/src/rules/rules.ts index 7feee103..7c9f4159 100644 --- a/packages/node-server/src/rules/rules.ts +++ b/packages/node-server/src/rules/rules.ts @@ -70,6 +70,20 @@ export function matchesRule(rule: Rule, subjectAttributes: EvaluationContext): b return !conditionEvaluations.includes(false) } +/** + * Returns the 0-indexed position of the first rule that matched, + * undefined if no rules are defined (implicit match-all), or + * null if rules are present and none matched. + */ +export function findMatchingRuleIndex( + rules: Rule[] | undefined, + subjectAttributes: EvaluationContext, +): number | null | undefined { + if (!rules?.length) return undefined // no rules → implicit match-all + const idx = rules.findIndex((rule) => matchesRule(rule, subjectAttributes)) + return idx === -1 ? null : idx +} + function evaluateRuleConditions(subjectAttributes: EvaluationContext, conditions: Condition[]): boolean[] { return conditions.map((condition) => evaluateCondition(subjectAttributes, condition)) } @@ -83,6 +97,10 @@ function evaluateCondition(subjectAttributes: EvaluationContext, condition: Cond return value !== null && value !== undefined } + // When value is null/undefined, all non-IS_NULL operators return false below. + // This means NOT_ONE_OF with a missing attribute evaluates to false (subject excluded), + // even though "not in list" is semantically true for an absent value. This is pre-existing + // behavior. Use IS_NULL to explicitly gate on attribute presence when needed. if (value !== null && value !== undefined) { switch (condition.operator) { case OperatorType.GTE: diff --git a/packages/node-server/test/demo.spec.ts b/packages/node-server/test/demo.spec.ts new file mode 100644 index 00000000..47f1c59a --- /dev/null +++ b/packages/node-server/test/demo.spec.ts @@ -0,0 +1,88 @@ +/** + * Demo of the DDFlagEvaluationDetails waterfall trace via the OpenFeature client. + * Not committed — run with: + * npx jest --testPathPatterns=demo --verbose --silent=false + */ +import type { Channel } from 'node:diagnostics_channel' +import type { ExposureEvent } from '@datadog/flagging-core' +import { OpenFeature } from '@openfeature/server-sdk' +import type { UniversalFlagConfigurationV1 } from '../src/configuration/ufc-v1' +import { DatadogNodeServerProvider } from '../src/provider' + +const exposureChannel = { + hasSubscribers: false, + publish: () => {}, + subscribe: () => {}, +} as unknown as Channel + +const PAST = new Date(Date.now() - 86400_000).toISOString() +const FUTURE = new Date(Date.now() + 86400_000).toISOString() + +const config: UniversalFlagConfigurationV1 = { + createdAt: '2026-04-15T12:00:00Z', + format: 'universal-flag-configuration', + environment: { name: 'staging' }, + flags: { + 'checkout-v2': { + key: 'checkout-v2', + enabled: true, + variationType: 'BOOLEAN', + variations: { + on: { key: 'on', value: true }, + off: { key: 'off', value: false }, + }, + allocations: [ + // 1. expired — AFTER_END_TIME + { key: 'old-rollout', endAt: PAST as unknown as Date, splits: [{ variationKey: 'on', shards: [] }] }, + // 2. future — BEFORE_START_TIME + { key: 'future-rollout', startAt: FUTURE as unknown as Date, splits: [{ variationKey: 'on', shards: [] }] }, + // 3. rules: CA/AU (index 0), US (index 1) + { + key: 'us-beta', + rules: [ + { conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['CA', 'AU'] }] }, + { conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['US'] }] }, + ], + splits: [{ variationKey: 'on', shards: [] }], + }, + // 4. default fallback + { key: 'default-off', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }, + }, +} + +async function show(label: string, context: Record, flagKey = 'checkout-v2') { + const provider = new DatadogNodeServerProvider({ + exposureChannel, + includeEvaluationTrace: true, + }) + provider.setConfiguration(config) + OpenFeature.setProvider(provider) + const client = OpenFeature.getClient() + + const details = await client.getBooleanDetails(flagKey, false, context as never) + const detailsPretty = { + ...details, + flagMetadata: { + ...details.flagMetadata, + ddEvaluationTrace: details.flagMetadata?.ddEvaluationTrace + ? JSON.parse(details.flagMetadata.ddEvaluationTrace as string) + : undefined, + }, + } + + const trace = detailsPretty.flagMetadata?.ddEvaluationTrace + + console.log(`\n${'═'.repeat(60)}\n${label}\n${'═'.repeat(60)}`) + console.log('EvaluationDetails:', JSON.stringify({ ...detailsPretty, flagMetadata: { ...detailsPretty.flagMetadata, ddEvaluationTrace: '(see below)' } }, null, 2)) + console.log('\nddEvaluationTrace:', JSON.stringify(trace, null, 2)) +} + +describe('DDFlagEvaluationDetails demo', () => { + it('runs all scenarios', async () => { + await show('US user', { targetingKey: 'user-us', country: 'US' }) + await show('GB user (rules mismatch → default-off)', { targetingKey: 'user-gb', country: 'GB' }) + await show('flag not found', { targetingKey: 'u' }, 'no-such-flag') + }) +}) diff --git a/packages/node-server/test/flagEvaluationDetails.spec.ts b/packages/node-server/test/flagEvaluationDetails.spec.ts new file mode 100644 index 00000000..3d2d3255 --- /dev/null +++ b/packages/node-server/test/flagEvaluationDetails.spec.ts @@ -0,0 +1,613 @@ +import type { Logger } from '@openfeature/server-sdk' +import { evaluate } from '../src/configuration/evaluation' +import { evaluateForSubject } from '../src/configuration/evaluateForSubject' +import { + AllocationOutcomeCode, + DDFlagEvaluationDetailsBuilder, + FlagEvaluationOutcomeCode, + toResolutionDetails, +} from '../src/configuration/flagEvaluationDetails' +import type { Flag, UniversalFlagConfigurationV1 } from '../src/configuration/ufc-v1' + +const logger: Logger = { + error: jest.fn(), + warn: jest.fn(), + info: jest.fn(), + debug: jest.fn(), +} + +const PAST = new Date(Date.now() - 86400_000).toISOString() +const FUTURE = new Date(Date.now() + 86400_000).toISOString() + +const baseConfig: UniversalFlagConfigurationV1 = { + createdAt: '2026-01-01T00:00:00Z', + format: 'universal-flag-configuration', + environment: { name: 'test-env' }, + flags: {}, +} + +function makeBooleanFlag(overrides: Partial = {}): Flag { + return { + key: 'test-flag', + enabled: true, + variationType: 'BOOLEAN', + variations: { + on: { key: 'on', value: true }, + off: { key: 'off', value: false }, + }, + allocations: [ + { + key: 'default-alloc', + splits: [{ variationKey: 'on', shards: [] }], + }, + ], + ...overrides, + } +} + +// ── Builder unit tests ──────────────────────────────────────────────────────── + +describe('DDFlagEvaluationDetailsBuilder', () => { + const allocations = [ + { key: 'alloc-a', splits: [] }, + { key: 'alloc-b', splits: [] }, + { key: 'alloc-c', splits: [] }, + ] + + it('builds a MATCH outcome with correct allocation trace', () => { + const builder = new DDFlagEvaluationDetailsBuilder('flag-x', allocations as never, null, null) + builder.recordUnmatched(allocations[0] as never, 1, AllocationOutcomeCode.BEFORE_START_TIME) + builder.recordMatch(allocations[1] as never, 2, 'variant-key', 0) + const result = builder.build(true, FlagEvaluationOutcomeCode.MATCH, 'Matched', 'BOOLEAN') + + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.MATCH) + expect(result.variationKey).toBe('variant-key') + expect(result.allocationKey).toBe('alloc-b') + expect(result.matchedAllocation).toMatchObject({ + key: 'alloc-b', + orderPosition: 2, + outcomeCode: AllocationOutcomeCode.MATCH, + matchedRuleIndex: 0, + }) + expect(result.unmatchedAllocations).toHaveLength(1) + expect(result.unmatchedAllocations[0]).toMatchObject({ + key: 'alloc-a', + orderPosition: 1, + outcomeCode: AllocationOutcomeCode.BEFORE_START_TIME, + }) + expect(result.unevaluatedAllocations).toHaveLength(1) + expect(result.unevaluatedAllocations[0]).toMatchObject({ + key: 'alloc-c', + orderPosition: 3, + outcomeCode: AllocationOutcomeCode.UNEVALUATED, + }) + }) + + it('builds a DEFAULT outcome with all allocations unmatched', () => { + const builder = new DDFlagEvaluationDetailsBuilder('flag-x', allocations as never, null, null) + builder.recordUnmatched(allocations[0] as never, 1, AllocationOutcomeCode.RULES_MISMATCH) + builder.recordUnmatched(allocations[1] as never, 2, AllocationOutcomeCode.TRAFFIC_MISS, 1) + builder.recordUnmatched(allocations[2] as never, 3, AllocationOutcomeCode.AFTER_END_TIME) + const result = builder.build(false, FlagEvaluationOutcomeCode.DEFAULT, 'No match', 'BOOLEAN') + + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.DEFAULT) + expect(result.matchedAllocation).toBeNull() + expect(result.unmatchedAllocations).toHaveLength(3) + expect(result.unevaluatedAllocations).toHaveLength(0) + expect(result.unmatchedAllocations[1].matchedRuleIndex).toBe(1) + }) + + it('suppresses allocation lists for pre-waterfall outcomes', () => { + for (const code of [ + FlagEvaluationOutcomeCode.PROVIDER_NOT_READY, + FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING, + FlagEvaluationOutcomeCode.FLAG_NOT_FOUND, + FlagEvaluationOutcomeCode.DISABLED, + FlagEvaluationOutcomeCode.TYPE_MISMATCH, + FlagEvaluationOutcomeCode.ERROR, + ]) { + const builder = new DDFlagEvaluationDetailsBuilder('flag-x', allocations as never, null, null) + const result = builder.build(false, code, 'desc', 'BOOLEAN') + expect(result.unmatchedAllocations).toHaveLength(0) + expect(result.unevaluatedAllocations).toHaveLength(0) + } + }) + + it('throws when recordMatch is called twice', () => { + const builder = new DDFlagEvaluationDetailsBuilder('flag-x', allocations as never, null, null) + builder.recordMatch(allocations[0] as never, 1, 'v1') + expect(() => builder.recordMatch(allocations[1] as never, 2, 'v2')).toThrow(/recordMatch called twice/) + }) + + it('includes configFetchedAt and environmentName', () => { + const builder = new DDFlagEvaluationDetailsBuilder('flag-x', [], '2026-01-01T00:00:00Z', 'prod') + const result = builder.build(false, FlagEvaluationOutcomeCode.DEFAULT, 'desc', 'BOOLEAN') + expect(result.configFetchedAt).toBe('2026-01-01T00:00:00Z') + expect(result.environmentName).toBe('prod') + }) +}) + +// ── evaluateForSubject integration tests ───────────────────────────────────── + +describe('evaluateForSubject waterfall trace', () => { + it('records BEFORE_START_TIME for a future allocation — Date value', () => { + const flag = makeBooleanFlag({ + allocations: [ + { key: 'future-alloc', startAt: new Date(FUTURE), splits: [{ variationKey: 'on', shards: [] }] }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject(flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.BEFORE_START_TIME) + expect(result.matchedAllocation?.key).toBe('fallback') + }) + + it('records BEFORE_START_TIME for a future allocation — ISO string value (JSON deserialization)', () => { + // In production, config arrives as deserialized JSON so startAt/endAt are ISO strings, + // not Date objects. The implementation handles this via the "as unknown as string" cast. + const flag = makeBooleanFlag({ + allocations: [ + { key: 'future-alloc', startAt: FUTURE as unknown as Date, splits: [{ variationKey: 'on', shards: [] }] }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject(flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.BEFORE_START_TIME) + expect(result.matchedAllocation?.key).toBe('fallback') + }) + + it('records AFTER_END_TIME for an expired allocation — Date value', () => { + const flag = makeBooleanFlag({ + allocations: [ + { key: 'expired-alloc', endAt: new Date(PAST), splits: [{ variationKey: 'on', shards: [] }] }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject(flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.AFTER_END_TIME) + expect(result.matchedAllocation?.key).toBe('fallback') + }) + + it('records AFTER_END_TIME for an expired allocation — ISO string value (JSON deserialization)', () => { + const flag = makeBooleanFlag({ + allocations: [ + { key: 'expired-alloc', endAt: PAST as unknown as Date, splits: [{ variationKey: 'on', shards: [] }] }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject(flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.AFTER_END_TIME) + expect(result.matchedAllocation?.key).toBe('fallback') + }) + + it('records RULES_MISMATCH when targeting rules fail; rulesPresent is true', () => { + const flag = makeBooleanFlag({ + allocations: [ + { + key: 'us-only', + rules: [{ conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['US'] }] }], + splits: [{ variationKey: 'on', shards: [] }], + }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject( + flag, 'boolean', 'user-1', { targetingKey: 'user-1', country: 'CA' }, false, logger, null, null, + ) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.RULES_MISMATCH) + expect(result.unmatchedAllocations[0].rulesPresent).toBe(true) + expect(result.matchedAllocation?.key).toBe('fallback') + // fallback has no rules → rulesPresent false + expect(result.matchedAllocation?.rulesPresent).toBe(false) + }) + + it('records matchedRuleIndex when a rule matches; rulesPresent is true', () => { + const flag = makeBooleanFlag({ + allocations: [ + { + key: 'us-only', + rules: [ + { conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['DE'] }] }, + { conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['US'] }] }, + ], + splits: [{ variationKey: 'on', shards: [] }], + }, + ], + }) + const result = evaluateForSubject( + flag, 'boolean', 'user-1', { targetingKey: 'user-1', country: 'US' }, false, logger, null, null, + ) + expect(result.matchedAllocation?.matchedRuleIndex).toBe(1) + expect(result.matchedAllocation?.rulesPresent).toBe(true) + }) + + it('records TRAFFIC_MISS when subject falls outside shard range; preserves matchedRuleIndex', () => { + // Shards with an empty range [0,0) so no subject ever matches the split. + const flag = makeBooleanFlag({ + allocations: [ + { + key: 'narrow-split', + rules: [{ conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['US'] }] }], + splits: [{ variationKey: 'on', shards: [{ salt: 'test', ranges: [{ start: 0, end: 0 }], totalShards: 10000 }] }], + }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject( + flag, 'boolean', 'user-1', { targetingKey: 'user-1', country: 'US' }, false, logger, null, null, + ) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.TRAFFIC_MISS) + expect(result.unmatchedAllocations[0].rulesPresent).toBe(true) + // Rule index 0 matched (the only rule), so matchedRuleIndex should be 0 + expect(result.unmatchedAllocations[0].matchedRuleIndex).toBe(0) + expect(result.matchedAllocation?.key).toBe('fallback') + }) + + it('records TRAFFIC_MISS with no matchedRuleIndex when allocation has no rules', () => { + const flag = makeBooleanFlag({ + allocations: [ + { + key: 'no-rules-narrow', + splits: [{ variationKey: 'on', shards: [{ salt: 'test', ranges: [{ start: 0, end: 0 }], totalShards: 10000 }] }], + }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject( + flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null, + ) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.TRAFFIC_MISS) + expect(result.unmatchedAllocations[0].rulesPresent).toBe(false) + expect(result.unmatchedAllocations[0].matchedRuleIndex).toBeUndefined() + }) + + it('records MISSING_VARIATION when split points to non-existent variation; rulesPresent and matchedRuleIndex are false/undefined', () => { + const flag = makeBooleanFlag({ + allocations: [ + { key: 'corrupt-alloc', splits: [{ variationKey: 'does-not-exist', shards: [] }] }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject(flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.MISSING_VARIATION) + expect(result.unmatchedAllocations[0].rulesPresent).toBe(false) + expect(result.unmatchedAllocations[0].matchedRuleIndex).toBeUndefined() + expect(result.matchedAllocation?.key).toBe('fallback') + }) + + it('records MISSING_VARIATION with matchedRuleIndex when allocation has rules', () => { + const flag = makeBooleanFlag({ + allocations: [ + { + key: 'corrupt-alloc', + rules: [{ conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['US'] }] }], + splits: [{ variationKey: 'does-not-exist', shards: [] }], + }, + { key: 'fallback', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject( + flag, 'boolean', 'user-1', { targetingKey: 'user-1', country: 'US' }, false, logger, null, null, + ) + expect(result.unmatchedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.MISSING_VARIATION) + expect(result.unmatchedAllocations[0].rulesPresent).toBe(true) + expect(result.unmatchedAllocations[0].matchedRuleIndex).toBe(0) + expect(result.matchedAllocation?.key).toBe('fallback') + }) + + it('marks remaining allocations as UNEVALUATED after a match', () => { + const flag = makeBooleanFlag({ + allocations: [ + { key: 'first', splits: [{ variationKey: 'on', shards: [] }] }, + { key: 'second', splits: [{ variationKey: 'off', shards: [] }] }, + { key: 'third', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }) + const result = evaluateForSubject(flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null) + expect(result.matchedAllocation?.key).toBe('first') + expect(result.unevaluatedAllocations).toHaveLength(2) + expect(result.unevaluatedAllocations[0].outcomeCode).toBe(AllocationOutcomeCode.UNEVALUATED) + expect(result.unevaluatedAllocations[0].orderPosition).toBe(2) + expect(result.unevaluatedAllocations[1].orderPosition).toBe(3) + }) + + it('returns DISABLED outcome without allocation lists', () => { + const flag = makeBooleanFlag({ enabled: false }) + const result = evaluateForSubject(flag, 'boolean', 'user-1', { targetingKey: 'user-1' }, false, logger, null, null) + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.DISABLED) + expect(result.unmatchedAllocations).toHaveLength(0) + expect(result.unevaluatedAllocations).toHaveLength(0) + }) + + it('returns TYPE_MISMATCH outcome without allocation lists', () => { + const flag = makeBooleanFlag() // BOOLEAN flag + const result = evaluateForSubject(flag, 'string', 'user-1', { targetingKey: 'user-1' }, 'default', logger, null, null) + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.TYPE_MISMATCH) + expect(result.unmatchedAllocations).toHaveLength(0) + expect(result.unevaluatedAllocations).toHaveLength(0) + }) +}) + +// ── evaluate() pre-waterfall paths ─────────────────────────────────────────── + +describe('evaluate() pre-waterfall paths', () => { + it('returns PROVIDER_NOT_READY when config is undefined; variationType is null', () => { + const result = evaluate(undefined, 'boolean', 'my-flag', false, { targetingKey: 'u' }, logger) + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.PROVIDER_NOT_READY) + expect(result.variationType).toBeNull() + expect(result.configFetchedAt).toBeNull() + expect(result.environmentName).toBeNull() + }) + + it('returns FLAG_NOT_FOUND when flag does not exist; variationType is null', () => { + const result = evaluate(baseConfig, 'boolean', 'missing-flag', false, { targetingKey: 'u' }, logger) + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.FLAG_NOT_FOUND) + expect(result.variationType).toBeNull() + expect(result.configFetchedAt).toBe(baseConfig.createdAt) + expect(result.environmentName).toBe('test-env') + }) + + it('returns ERROR outcome when evaluateForSubject throws unexpectedly', () => { + // A flag with a variationType that is unknown at runtime causes variantTypeToFlagValueType + // to throw inside validateTypeMatch, which propagates to evaluate()'s catch block. + const config = { + ...baseConfig, + flags: { + 'bad-flag': { + ...makeBooleanFlag(), + variationType: 'UNKNOWN_TYPE' as never, + }, + }, + } + const result = evaluate(config, 'boolean', 'bad-flag', false, { targetingKey: 'u' }, logger) + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.ERROR) + expect(result.unmatchedAllocations).toHaveLength(0) + expect(result.unevaluatedAllocations).toHaveLength(0) + }) + + it('returns TARGETING_KEY_MISSING when sharding requires a key but none is provided', () => { + const config: UniversalFlagConfigurationV1 = { + ...baseConfig, + flags: { + 'sharded-flag': makeBooleanFlag({ + key: 'sharded-flag', + allocations: [ + { + key: 'alloc', + splits: [{ variationKey: 'on', shards: [{ salt: 's', ranges: [{ start: 0, end: 10000 }], totalShards: 10000 }] }], + }, + ], + }), + }, + } + const result = evaluate(config, 'boolean', 'sharded-flag', false, {}, logger) + expect(result.outcomeCode).toBe(FlagEvaluationOutcomeCode.TARGETING_KEY_MISSING) + // variationType comes from the flag config, not null, since we reached evaluateForSubject + expect(result.variationType).toBe('BOOLEAN') + }) + + it('threads configFetchedAt and environmentName into the result', () => { + const config: UniversalFlagConfigurationV1 = { + ...baseConfig, + flags: { 'test-flag': makeBooleanFlag() }, + } + const result = evaluate(config, 'boolean', 'test-flag', false, { targetingKey: 'u' }, logger) + expect(result.configFetchedAt).toBe('2026-01-01T00:00:00Z') + expect(result.environmentName).toBe('test-env') + }) +}) + +// ── toResolutionDetails ─────────────────────────────────────────────────────── + +describe('toResolutionDetails', () => { + const config: UniversalFlagConfigurationV1 = { + ...baseConfig, + flags: { 'test-flag': makeBooleanFlag() }, + } + + it('returns correct ResolutionDetails for a MATCH', () => { + const details = evaluate(config, 'boolean', 'test-flag', false, { targetingKey: 'u' }, logger) + const resolution = toResolutionDetails(details, 'boolean', false) + expect(resolution.value).toBe(true) + expect(resolution.reason).toBe('TARGETING_MATCH') + expect(resolution.errorCode).toBeUndefined() + expect(resolution.flagMetadata?.ddEvaluationTrace).toBeUndefined() + }) + + it('includes ddEvaluationTrace when includeTrace is true', () => { + const details = evaluate(config, 'boolean', 'test-flag', false, { targetingKey: 'u' }, logger) + const resolution = toResolutionDetails(details, 'boolean', true) + expect(resolution.flagMetadata?.ddEvaluationTrace).toBeDefined() + const trace = JSON.parse(resolution.flagMetadata!.ddEvaluationTrace as string) + expect(trace.outcomeCode).toBe(FlagEvaluationOutcomeCode.MATCH) + expect(trace.matchedAllocation).toBeDefined() + expect(trace.unmatchedAllocations).toHaveLength(0) + expect(trace.environmentName).toBe('test-env') + expect(trace.configFetchedAt).toBe('2026-01-01T00:00:00Z') + }) + + it('maps DISABLED to DISABLED reason', () => { + const details = evaluate( + { ...config, flags: { 'test-flag': makeBooleanFlag({ enabled: false }) } }, + 'boolean', 'test-flag', false, { targetingKey: 'u' }, logger, + ) + const resolution = toResolutionDetails(details, 'boolean', false) + expect(resolution.reason).toBe('DISABLED') + }) + + it('maps FLAG_NOT_FOUND to ERROR reason with FLAG_NOT_FOUND errorCode', () => { + const details = evaluate(config, 'boolean', 'no-such-flag', false, { targetingKey: 'u' }, logger) + const resolution = toResolutionDetails(details, 'boolean', false) + expect(resolution.reason).toBe('ERROR') + expect(resolution.errorCode).toBe('FLAG_NOT_FOUND') + }) + + it('maps TYPE_MISMATCH to ERROR reason with TYPE_MISMATCH errorCode', () => { + const details = evaluate(config, 'string', 'test-flag', 'def', { targetingKey: 'u' }, logger) + const resolution = toResolutionDetails(details, 'string', false) + expect(resolution.reason).toBe('ERROR') + expect(resolution.errorCode).toBe('TYPE_MISMATCH') + }) + + it('falls back to requestedType for variationType when flag is not found', () => { + const details = evaluate(config, 'boolean', 'no-such-flag', false, { targetingKey: 'u' }, logger) + const resolution = toResolutionDetails(details, 'boolean', false) + expect(resolution.flagMetadata?.variationType).toBe('boolean') + }) + + it('maps DEFAULT to DEFAULT reason with no errorCode', () => { + const details = evaluate( + { ...config, flags: { 'test-flag': makeBooleanFlag({ allocations: [] }) } }, + 'boolean', 'test-flag', false, { targetingKey: 'u' }, logger, + ) + expect(details.outcomeCode).toBe(FlagEvaluationOutcomeCode.DEFAULT) + const resolution = toResolutionDetails(details, 'boolean', false) + expect(resolution.reason).toBe('DEFAULT') + expect(resolution.errorCode).toBeUndefined() + }) + + it('maps PROVIDER_NOT_READY to ERROR reason with PROVIDER_NOT_READY errorCode', () => { + const details = evaluate(undefined, 'boolean', 'any-flag', false, { targetingKey: 'u' }, logger) + const resolution = toResolutionDetails(details, 'boolean', false) + expect(resolution.reason).toBe('ERROR') + expect(resolution.errorCode).toBe('PROVIDER_NOT_READY') + }) + + it('maps TARGETING_KEY_MISSING to ERROR reason with TARGETING_KEY_MISSING errorCode', () => { + const shardedConfig: UniversalFlagConfigurationV1 = { + ...config, + flags: { + 'test-flag': makeBooleanFlag({ + allocations: [{ + key: 'alloc', + splits: [{ variationKey: 'on', shards: [{ salt: 's', ranges: [{ start: 0, end: 10000 }], totalShards: 10000 }] }], + }], + }), + }, + } + const details = evaluate(shardedConfig, 'boolean', 'test-flag', false, {}, logger) + const resolution = toResolutionDetails(details, 'boolean', false) + expect(resolution.reason).toBe('ERROR') + expect(resolution.errorCode).toBe('TARGETING_KEY_MISSING') + }) +}) + +// ── ddEvaluationTrace content ───────────────────────────────────────────────── + +describe('ddEvaluationTrace content', () => { + function parseTrace(resolution: ReturnType) { + return JSON.parse(resolution.flagMetadata!.ddEvaluationTrace as string) + } + + it('includes all top-level trace fields for a MATCH', () => { + const matchConfig: UniversalFlagConfigurationV1 = { + ...baseConfig, + flags: { 'test-flag': makeBooleanFlag() }, + } + const details = evaluate(matchConfig, 'boolean', 'test-flag', false, { targetingKey: 'u' }, logger) + const trace = parseTrace(toResolutionDetails(details, 'boolean', true)) + + expect(trace.flagKey).toBe('test-flag') + expect(trace.variationKey).toBe('on') + expect(trace.allocationKey).toBe('default-alloc') + expect(trace.outcomeCode).toBe(FlagEvaluationOutcomeCode.MATCH) + expect(trace.outcomeDescription).toBeDefined() + expect(trace.matchedAllocation).toMatchObject({ key: 'default-alloc', orderPosition: 1 }) + expect(trace.unmatchedAllocations).toHaveLength(0) + expect(trace.unevaluatedAllocations).toHaveLength(0) + expect(trace.environmentName).toBe('test-env') + expect(trace.configFetchedAt).toBe('2026-01-01T00:00:00Z') + }) + + it('records full waterfall for MATCH after expired, future, and rules-mismatch allocations', () => { + const waterfallConfig: UniversalFlagConfigurationV1 = { + ...baseConfig, + flags: { + 'wf-flag': makeBooleanFlag({ + key: 'wf-flag', + allocations: [ + { key: 'expired', endAt: PAST as unknown as Date, splits: [{ variationKey: 'on', shards: [] }] }, + { key: 'future', startAt: FUTURE as unknown as Date, splits: [{ variationKey: 'on', shards: [] }] }, + { + key: 'us-only', + rules: [{ conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['US'] }] }], + splits: [{ variationKey: 'on', shards: [] }], + }, + { key: 'default-off', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }), + }, + } + const details = evaluate(waterfallConfig, 'boolean', 'wf-flag', false, { targetingKey: 'u', country: 'GB' }, logger) + const trace = parseTrace(toResolutionDetails(details, 'boolean', true)) + + expect(trace.outcomeCode).toBe(FlagEvaluationOutcomeCode.MATCH) + expect(trace.matchedAllocation).toMatchObject({ key: 'default-off', orderPosition: 4 }) + expect(trace.unmatchedAllocations).toHaveLength(3) + expect(trace.unmatchedAllocations[0]).toMatchObject({ key: 'expired', orderPosition: 1, outcomeCode: AllocationOutcomeCode.AFTER_END_TIME }) + expect(trace.unmatchedAllocations[1]).toMatchObject({ key: 'future', orderPosition: 2, outcomeCode: AllocationOutcomeCode.BEFORE_START_TIME }) + expect(trace.unmatchedAllocations[2]).toMatchObject({ key: 'us-only', orderPosition: 3, outcomeCode: AllocationOutcomeCode.RULES_MISMATCH, rulesPresent: true }) + expect(trace.unevaluatedAllocations).toHaveLength(0) + }) + + it('records full waterfall for DEFAULT when all allocations are unmatched', () => { + const allMissConfig: UniversalFlagConfigurationV1 = { + ...baseConfig, + flags: { + 'miss-flag': makeBooleanFlag({ + key: 'miss-flag', + allocations: [ + { + key: 'us-only', + rules: [{ conditions: [{ operator: 'ONE_OF' as never, attribute: 'country', value: ['US'] }] }], + splits: [{ variationKey: 'on', shards: [] }], + }, + ], + }), + }, + } + const details = evaluate(allMissConfig, 'boolean', 'miss-flag', false, { targetingKey: 'u', country: 'CA' }, logger) + const trace = parseTrace(toResolutionDetails(details, 'boolean', true)) + + expect(trace.outcomeCode).toBe(FlagEvaluationOutcomeCode.DEFAULT) + expect(trace.matchedAllocation).toBeNull() + expect(trace.variationKey).toBeNull() + expect(trace.allocationKey).toBeNull() + expect(trace.unmatchedAllocations).toHaveLength(1) + expect(trace.unmatchedAllocations[0]).toMatchObject({ key: 'us-only', outcomeCode: AllocationOutcomeCode.RULES_MISMATCH }) + expect(trace.unevaluatedAllocations).toHaveLength(0) + }) + + it('records empty allocation arrays for FLAG_NOT_FOUND in trace', () => { + const details = evaluate(baseConfig, 'boolean', 'no-flag', false, { targetingKey: 'u' }, logger) + const trace = parseTrace(toResolutionDetails(details, 'boolean', true)) + + expect(trace.outcomeCode).toBe(FlagEvaluationOutcomeCode.FLAG_NOT_FOUND) + expect(trace.matchedAllocation).toBeNull() + expect(trace.unmatchedAllocations).toHaveLength(0) + expect(trace.unevaluatedAllocations).toHaveLength(0) + }) + + it('records unevaluated allocations for early MATCH', () => { + const earlyMatchConfig: UniversalFlagConfigurationV1 = { + ...baseConfig, + flags: { + 'test-flag': makeBooleanFlag({ + allocations: [ + { key: 'first', splits: [{ variationKey: 'on', shards: [] }] }, + { key: 'second', splits: [{ variationKey: 'off', shards: [] }] }, + { key: 'third', splits: [{ variationKey: 'off', shards: [] }] }, + ], + }), + }, + } + const details = evaluate(earlyMatchConfig, 'boolean', 'test-flag', false, { targetingKey: 'u' }, logger) + const trace = parseTrace(toResolutionDetails(details, 'boolean', true)) + + expect(trace.matchedAllocation).toMatchObject({ key: 'first', orderPosition: 1 }) + expect(trace.unmatchedAllocations).toHaveLength(0) + expect(trace.unevaluatedAllocations).toHaveLength(2) + expect(trace.unevaluatedAllocations[0]).toMatchObject({ key: 'second', orderPosition: 2, outcomeCode: AllocationOutcomeCode.UNEVALUATED }) + expect(trace.unevaluatedAllocations[1]).toMatchObject({ key: 'third', orderPosition: 3, outcomeCode: AllocationOutcomeCode.UNEVALUATED }) + }) +})