From fbe5d3123cc2a0c1148e1c52eea30ce3833a0953 Mon Sep 17 00:00:00 2001 From: Rohan Subramaniam Date: Tue, 2 Jun 2026 16:02:57 -0400 Subject: [PATCH 1/2] Holdout exposure metadata prototype --- packages/browser/src/evaluation.ts | 18 ++++++++ packages/browser/test/evaluation.spec.ts | 44 ++++++++++++++++++ .../core/src/configuration/configuration.ts | 26 +++++++++++ .../core/src/configuration/exposureEvent.ts | 11 +++++ .../src/configuration/exposureEvent.types.ts | 6 +++ .../test/configuration/exposureEvent.spec.ts | 43 +++++++++++++++++ .../src/configuration/evaluateForSubject.ts | 14 ++++++ .../test/evaluateForSubject.spec.ts | 46 +++++++++++++++++++ 8 files changed, 208 insertions(+) create mode 100644 packages/core/test/configuration/exposureEvent.spec.ts diff --git a/packages/browser/src/evaluation.ts b/packages/browser/src/evaluation.ts index 48ae3d98..0ada7312 100644 --- a/packages/browser/src/evaluation.ts +++ b/packages/browser/src/evaluation.ts @@ -54,6 +54,7 @@ function evaluatePrecomputed( allocationKey: flag.allocationKey, variationType: flag.variationType, doLog: flag.doLog, + ...holdoutMetadataFromExtraLogging(flag.extraLogging), } as PrecomputedFlagMetadata, reason: flag.reason, } as ResolutionDetails> @@ -77,3 +78,20 @@ function variationTypeToOpenFeature(s: string): FlagValueType { return typeMap[s] || s.toLowerCase() } + +function holdoutMetadataFromExtraLogging(extraLogging?: Record): Partial { + if (!extraLogging) { + return {} + } + + return { + __dd_holdout_key: stringValue(extraLogging.holdoutKey), + __dd_holdout_experiment_id: stringValue(extraLogging.holdoutAnalysisExperimentId), + __dd_holdout_variation: stringValue(extraLogging.holdoutVariation), + __dd_holdout_base_allocation_key: stringValue(extraLogging.holdoutBaseAllocationKey), + } +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} diff --git a/packages/browser/test/evaluation.spec.ts b/packages/browser/test/evaluation.spec.ts index d3fdbf91..d0b561a6 100644 --- a/packages/browser/test/evaluation.spec.ts +++ b/packages/browser/test/evaluation.spec.ts @@ -67,4 +67,48 @@ describe('evaluate', () => { }, }) }) + + it('passes holdout extraLogging through precomputed flag metadata', () => { + const result = evaluate( + { + precomputed: { + response: { + data: { + attributes: { + createdAt: '2026-06-02T00:00:00.000Z', + flags: { + 'checkout-redesign': { + allocationKey: 'allocation-a-holdout-q2-global-holdout', + variationKey: 'control', + variationType: 'boolean', + variationValue: false, + reason: 'TARGETING_MATCH', + doLog: true, + extraLogging: { + holdoutKey: 'q2-global-holdout', + holdoutAnalysisExperimentId: 'holdout-analysis-experiment-id', + holdoutVariation: 'status_quo', + holdoutBaseAllocationKey: 'allocation-a', + }, + }, + }, + }, + }, + }, + }, + }, + 'boolean', + 'checkout-redesign', + true, + {} + ) + + expect(result.flagMetadata).toMatchObject({ + allocationKey: 'allocation-a-holdout-q2-global-holdout', + __dd_holdout_key: 'q2-global-holdout', + __dd_holdout_experiment_id: 'holdout-analysis-experiment-id', + __dd_holdout_variation: 'status_quo', + __dd_holdout_base_allocation_key: 'allocation-a', + }) + }) }) diff --git a/packages/core/src/configuration/configuration.ts b/packages/core/src/configuration/configuration.ts index ab7f473a..5ae947f5 100644 --- a/packages/core/src/configuration/configuration.ts +++ b/packages/core/src/configuration/configuration.ts @@ -58,12 +58,38 @@ export type PrecomputedFlagMetadata = { variationType: FlagValueType doLog: boolean + // Holdout metadata copied from UFC split.extraLogging + __dd_holdout_key?: string + __dd_holdout_experiment_id?: string + __dd_holdout_variation?: string + __dd_holdout_base_allocation_key?: string + // Server-side tracing keys (consumed by dd-trace-js) __dd_allocation_key?: string __dd_do_log?: boolean __dd_split_serial_id?: number } +/** @internal */ +export function holdoutMetadataFromExtraLogging( + extraLogging?: Record +): Partial { + if (!extraLogging) { + return {} + } + + return { + __dd_holdout_key: stringValue(extraLogging.holdoutKey), + __dd_holdout_experiment_id: stringValue(extraLogging.holdoutAnalysisExperimentId), + __dd_holdout_variation: stringValue(extraLogging.holdoutVariation), + __dd_holdout_base_allocation_key: stringValue(extraLogging.holdoutBaseAllocationKey), + } +} + +function stringValue(value: unknown): string | undefined { + return typeof value === 'string' && value.length > 0 ? value : undefined +} + /** * Check if a stored configuration matches the requested evaluation context. * - If the config has no context, it's context-agnostic and matches any request diff --git a/packages/core/src/configuration/exposureEvent.ts b/packages/core/src/configuration/exposureEvent.ts index 143a5465..e1cfecc3 100644 --- a/packages/core/src/configuration/exposureEvent.ts +++ b/packages/core/src/configuration/exposureEvent.ts @@ -18,6 +18,8 @@ export function createExposureEvent( } const { targetingKey: id = '', ...attributes } = context + const holdoutKey = details.flagMetadata?.__dd_holdout_key as string | undefined + const holdoutVariation = details.flagMetadata?.__dd_holdout_variation as string | undefined return { allocation: { @@ -33,5 +35,14 @@ export function createExposureEvent( id, attributes, }, + ...(holdoutKey && + holdoutVariation && { + holdout: { + key: holdoutKey, + experiment_id: details.flagMetadata?.__dd_holdout_experiment_id as string | undefined, + variation: holdoutVariation, + base_allocation_key: details.flagMetadata?.__dd_holdout_base_allocation_key as string | undefined, + }, + }), } satisfies ExposureEvent } diff --git a/packages/core/src/configuration/exposureEvent.types.ts b/packages/core/src/configuration/exposureEvent.types.ts index d60fe750..b8e17176 100644 --- a/packages/core/src/configuration/exposureEvent.types.ts +++ b/packages/core/src/configuration/exposureEvent.types.ts @@ -14,6 +14,12 @@ export interface ExposureEvent { id: string attributes: EvaluationContext } + holdout?: { + key: string + experiment_id?: string + variation: string + base_allocation_key?: string + } service?: string rum?: { application?: { diff --git a/packages/core/test/configuration/exposureEvent.spec.ts b/packages/core/test/configuration/exposureEvent.spec.ts new file mode 100644 index 00000000..95d81031 --- /dev/null +++ b/packages/core/test/configuration/exposureEvent.spec.ts @@ -0,0 +1,43 @@ +import { createExposureEvent } from '../../src/configuration/exposureEvent' + +describe('createExposureEvent', () => { + it('includes holdout metadata from flag metadata on exposure events', () => { + const event = createExposureEvent( + { + targetingKey: 'user-123', + plan: 'pro', + }, + { + flagKey: 'checkout-redesign', + value: false, + variant: 'control', + reason: 'TARGETING_MATCH', + flagMetadata: { + allocationKey: 'allocation-a-holdout-q2-global-holdout', + variationType: 'boolean', + doLog: true, + __dd_holdout_key: 'q2-global-holdout', + __dd_holdout_experiment_id: 'holdout-analysis-experiment-id', + __dd_holdout_variation: 'status_quo', + __dd_holdout_base_allocation_key: 'allocation-a', + }, + } + ) + + expect(event).toMatchObject({ + allocation: { key: 'allocation-a-holdout-q2-global-holdout' }, + flag: { key: 'checkout-redesign' }, + variant: { key: 'control' }, + subject: { + id: 'user-123', + attributes: { plan: 'pro' }, + }, + holdout: { + key: 'q2-global-holdout', + experiment_id: 'holdout-analysis-experiment-id', + variation: 'status_quo', + base_allocation_key: 'allocation-a', + }, + }) + }) +}) diff --git a/packages/node-server/src/configuration/evaluateForSubject.ts b/packages/node-server/src/configuration/evaluateForSubject.ts index af3b4b74..f7d5cac4 100644 --- a/packages/node-server/src/configuration/evaluateForSubject.ts +++ b/packages/node-server/src/configuration/evaluateForSubject.ts @@ -96,6 +96,7 @@ export function evaluateForSubject( allocationKey: allocation.key, variationType: variantTypeToFlagValueType(flag.variationType), doLog: !!allocation.doLog, + ...holdoutMetadataFromExtraLogging(selectedSplit.extraLogging), } as PrecomputedFlagMetadata, } } @@ -136,6 +137,19 @@ function validateTypeMatch(expectedType: FlagValueType, variantType: VariantType throw new Error(`Invalid expected type: ${expectedType}`) } +function holdoutMetadataFromExtraLogging(extraLogging?: Record): Partial { + if (!extraLogging) { + return {} + } + + return { + __dd_holdout_key: extraLogging.holdoutKey, + __dd_holdout_experiment_id: extraLogging.holdoutAnalysisExperimentId, + __dd_holdout_variation: extraLogging.holdoutVariation, + __dd_holdout_base_allocation_key: extraLogging.holdoutBaseAllocationKey, + } +} + export function containsMatchingRule( rules: Rule[] | undefined, subjectAttributes: EvaluationContext, diff --git a/packages/node-server/test/evaluateForSubject.spec.ts b/packages/node-server/test/evaluateForSubject.spec.ts index b3e77f1a..1ed1f23f 100644 --- a/packages/node-server/test/evaluateForSubject.spec.ts +++ b/packages/node-server/test/evaluateForSubject.spec.ts @@ -15,6 +15,52 @@ describe('evaluateForSubject', () => { }) describe('__dd_split_serial_id passthrough', () => { + it('should pass through holdout extraLogging to flagMetadata', () => { + const flag: Flag = { + key: 'checkout-redesign', + enabled: true, + variationType: 'BOOLEAN', + variations: { + control: { key: 'control', value: false }, + }, + allocations: [ + { + key: 'allocation-a-holdout-q2-global-holdout', + doLog: true, + splits: [ + { + variationKey: 'control', + extraLogging: { + holdoutKey: 'q2-global-holdout', + holdoutAnalysisExperimentId: 'holdout-analysis-experiment-id', + holdoutVariation: 'status_quo', + holdoutBaseAllocationKey: 'allocation-a', + }, + shards: [ + { + salt: 'holdout-salt', + ranges: [{ start: 0, end: 10000 }], + totalShards: 10000, + }, + ], + }, + ], + }, + ], + } + + const result = evaluateForSubject(flag, 'boolean', 'user-123', { id: 'user-123' }, true, logger) + + expect(result.value).toBe(false) + expect(result.flagMetadata).toMatchObject({ + allocationKey: 'allocation-a-holdout-q2-global-holdout', + __dd_holdout_key: 'q2-global-holdout', + __dd_holdout_experiment_id: 'holdout-analysis-experiment-id', + __dd_holdout_variation: 'status_quo', + __dd_holdout_base_allocation_key: 'allocation-a', + }) + }) + it('should pass through serialId from split to flagMetadata.__dd_split_serial_id', () => { const serialId = 12345 const flag: Flag = { From 05e550cd50a6fedbd08d0d92f9a8dbc58ebc15fe Mon Sep 17 00:00:00 2001 From: Rohan Subramaniam Date: Wed, 17 Jun 2026 10:36:12 -0400 Subject: [PATCH 2/2] Update holdout prototype to use first-class UFC holdout field Replace extraLogging-based holdout metadata with first-class split.holdout field per RFC. Only key and variation round-trip through the SDK; analysis experiment ID, covered experiment ID, and base allocation key are resolved server-side. Co-Authored-By: Claude Sonnet 4.6 --- packages/browser/src/evaluation.ts | 22 +++---------- packages/browser/test/evaluation.spec.ts | 13 +++----- .../core/src/configuration/configuration.ts | 31 +++++-------------- .../core/src/configuration/exposureEvent.ts | 3 +- .../src/configuration/exposureEvent.types.ts | 2 -- .../test/configuration/exposureEvent.spec.ts | 4 --- 6 files changed, 18 insertions(+), 57 deletions(-) diff --git a/packages/browser/src/evaluation.ts b/packages/browser/src/evaluation.ts index 0ada7312..dcadbf7e 100644 --- a/packages/browser/src/evaluation.ts +++ b/packages/browser/src/evaluation.ts @@ -54,7 +54,10 @@ function evaluatePrecomputed( allocationKey: flag.allocationKey, variationType: flag.variationType, doLog: flag.doLog, - ...holdoutMetadataFromExtraLogging(flag.extraLogging), + ...(flag.holdout && { + __dd_holdout_key: flag.holdout.key, + __dd_holdout_variation: flag.holdout.variation, + }), } as PrecomputedFlagMetadata, reason: flag.reason, } as ResolutionDetails> @@ -78,20 +81,3 @@ function variationTypeToOpenFeature(s: string): FlagValueType { return typeMap[s] || s.toLowerCase() } - -function holdoutMetadataFromExtraLogging(extraLogging?: Record): Partial { - if (!extraLogging) { - return {} - } - - return { - __dd_holdout_key: stringValue(extraLogging.holdoutKey), - __dd_holdout_experiment_id: stringValue(extraLogging.holdoutAnalysisExperimentId), - __dd_holdout_variation: stringValue(extraLogging.holdoutVariation), - __dd_holdout_base_allocation_key: stringValue(extraLogging.holdoutBaseAllocationKey), - } -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined -} diff --git a/packages/browser/test/evaluation.spec.ts b/packages/browser/test/evaluation.spec.ts index d0b561a6..f6e9dfe8 100644 --- a/packages/browser/test/evaluation.spec.ts +++ b/packages/browser/test/evaluation.spec.ts @@ -68,7 +68,7 @@ describe('evaluate', () => { }) }) - it('passes holdout extraLogging through precomputed flag metadata', () => { + it('passes holdout metadata through precomputed flag metadata', () => { const result = evaluate( { precomputed: { @@ -84,11 +84,10 @@ describe('evaluate', () => { variationValue: false, reason: 'TARGETING_MATCH', doLog: true, - extraLogging: { - holdoutKey: 'q2-global-holdout', - holdoutAnalysisExperimentId: 'holdout-analysis-experiment-id', - holdoutVariation: 'status_quo', - holdoutBaseAllocationKey: 'allocation-a', + extraLogging: {}, + holdout: { + key: 'q2-global-holdout', + variation: 'status_quo', }, }, }, @@ -106,9 +105,7 @@ describe('evaluate', () => { expect(result.flagMetadata).toMatchObject({ allocationKey: 'allocation-a-holdout-q2-global-holdout', __dd_holdout_key: 'q2-global-holdout', - __dd_holdout_experiment_id: 'holdout-analysis-experiment-id', __dd_holdout_variation: 'status_quo', - __dd_holdout_base_allocation_key: 'allocation-a', }) }) }) diff --git a/packages/core/src/configuration/configuration.ts b/packages/core/src/configuration/configuration.ts index 5ae947f5..5430dd79 100644 --- a/packages/core/src/configuration/configuration.ts +++ b/packages/core/src/configuration/configuration.ts @@ -40,6 +40,12 @@ export type PrecomputedConfigurationResponse = { } } +/** @internal */ +export type HoldoutMetadata = { + key: string + variation: string +} + /** @internal */ export type PrecomputedFlag = { allocationKey: string @@ -49,6 +55,7 @@ export type PrecomputedFlag = { reason: ResolutionReason doLog: boolean extraLogging: Record + holdout?: HoldoutMetadata } /** @internal */ @@ -58,11 +65,9 @@ export type PrecomputedFlagMetadata = { variationType: FlagValueType doLog: boolean - // Holdout metadata copied from UFC split.extraLogging + // Holdout metadata from UFC split.holdout (first-class field) __dd_holdout_key?: string - __dd_holdout_experiment_id?: string __dd_holdout_variation?: string - __dd_holdout_base_allocation_key?: string // Server-side tracing keys (consumed by dd-trace-js) __dd_allocation_key?: string @@ -70,26 +75,6 @@ export type PrecomputedFlagMetadata = { __dd_split_serial_id?: number } -/** @internal */ -export function holdoutMetadataFromExtraLogging( - extraLogging?: Record -): Partial { - if (!extraLogging) { - return {} - } - - return { - __dd_holdout_key: stringValue(extraLogging.holdoutKey), - __dd_holdout_experiment_id: stringValue(extraLogging.holdoutAnalysisExperimentId), - __dd_holdout_variation: stringValue(extraLogging.holdoutVariation), - __dd_holdout_base_allocation_key: stringValue(extraLogging.holdoutBaseAllocationKey), - } -} - -function stringValue(value: unknown): string | undefined { - return typeof value === 'string' && value.length > 0 ? value : undefined -} - /** * Check if a stored configuration matches the requested evaluation context. * - If the config has no context, it's context-agnostic and matches any request diff --git a/packages/core/src/configuration/exposureEvent.ts b/packages/core/src/configuration/exposureEvent.ts index e1cfecc3..b61758b0 100644 --- a/packages/core/src/configuration/exposureEvent.ts +++ b/packages/core/src/configuration/exposureEvent.ts @@ -21,6 +21,7 @@ export function createExposureEvent( const holdoutKey = details.flagMetadata?.__dd_holdout_key as string | undefined const holdoutVariation = details.flagMetadata?.__dd_holdout_variation as string | undefined + return { allocation: { key: allocationKey, @@ -39,9 +40,7 @@ export function createExposureEvent( holdoutVariation && { holdout: { key: holdoutKey, - experiment_id: details.flagMetadata?.__dd_holdout_experiment_id as string | undefined, variation: holdoutVariation, - base_allocation_key: details.flagMetadata?.__dd_holdout_base_allocation_key as string | undefined, }, }), } satisfies ExposureEvent diff --git a/packages/core/src/configuration/exposureEvent.types.ts b/packages/core/src/configuration/exposureEvent.types.ts index b8e17176..53e65633 100644 --- a/packages/core/src/configuration/exposureEvent.types.ts +++ b/packages/core/src/configuration/exposureEvent.types.ts @@ -16,9 +16,7 @@ export interface ExposureEvent { } holdout?: { key: string - experiment_id?: string variation: string - base_allocation_key?: string } service?: string rum?: { diff --git a/packages/core/test/configuration/exposureEvent.spec.ts b/packages/core/test/configuration/exposureEvent.spec.ts index 95d81031..26a4ec9c 100644 --- a/packages/core/test/configuration/exposureEvent.spec.ts +++ b/packages/core/test/configuration/exposureEvent.spec.ts @@ -17,9 +17,7 @@ describe('createExposureEvent', () => { variationType: 'boolean', doLog: true, __dd_holdout_key: 'q2-global-holdout', - __dd_holdout_experiment_id: 'holdout-analysis-experiment-id', __dd_holdout_variation: 'status_quo', - __dd_holdout_base_allocation_key: 'allocation-a', }, } ) @@ -34,9 +32,7 @@ describe('createExposureEvent', () => { }, holdout: { key: 'q2-global-holdout', - experiment_id: 'holdout-analysis-experiment-id', variation: 'status_quo', - base_allocation_key: 'allocation-a', }, }) })