From 13a352a351c12a0d206156724eded6e0b47ba941 Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:44:19 -0400 Subject: [PATCH 1/3] feat(tfact_enrollment): add Emeritus/Global Alumni platform support - Add emeritus_enrollments/global_alumni_enrollments CTEs sourced from stg__emeritus__api__bigquery__user_enrollments / stg__global_alumni__api__bigquery__user_enrollments (the same staging models dim_user already uses for its emeritus_user_id/global_alumni_user_id columns), following the existing per-platform CTE pattern. - Add mitxpro_external_readable_id_lookup CTE (sourced from int__mitxpro__course_runs) to resolve these platforms' course run FK: Emeritus/Global Alumni don't own course run records, so their enrollments map onto existing MITxPro course runs via courserun_external_readable_id, mirroring the join logic previously used in int__combined__courserun_enrollments. - Extend user_lookup to select emeritus_user_id/global_alumni_user_id from dim_user (already present there) and thread through the user_fk coalesce chain. - Add dim_course_run_mitxpro CTE + join to resolve courserun_fk for these two platforms against MITxPro-platform dim_course_run rows. - enrollment_is_edx_enrolled is null for both (same treatment as program/bootcamps enrollments). Prerequisite for #2075/PR #2314 (migrate marts__combined_course_enrollment_detail to the dimensional layer): without this, sourcing that mart exclusively from tfact_enrollment would silently drop all Emeritus/Global Alumni enrollment rows. Closes #2405 Requires a --full-refresh of tfact_enrollment after deploy to backfill historical Emeritus/Global Alumni enrollment rows (incremental_strategy is delete+insert, so new CTEs alone only capture new activity going forward). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../models/dimensional/tfact_enrollment.sql | 93 ++++++++++++++++++- 1 file changed, 92 insertions(+), 1 deletion(-) diff --git a/src/ol_dbt/models/dimensional/tfact_enrollment.sql b/src/ol_dbt/models/dimensional/tfact_enrollment.sql index dd22941b3..305501276 100644 --- a/src/ol_dbt/models/dimensional/tfact_enrollment.sql +++ b/src/ol_dbt/models/dimensional/tfact_enrollment.sql @@ -157,6 +157,68 @@ with mitxonline_enrollments as ( from {{ ref('int__bootcamps__courserunenrollments') }} ) +-- Emeritus/Global Alumni don't own course run records; their enrollments map onto +-- existing MITxPro course runs via courserun_external_readable_id, mirroring the join +-- logic in int__combined__courserun_enrollments. This lookup resolves that external id +-- to the MITxPro-platform courserun_readable_id used elsewhere in dim_course_run. +, mitxpro_external_readable_id_lookup as ( + select courserun_external_readable_id, courserun_readable_id + from {{ ref('int__mitxpro__course_runs') }} + where courserun_external_readable_id is not null +) + +, emeritus_enrollments as ( + select + -- Stable surrogate key: source has no native enrollment_id + {{ dbt_utils.generate_surrogate_key(['cast(emeritus_enrollments.user_id as varchar)', 'emeritus_enrollments.courserun_external_readable_id']) }} + as enrollment_id + , emeritus_enrollments.user_id + , null as courserun_id + , coalesce( + mitxpro_external_readable_id_lookup.courserun_readable_id + , emeritus_enrollments.courserun_external_readable_id + ) as courserun_readable_id + , null as program_id + , emeritus_enrollments.enrollment_created_on + , cast(null as varchar) as enrollment_updated_on + , emeritus_enrollments.is_enrolled as enrollment_is_active + , cast(null as varchar) as enrollment_mode + , emeritus_enrollments.enrollment_status + , 'emeritus' as platform + , 'emeritus' as platform_code + , cast(null as boolean) as enrollment_is_edx_enrolled + from {{ ref('stg__emeritus__api__bigquery__user_enrollments') }} as emeritus_enrollments + left join mitxpro_external_readable_id_lookup + on emeritus_enrollments.courserun_external_readable_id + = mitxpro_external_readable_id_lookup.courserun_external_readable_id +) + +, global_alumni_enrollments as ( + select + -- Stable surrogate key: source has no native enrollment_id + {{ dbt_utils.generate_surrogate_key(['cast(global_alumni_enrollments.user_id as varchar)', 'global_alumni_enrollments.courserun_external_readable_id']) }} + as enrollment_id + , global_alumni_enrollments.user_id + , null as courserun_id + , coalesce( + mitxpro_external_readable_id_lookup.courserun_readable_id + , global_alumni_enrollments.courserun_external_readable_id + ) as courserun_readable_id + , null as program_id + , global_alumni_enrollments.enrollment_created_on + , cast(null as varchar) as enrollment_updated_on + , global_alumni_enrollments.is_enrolled as enrollment_is_active + , cast(null as varchar) as enrollment_mode + , null as enrollment_status + , 'global_alumni' as platform + , 'global_alumni' as platform_code + , cast(null as boolean) as enrollment_is_edx_enrolled + from {{ ref('stg__global_alumni__api__bigquery__user_enrollments') }} as global_alumni_enrollments + left join mitxpro_external_readable_id_lookup + on global_alumni_enrollments.courserun_external_readable_id + = mitxpro_external_readable_id_lookup.courserun_external_readable_id +) + , combined_enrollments as ( select * from mitxonline_enrollments union all @@ -169,6 +231,10 @@ with mitxonline_enrollments as ( select * from program_enrollments union all select * from bootcamps_enrollments + union all + select * from emeritus_enrollments + union all + select * from global_alumni_enrollments ) -- Join to dimensions for FKs @@ -183,6 +249,8 @@ with mitxonline_enrollments as ( , micromasters_user_id , residential_openedx_user_id , bootcamps_application_user_id + , emeritus_user_id + , global_alumni_user_id from {{ ref('dim_user') }} where user_pk is not null ) @@ -193,6 +261,14 @@ with mitxonline_enrollments as ( where is_current = true ) +-- Emeritus/Global Alumni enrollments resolve their course run FK against MITxPro-platform +-- dim_course_run rows (see mitxpro_external_readable_id_lookup above). +, dim_course_run_mitxpro as ( + select courserun_pk, courserun_readable_id + from {{ ref('dim_course_run') }} + where is_current = true and platform = '{{ var("mitxpro") }}' +) + , dim_program as ( select program_pk, source_id, platform_code from {{ ref('dim_program') }} @@ -235,9 +311,15 @@ with mitxonline_enrollments as ( end, case when combined_enrollments.platform = 'bootcamps' then ul_bootcamps.user_pk + end, + case when combined_enrollments.platform = 'emeritus' + then ul_emeritus.user_pk + end, + case when combined_enrollments.platform = 'global_alumni' + then ul_global_alumni.user_pk end ) as user_fk - , dim_course_run.courserun_pk as courserun_fk + , coalesce(dim_course_run.courserun_pk, dim_course_run_mitxpro.courserun_pk) as courserun_fk , coalesce(dim_program.program_pk, micromasters_program_lookup.micromasters_program_pk) as program_fk , dim_platform_lookup.platform_pk as platform_fk , {{ iso8601_to_date_key('enrollment_created_on') }} as enrollment_date_key @@ -257,9 +339,18 @@ with mitxonline_enrollments as ( left join user_lookup as ul_bootcamps on combined_enrollments.platform = 'bootcamps' and combined_enrollments.user_id = ul_bootcamps.bootcamps_application_user_id + left join user_lookup as ul_emeritus + on combined_enrollments.platform = 'emeritus' + and combined_enrollments.user_id = ul_emeritus.emeritus_user_id + left join user_lookup as ul_global_alumni + on combined_enrollments.platform = 'global_alumni' + and combined_enrollments.user_id = ul_global_alumni.global_alumni_user_id left join dim_course_run on combined_enrollments.courserun_readable_id = dim_course_run.courserun_readable_id and combined_enrollments.platform = dim_course_run.platform + left join dim_course_run_mitxpro + on combined_enrollments.platform in ('emeritus', 'global_alumni') + and combined_enrollments.courserun_readable_id = dim_course_run_mitxpro.courserun_readable_id left join dim_program on cast(combined_enrollments.program_id as varchar) = dim_program.source_id and combined_enrollments.platform_code = dim_program.platform_code From a1b7b5a63c62d824a87d044402ea7d0499bd3438 Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Wed, 8 Jul 2026 14:37:01 -0400 Subject: [PATCH 2/3] fix(tfact_enrollment): correct mitxpro platform filter and Emeritus/Global Alumni column mapping - Use internal platform code 'mitxpro' instead of var("mitxpro") (display label "xPro") when filtering dim_course_run_mitxpro, which was leaving courserun_fk NULL for all Emeritus/Global Alumni enrollments. - global_alumni_enrollments: source has no enrollment_created_on column (would fail compilation against real data); null it out and surface the enrollment_status the staging model does provide instead of dropping it. - emeritus_enrollments: source provides enrollment_updated_on; use it instead of NULL so incremental per-platform watermark logic works instead of falling back to the 7-day lookback path. --- src/ol_dbt/models/dimensional/tfact_enrollment.sql | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ol_dbt/models/dimensional/tfact_enrollment.sql b/src/ol_dbt/models/dimensional/tfact_enrollment.sql index 305501276..d29f38cc5 100644 --- a/src/ol_dbt/models/dimensional/tfact_enrollment.sql +++ b/src/ol_dbt/models/dimensional/tfact_enrollment.sql @@ -180,7 +180,7 @@ with mitxonline_enrollments as ( ) as courserun_readable_id , null as program_id , emeritus_enrollments.enrollment_created_on - , cast(null as varchar) as enrollment_updated_on + , emeritus_enrollments.enrollment_updated_on , emeritus_enrollments.is_enrolled as enrollment_is_active , cast(null as varchar) as enrollment_mode , emeritus_enrollments.enrollment_status @@ -205,11 +205,13 @@ with mitxonline_enrollments as ( , global_alumni_enrollments.courserun_external_readable_id ) as courserun_readable_id , null as program_id - , global_alumni_enrollments.enrollment_created_on + -- source has no enrollment_created_on/enrollment_updated_on; falls back to the + -- 7-day lookback path in incremental_watermarks for platforms without timestamps + , cast(null as varchar) as enrollment_created_on , cast(null as varchar) as enrollment_updated_on , global_alumni_enrollments.is_enrolled as enrollment_is_active , cast(null as varchar) as enrollment_mode - , null as enrollment_status + , global_alumni_enrollments.enrollment_status , 'global_alumni' as platform , 'global_alumni' as platform_code , cast(null as boolean) as enrollment_is_edx_enrolled @@ -266,7 +268,7 @@ with mitxonline_enrollments as ( , dim_course_run_mitxpro as ( select courserun_pk, courserun_readable_id from {{ ref('dim_course_run') }} - where is_current = true and platform = '{{ var("mitxpro") }}' + where is_current = true and platform = 'mitxpro' ) , dim_program as ( From 438018c4307e5199d0b7ca6745d50d5496fce63c Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:31:58 -0400 Subject: [PATCH 3/3] docs(tfact_enrollment): correct comment on Global Alumni incremental behavior The comment claimed Global Alumni rows fall back to the 7-day lookback path, but that path requires enrollment_created_on to be non-null. Since Global Alumni has neither enrollment_created_on nor enrollment_updated_on, it actually hits the unconditional 'or ewf.enrollment_created_on is null' branch and is fully reprocessed on every incremental run. --- src/ol_dbt/models/dimensional/tfact_enrollment.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ol_dbt/models/dimensional/tfact_enrollment.sql b/src/ol_dbt/models/dimensional/tfact_enrollment.sql index d29f38cc5..c751097c2 100644 --- a/src/ol_dbt/models/dimensional/tfact_enrollment.sql +++ b/src/ol_dbt/models/dimensional/tfact_enrollment.sql @@ -205,8 +205,12 @@ with mitxonline_enrollments as ( , global_alumni_enrollments.courserun_external_readable_id ) as courserun_readable_id , null as program_id - -- source has no enrollment_created_on/enrollment_updated_on; falls back to the - -- 7-day lookback path in incremental_watermarks for platforms without timestamps + -- source has no enrollment_created_on/enrollment_updated_on. Unlike edxorg/residential + -- (which have enrollment_created_on and use the 7-day lookback path), the unconditional + -- `or ewf.enrollment_created_on is null` branch in incremental_watermarks means every + -- Global Alumni row is reprocessed on every incremental run (volume is low, so this is + -- an acceptable tradeoff for now; revisit if an ingestion timestamp becomes available + -- upstream). , cast(null as varchar) as enrollment_created_on , cast(null as varchar) as enrollment_updated_on , global_alumni_enrollments.is_enrolled as enrollment_is_active