From e5f14a52f098fae3b60821f5bda7ef77aac41b31 Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:56:49 -0400 Subject: [PATCH 1/4] feat(#2088): migrate marts__micromasters_dedp_exam_grades to dim_course_run Sources semester and passing_grade for the MITxOnline branch from dim_course_run (added in #2319) instead of re-deriving them via a duplicate join to the MicroMasters staging model in int__mitxonline__proctored_exam_grades. The pure-MicroMasters branch still reads these fields from int__micromasters__dedp_proctored_exam_grades, since those exam runs have no corresponding dim_course_run row until MicroMasters grades are added to tfact_grade (epic #2072). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../marts__micromasters_dedp_exam_grades.sql | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql index 2a1190c53..2bdc48853 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql @@ -10,6 +10,17 @@ with micromasters_exam_grades as ( select * from {{ ref('int__micromasters__users') }} ) +-- semester + passing_grade for MITxOnline proctored exam runs are now sourced from +-- dim_course_run (added in #2088) instead of being re-derived here. The pure MicroMasters +-- branch below (micromasters_exam_grades) still reads these fields directly from +-- int__micromasters__dedp_proctored_exam_grades because those exam runs are not represented +-- in dim_course_run until MicroMasters grades are added to tfact_grade (tracked in epic #2072). +, mitxonline_courserun_semester_grade as ( + select courserun_readable_id, semester, passing_grade + from {{ ref('dim_course_run') }} + where platform = 'mitxonline' and is_current +) + select course_number , course_title @@ -36,12 +47,14 @@ select , mitxonline_exam_grades.user_full_name , micromasters_users.user_email as user_micromasters_email , mitxonline_exam_grades.user_email as user_mitxonline_email - , mitxonline_exam_grades.proctoredexamgrade_passing_grade + , mitxonline_courserun_semester_grade.passing_grade as proctoredexamgrade_passing_grade , mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade , mitxonline_exam_grades.proctoredexamgrade_created_on - , mitxonline_exam_grades.semester + , mitxonline_courserun_semester_grade.semester from mitxonline_exam_grades left join micromasters_users on mitxonline_exam_grades.user_username = micromasters_users.user_mitxonline_username +left join mitxonline_courserun_semester_grade + on mitxonline_exam_grades.courserun_readable_id = mitxonline_courserun_semester_grade.courserun_readable_id left join micromasters_exam_grades on mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id From 3de7dc9585ff413674176fd14fbb84126db7e47f Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Tue, 7 Jul 2026 16:30:40 -0400 Subject: [PATCH 2/4] docs(marts): fix issue reference in comment (#2319, not #2088) Corrects a comment that incorrectly attributed the dim_course_run semester/passing_grade columns to #2088 instead of the PR that actually adds them, #2319. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../marts/micromasters/marts__micromasters_dedp_exam_grades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql index 2bdc48853..ab8a9d88e 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql @@ -11,7 +11,7 @@ with micromasters_exam_grades as ( ) -- semester + passing_grade for MITxOnline proctored exam runs are now sourced from --- dim_course_run (added in #2088) instead of being re-derived here. The pure MicroMasters +-- dim_course_run (added in #2319) instead of being re-derived here. The pure MicroMasters -- branch below (micromasters_exam_grades) still reads these fields directly from -- int__micromasters__dedp_proctored_exam_grades because those exam runs are not represented -- in dim_course_run until MicroMasters grades are added to tfact_grade (tracked in epic #2072). From 86b8d3357309f0656392a743a32a7ec5dfc9dee0 Mon Sep 17 00:00:00 2001 From: quazi-h <59845076+quazi-h@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:02:46 -0400 Subject: [PATCH 3/4] fix(marts): guard SCD2 fan-out and rename CTE in dedp exam grades mart - Add row_number()-based dedup (partition by courserun_readable_id, order by effective_date desc) before joining dim_course_run, guarding against the same SCD2 expiration-gap fan-out risk already handled in dim_product. - Rename mitxonline_courserun_semester_grade -> mitxonline_courserun_metadata since the CTE selects course-run metadata (semester, passing_grade), not grade facts. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- logs/dbt.log | 16 +++++++++++ .../marts__micromasters_dedp_exam_grades.sql | 28 ++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 logs/dbt.log diff --git a/logs/dbt.log b/logs/dbt.log new file mode 100644 index 000000000..4b0fd5a8a --- /dev/null +++ b/logs/dbt.log @@ -0,0 +1,16 @@ +17:02:16.333324 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [, , ]} + + +============================== 17:02:16.336997 | c9e4b7ca-f97c-48a7-9dfe-89aee8ad9c0f ============================== +17:02:16.336997 [info ] [MainThread]: Running with dbt=1.11.12 +17:02:16.337420 [debug] [MainThread]: running dbt with arguments {'log_format': 'default', 'warn_error': 'None', 'introspect': 'True', 'no_print': 'None', 'quiet': 'False', 'use_colors': 'True', 'cache_selected_only': 'False', 'warn_error_options': 'WarnErrorOptionsV2(error=[], warn=[], silence=[])', 'log_path': 'logs', 'indirect_selection': 'eager', 'profiles_dir': '/Users/qhoque/.dbt', 'partial_parse': 'True', 'write_json': 'True', 'empty': 'False', 'use_experimental_parser': 'False', 'static_parser': 'True', 'target_path': 'None', 'log_cache_events': 'False', 'fail_fast': 'False', 'invocation_command': 'dbt compile --target dev --select marts__micromasters_dedp_exam_grades', 'debug': 'False', 'printer_width': '80', 'send_anonymous_usage_stats': 'True', 'version_check': 'True'} +17:02:16.337813 [error] [MainThread]: Encountered an error: +Runtime Error + No dbt_project.yml found at expected path /private/tmp/ol-data-platform/dbt_project.yml + Verify that each entry within packages.yml (and their transitive dependencies) contains a file named dbt_project.yml + +17:02:16.338215 [debug] [MainThread]: Resource report: {"command_name": "compile", "command_success": false, "command_wall_clock_time": 0.058058582, "process_in_blocks": "0", "process_kernel_time": 0.195369, "process_mem_max_rss": "121487360", "process_out_blocks": "0", "process_user_time": 0.919255} +17:02:16.338519 [debug] [MainThread]: Command `dbt compile` failed at 17:02:16.338466 after 0.06 seconds +17:02:16.338713 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [, , ]} +17:02:16.338893 [debug] [MainThread]: Flushing usage events +17:02:16.527810 [debug] [MainThread]: An error was encountered while trying to flush usage events diff --git a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql index ab8a9d88e..1156e729b 100644 --- a/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql +++ b/src/ol_dbt/models/marts/micromasters/marts__micromasters_dedp_exam_grades.sql @@ -15,10 +15,24 @@ with micromasters_exam_grades as ( -- branch below (micromasters_exam_grades) still reads these fields directly from -- int__micromasters__dedp_proctored_exam_grades because those exam runs are not represented -- in dim_course_run until MicroMasters grades are added to tfact_grade (tracked in epic #2072). -, mitxonline_courserun_semester_grade as ( +-- Guard against dim_course_run SCD2 expiration gap: multiple is_current=true rows +-- for the same courserun_readable_id can fan out mitxonline_exam_grades rows. +-- Pick the latest, matching the established pattern in dim_product. +, mitxonline_courserun_metadata as ( select courserun_readable_id, semester, passing_grade - from {{ ref('dim_course_run') }} - where platform = 'mitxonline' and is_current + from ( + select + courserun_readable_id + , semester + , passing_grade + , row_number() over ( + partition by courserun_readable_id + order by effective_date desc nulls last + ) as _row_num + from {{ ref('dim_course_run') }} + where platform = 'mitxonline' and is_current + ) + where _row_num = 1 ) select @@ -47,14 +61,14 @@ select , mitxonline_exam_grades.user_full_name , micromasters_users.user_email as user_micromasters_email , mitxonline_exam_grades.user_email as user_mitxonline_email - , mitxonline_courserun_semester_grade.passing_grade as proctoredexamgrade_passing_grade + , mitxonline_courserun_metadata.passing_grade as proctoredexamgrade_passing_grade , mitxonline_exam_grades.proctoredexamgrade_grade as proctoredexamgrade_percentage_grade , mitxonline_exam_grades.proctoredexamgrade_created_on - , mitxonline_courserun_semester_grade.semester + , mitxonline_courserun_metadata.semester from mitxonline_exam_grades left join micromasters_users on mitxonline_exam_grades.user_username = micromasters_users.user_mitxonline_username -left join mitxonline_courserun_semester_grade - on mitxonline_exam_grades.courserun_readable_id = mitxonline_courserun_semester_grade.courserun_readable_id +left join mitxonline_courserun_metadata + on mitxonline_exam_grades.courserun_readable_id = mitxonline_courserun_metadata.courserun_readable_id left join micromasters_exam_grades on mitxonline_exam_grades.courserun_readable_id = micromasters_exam_grades.examrun_readable_id From 1ba882346e87a3164bd983ecf53d7dc01d159bba Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 7 Jul 2026 21:03:49 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- logs/dbt.log | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs/dbt.log b/logs/dbt.log index 4b0fd5a8a..db1f03631 100644 --- a/logs/dbt.log +++ b/logs/dbt.log @@ -8,7 +8,7 @@ Runtime Error No dbt_project.yml found at expected path /private/tmp/ol-data-platform/dbt_project.yml Verify that each entry within packages.yml (and their transitive dependencies) contains a file named dbt_project.yml - + 17:02:16.338215 [debug] [MainThread]: Resource report: {"command_name": "compile", "command_success": false, "command_wall_clock_time": 0.058058582, "process_in_blocks": "0", "process_kernel_time": 0.195369, "process_mem_max_rss": "121487360", "process_out_blocks": "0", "process_user_time": 0.919255} 17:02:16.338519 [debug] [MainThread]: Command `dbt compile` failed at 17:02:16.338466 after 0.06 seconds 17:02:16.338713 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [, , ]}