diff --git a/dbt_project.yml b/dbt_project.yml index 67fa980e..44e8db7a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -29,6 +29,9 @@ models: +schema: wh qc: +schema: qc + observability: + +schema: observability + +materialized: view vars: # labels for generated race/ethnicity groups diff --git a/models/core_warehouse/dim_assessment.yml b/models/core_warehouse/dim_assessment.yml index 573a8fe9..675f8f39 100644 --- a/models/core_warehouse/dim_assessment.yml +++ b/models/core_warehouse/dim_assessment.yml @@ -16,7 +16,7 @@ models: - name: k_assessment description: Generated primary key composed of `tenant_code`, `api_year`, `academicSubjectDescriptor`, `assessmentIdentifier`, and `namespace`. tests: - - unique + - edu_edfi_source.edu_unique - name: tenant_code - name: school_year - name: assessment_identifier diff --git a/models/core_warehouse/dim_calendar_date.yml b/models/core_warehouse/dim_calendar_date.yml index 6a40c587..933abb06 100644 --- a/models/core_warehouse/dim_calendar_date.yml +++ b/models/core_warehouse/dim_calendar_date.yml @@ -18,7 +18,7 @@ models: Generated primary key for an individual day in a single calendar at a single school. tests: - - unique + - edu_edfi_source.edu_unique - name: k_school_calendar description: Association to a single named calendar at a school. - name: k_school diff --git a/models/core_warehouse/dim_classroom.yml b/models/core_warehouse/dim_classroom.yml index 8b8e2481..b89479b8 100644 --- a/models/core_warehouse/dim_classroom.yml +++ b/models/core_warehouse/dim_classroom.yml @@ -15,7 +15,7 @@ models: - name: k_classroom description: Defining key for school locations tests: - - unique + - edu_edfi_source.edu_unique - name: k_school description: Association to the school containing the location - name: tenant_code diff --git a/models/core_warehouse/dim_cohort.yml b/models/core_warehouse/dim_cohort.yml index fdfcd703..c0980082 100644 --- a/models/core_warehouse/dim_cohort.yml +++ b/models/core_warehouse/dim_cohort.yml @@ -21,7 +21,7 @@ models: - name: k_cohort description: Defining key for cohorts. Surrogate key for [`api_year` + `ed_org_id` + `cohort_id`] tests: - - unique + - edu_edfi_source.edu_unique - name: k_lea description: Association to the LEA with the cohort. - name: k_school diff --git a/models/core_warehouse/dim_course.yml b/models/core_warehouse/dim_course.yml index 6e34b747..a496d0ea 100644 --- a/models/core_warehouse/dim_course.yml +++ b/models/core_warehouse/dim_course.yml @@ -15,7 +15,7 @@ models: - name: k_course description: Defining key for courses. Generated primary key composed of `tenant_code`, `api_year`, `course_code`, and `ed_org_id`. tests: - - unique + - edu_edfi_source.edu_unique - name: tenant_code description: "Code defining the Tenant (may be an LEA, SEA, etc.) of the Ed-Fi ODS from which this record was pulled" - name: school_year diff --git a/models/core_warehouse/dim_course_section.yml b/models/core_warehouse/dim_course_section.yml index 20550ec0..fd1b0277 100644 --- a/models/core_warehouse/dim_course_section.yml +++ b/models/core_warehouse/dim_course_section.yml @@ -24,7 +24,7 @@ models: - name: k_course_section description: Defining key for course sections tests: - - unique + - edu_edfi_source.edu_unique - name: k_course description: > Unique identifier for the course. Foreign key reference to diff --git a/models/core_warehouse/dim_discipline_incident.yml b/models/core_warehouse/dim_discipline_incident.yml index 2ecb7e3e..f6d57654 100644 --- a/models/core_warehouse/dim_discipline_incident.yml +++ b/models/core_warehouse/dim_discipline_incident.yml @@ -12,11 +12,11 @@ models: - name: k_discipline_incident description: Generated primary key composed of `tenant_code`, `api_year`, `incident_id`, and `school_id`. tests: - - unique + - edu_edfi_source.edu_unique - name: k_school description: The school where the incident occurred. References dim_school tests: - - not_null + - edu_edfi_source.edu_not_null - name: tenant_code description: "Code defining the Tenant (may be an LEA, SEA, etc.) of the Ed-Fi ODS from which this record was pulled" - name: school_year diff --git a/models/core_warehouse/dim_grading_period.yml b/models/core_warehouse/dim_grading_period.yml index 980291a4..f3d0902e 100644 --- a/models/core_warehouse/dim_grading_period.yml +++ b/models/core_warehouse/dim_grading_period.yml @@ -14,7 +14,7 @@ models: - name: k_grading_period description: Generated primary key composed of 'grading_period', 'period_sequence', 'school_id', and 'school_year' (generated in [stg_ef3__grading_periods](#!/model/model.edu_edfi_source.stg_ef3__grading_periods)) tests: - - unique + - edu_edfi_source.edu_unique - name: k_school description: Unique identifier for the school. Foreign key reference to [dim_school](#!/model/model.edu_wh.dim_school). - name: tenant_code diff --git a/models/core_warehouse/dim_graduation_plan.yml b/models/core_warehouse/dim_graduation_plan.yml index 99483960..9586c066 100644 --- a/models/core_warehouse/dim_graduation_plan.yml +++ b/models/core_warehouse/dim_graduation_plan.yml @@ -14,7 +14,7 @@ models: - name: k_graduation_plan description: Generated primary key composed of 'graduation_plan_type', 'ed_org_id', and 'graduation_school_year' (generated in [stg_ef3__graduation_plans](#!/model/model.edu_edfi_source.stg_ef3__graduation_plans)) tests: - - unique + - edu_edfi_source.edu_unique - name: tenant_code description: Code defining the Tenant (may be an LEA, SEA, etc.) of the Ed-Fi ODS from which this record was pulled - name: school_year diff --git a/models/core_warehouse/dim_learning_standard.yml b/models/core_warehouse/dim_learning_standard.yml index 92baa171..a9667028 100644 --- a/models/core_warehouse/dim_learning_standard.yml +++ b/models/core_warehouse/dim_learning_standard.yml @@ -16,7 +16,7 @@ models: - name: k_learning_standard description: Unique identifier for a learning standard. tests: - - unique + - edu_edfi_source.edu_unique - name: k_learning_standard__parent description: A reference linking to a hierarchical parent learning standard id. - name: tenant_code diff --git a/models/core_warehouse/dim_objective_assessment.yml b/models/core_warehouse/dim_objective_assessment.yml index 22435ea9..b368401d 100644 --- a/models/core_warehouse/dim_objective_assessment.yml +++ b/models/core_warehouse/dim_objective_assessment.yml @@ -13,7 +13,7 @@ models: Generated primary key composed of `tenant_code`, `api_year`, `academicSubjectDescriptor`, `assessmentIdentifier`, `namespace`, and `objectiveAssessmentIdentificationCode`. tests: - - unique + - edu_edfi_source.edu_unique - name: k_assessment - name: tenant_code - name: school_year diff --git a/models/core_warehouse/dim_parent.yml b/models/core_warehouse/dim_parent.yml index 613ab405..202fd373 100644 --- a/models/core_warehouse/dim_parent.yml +++ b/models/core_warehouse/dim_parent.yml @@ -13,8 +13,8 @@ models: - name: k_parent description: Defining key for the parent, which is consistent across years. tests: - - unique - - not_null + - edu_edfi_source.edu_unique + - edu_edfi_source.edu_not_null - name: tenant_code - name: school_year - name: parent_unique_id diff --git a/models/core_warehouse/dim_program.yml b/models/core_warehouse/dim_program.yml index 0cff7020..99d574bc 100644 --- a/models/core_warehouse/dim_program.yml +++ b/models/core_warehouse/dim_program.yml @@ -23,7 +23,7 @@ models: - name: k_program description: Defining key for programs. Surrogate key for [`api_year` + `ed_org_id` + `program_name` + `program_type`] tests: - - unique + - edu_edfi_source.edu_unique - name: k_lea description: Association to the LEA with the program. - name: k_school diff --git a/models/core_warehouse/dim_school.yml b/models/core_warehouse/dim_school.yml index 4a61895c..81a5781e 100644 --- a/models/core_warehouse/dim_school.yml +++ b/models/core_warehouse/dim_school.yml @@ -14,7 +14,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_school columns: diff --git a/models/core_warehouse/dim_student.yml b/models/core_warehouse/dim_student.yml index 290a5d3c..c67bec72 100644 --- a/models/core_warehouse/dim_student.yml +++ b/models/core_warehouse/dim_student.yml @@ -24,20 +24,20 @@ models: columns: - name: k_student description: Defining key for the student in this school year. - tests: - - unique - - not_null + tests: + - edu_edfi_source.edu_unique + - edu_edfi_source.edu_not_null - name: k_student_xyear description: Defining key for the student, which is consistent across years. - tests: - - not_null + tests: + - edu_edfi_source.edu_not_null - name: tenant_code - name: school_year description: > School year specified by Spring year. e.g. the 2021-2022 year would be 2022. - tests: - - not_null + tests: + - edu_edfi_source.edu_not_null - name: student_unique_id description: The student's unique id number in the education organization. - name: first_name diff --git a/models/core_warehouse/dim_subgroup.yml b/models/core_warehouse/dim_subgroup.yml index c71490e3..f8c88ed4 100644 --- a/models/core_warehouse/dim_subgroup.yml +++ b/models/core_warehouse/dim_subgroup.yml @@ -21,7 +21,7 @@ models: - name: k_subgroup description: Defining key for subgroups. A surrogate key for [`subgroup_category` + `subgroup_value`] tests: - - unique + - edu_edfi_source.edu_unique - name: subgroup_category description: Category of subgroup. e.g. "grade_level" - name: subgroup_category_display_name diff --git a/models/core_warehouse/fct_course_transcripts.yml b/models/core_warehouse/fct_course_transcripts.yml index 963648b2..d71ff981 100644 --- a/models/core_warehouse/fct_course_transcripts.yml +++ b/models/core_warehouse/fct_course_transcripts.yml @@ -40,7 +40,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_course - k_student_academic_record diff --git a/models/core_warehouse/fct_student_assessment.yml b/models/core_warehouse/fct_student_assessment.yml index aa545f87..2a4ab07b 100644 --- a/models/core_warehouse/fct_student_assessment.yml +++ b/models/core_warehouse/fct_student_assessment.yml @@ -48,7 +48,7 @@ models: description: > Generated primary key composed of `tenant_code`, `api_year`, and `studentAssessmentIdentifier`. tests: - - unique + - edu_edfi_source.edu_unique - name: k_assessment description: Unique identifier of the assessment. Foreign key reference to `dim_assessment`. - name: k_student diff --git a/models/core_warehouse/fct_student_cohort_association.yml b/models/core_warehouse/fct_student_cohort_association.yml index a6a4ec10..6e90dfc0 100644 --- a/models/core_warehouse/fct_student_cohort_association.yml +++ b/models/core_warehouse/fct_student_cohort_association.yml @@ -13,7 +13,7 @@ models: tags: ['cohort'] enabled: "{{ var('src:domain:cohort:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_cohort diff --git a/models/core_warehouse/fct_student_cte_program_associations.yml b/models/core_warehouse/fct_student_cte_program_associations.yml index eb109d6a..24def7d8 100644 --- a/models/core_warehouse/fct_student_cte_program_associations.yml +++ b/models/core_warehouse/fct_student_cte_program_associations.yml @@ -15,7 +15,7 @@ models: enabled: "{{ var('src:program:cte:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_daily_attendance.yml b/models/core_warehouse/fct_student_daily_attendance.yml index 817e01b7..abed4049 100644 --- a/models/core_warehouse/fct_student_daily_attendance.yml +++ b/models/core_warehouse/fct_student_daily_attendance.yml @@ -24,7 +24,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_school diff --git a/models/core_warehouse/fct_student_diploma.yml b/models/core_warehouse/fct_student_diploma.yml index cab7ca33..9270c54b 100644 --- a/models/core_warehouse/fct_student_diploma.yml +++ b/models/core_warehouse/fct_student_diploma.yml @@ -13,7 +13,7 @@ models: {{ doc(var('edu:custom_docs:fct_student_diploma')) if var('edu:custom_docs:fct_student_diploma', '') }} tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_student_xyear diff --git a/models/core_warehouse/fct_student_discipline_actions.yml b/models/core_warehouse/fct_student_discipline_actions.yml index a1ac6955..6a628f04 100644 --- a/models/core_warehouse/fct_student_discipline_actions.yml +++ b/models/core_warehouse/fct_student_discipline_actions.yml @@ -119,7 +119,7 @@ models: tags: ['discipline'] enabled: "{{ var('src:domain:discipline:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_student_xyear diff --git a/models/core_warehouse/fct_student_discipline_actions_summary.yml b/models/core_warehouse/fct_student_discipline_actions_summary.yml index df285d8b..cdb1df78 100644 --- a/models/core_warehouse/fct_student_discipline_actions_summary.yml +++ b/models/core_warehouse/fct_student_discipline_actions_summary.yml @@ -18,7 +18,7 @@ models: *Primary Key:* k_student, k_student_xyear, k_discipline_actions_event tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_student_xyear diff --git a/models/core_warehouse/fct_student_discipline_incident_behaviors.yml b/models/core_warehouse/fct_student_discipline_incident_behaviors.yml index 2347b9f7..f77a64ea 100644 --- a/models/core_warehouse/fct_student_discipline_incident_behaviors.yml +++ b/models/core_warehouse/fct_student_discipline_incident_behaviors.yml @@ -12,7 +12,7 @@ models: `k_student, k_discipline_incident, behavior_type` -- There is one record per student, year, incident ID, school, and behavior. tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_student_xyear diff --git a/models/core_warehouse/fct_student_discipline_incident_non_offenders.yml b/models/core_warehouse/fct_student_discipline_incident_non_offenders.yml index 87db8bcc..378d437c 100644 --- a/models/core_warehouse/fct_student_discipline_incident_non_offenders.yml +++ b/models/core_warehouse/fct_student_discipline_incident_non_offenders.yml @@ -12,7 +12,7 @@ models: `k_student, k_discipline_incident` -- There is one record per student, year, incident ID, and school. tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_student_xyear diff --git a/models/core_warehouse/fct_student_discipline_incident_summary.yml b/models/core_warehouse/fct_student_discipline_incident_summary.yml index 36986ac4..da3ff095 100644 --- a/models/core_warehouse/fct_student_discipline_incident_summary.yml +++ b/models/core_warehouse/fct_student_discipline_incident_summary.yml @@ -18,7 +18,7 @@ models: *Primary Key:* k_student, k_student_xyear, k_discipline_incident tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_student_xyear diff --git a/models/core_warehouse/fct_student_grades.yml b/models/core_warehouse/fct_student_grades.yml index d68434a3..20bcf2dd 100644 --- a/models/core_warehouse/fct_student_grades.yml +++ b/models/core_warehouse/fct_student_grades.yml @@ -23,7 +23,7 @@ models: config: tags: ['core', 'course'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_school diff --git a/models/core_warehouse/fct_student_homeless_program_association.yml b/models/core_warehouse/fct_student_homeless_program_association.yml index 8fe238e6..2d137495 100644 --- a/models/core_warehouse/fct_student_homeless_program_association.yml +++ b/models/core_warehouse/fct_student_homeless_program_association.yml @@ -15,7 +15,7 @@ models: enabled: "{{ var('src:program:homeless:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_language_instruction_program_association.yml b/models/core_warehouse/fct_student_language_instruction_program_association.yml index 1e51ceee..90934131 100644 --- a/models/core_warehouse/fct_student_language_instruction_program_association.yml +++ b/models/core_warehouse/fct_student_language_instruction_program_association.yml @@ -15,7 +15,7 @@ models: enabled: "{{ var('src:program:language_instruction:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_learning_standard_grades.yml b/models/core_warehouse/fct_student_learning_standard_grades.yml index c61633a3..9d8e4cc0 100644 --- a/models/core_warehouse/fct_student_learning_standard_grades.yml +++ b/models/core_warehouse/fct_student_learning_standard_grades.yml @@ -34,7 +34,7 @@ models: config: tags: ['core', 'course'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_school diff --git a/models/core_warehouse/fct_student_migrant_education_program_associations.yml b/models/core_warehouse/fct_student_migrant_education_program_associations.yml index 08c0bdc0..194606fe 100644 --- a/models/core_warehouse/fct_student_migrant_education_program_associations.yml +++ b/models/core_warehouse/fct_student_migrant_education_program_associations.yml @@ -15,7 +15,7 @@ models: enabled: "{{ var('src:program:migrant_education:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_objective_assessment.yml b/models/core_warehouse/fct_student_objective_assessment.yml index c4b343e3..01b72c1a 100644 --- a/models/core_warehouse/fct_student_objective_assessment.yml +++ b/models/core_warehouse/fct_student_objective_assessment.yml @@ -32,7 +32,7 @@ models: description: > Generated primary key composed of `tenant_code`, `api_year`, `studentAssessmentIdentifier`, and `objectiveAssessmentIdentificationCode`. tests: - - unique + - edu_edfi_source.edu_unique - name: k_objective_assessment - name: k_student_assessment - name: k_assessment diff --git a/models/core_warehouse/fct_student_parent_association.yml b/models/core_warehouse/fct_student_parent_association.yml index d3e99615..46002eb0 100644 --- a/models/core_warehouse/fct_student_parent_association.yml +++ b/models/core_warehouse/fct_student_parent_association.yml @@ -17,7 +17,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_parent diff --git a/models/core_warehouse/fct_student_program_association.yml b/models/core_warehouse/fct_student_program_association.yml index 77bade84..a553898b 100644 --- a/models/core_warehouse/fct_student_program_association.yml +++ b/models/core_warehouse/fct_student_program_association.yml @@ -10,7 +10,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_program_service.yml b/models/core_warehouse/fct_student_program_service.yml index 1553985c..4ce0e891 100644 --- a/models/core_warehouse/fct_student_program_service.yml +++ b/models/core_warehouse/fct_student_program_service.yml @@ -79,7 +79,7 @@ models: tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_school_association.yml b/models/core_warehouse/fct_student_school_association.yml index fa078161..f19d1187 100644 --- a/models/core_warehouse/fct_student_school_association.yml +++ b/models/core_warehouse/fct_student_school_association.yml @@ -54,7 +54,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_school diff --git a/models/core_warehouse/fct_student_school_attendance_event.yml b/models/core_warehouse/fct_student_school_attendance_event.yml index e0c61769..ba824763 100644 --- a/models/core_warehouse/fct_student_school_attendance_event.yml +++ b/models/core_warehouse/fct_student_school_attendance_event.yml @@ -17,7 +17,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_school diff --git a/models/core_warehouse/fct_student_school_food_service_program_associations.yml b/models/core_warehouse/fct_student_school_food_service_program_associations.yml index b5b987bf..e4331b4d 100644 --- a/models/core_warehouse/fct_student_school_food_service_program_associations.yml +++ b/models/core_warehouse/fct_student_school_food_service_program_associations.yml @@ -15,7 +15,7 @@ models: enabled: "{{ var('src:program:food_service:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_section_association.yml b/models/core_warehouse/fct_student_section_association.yml index c41bce6c..893800d4 100644 --- a/models/core_warehouse/fct_student_section_association.yml +++ b/models/core_warehouse/fct_student_section_association.yml @@ -15,7 +15,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_course_section diff --git a/models/core_warehouse/fct_student_special_education_program_association.yml b/models/core_warehouse/fct_student_special_education_program_association.yml index b3fd747e..58b31e0a 100644 --- a/models/core_warehouse/fct_student_special_education_program_association.yml +++ b/models/core_warehouse/fct_student_special_education_program_association.yml @@ -14,7 +14,7 @@ models: tags: ['special_ed'] enabled: "{{ var('src:program:special_ed:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/models/core_warehouse/fct_student_subgroup.yml b/models/core_warehouse/fct_student_subgroup.yml index 3d7282ca..4f70fb95 100644 --- a/models/core_warehouse/fct_student_subgroup.yml +++ b/models/core_warehouse/fct_student_subgroup.yml @@ -79,7 +79,7 @@ models: config: tags: ['core'] tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_subgroup diff --git a/models/core_warehouse/fct_student_title_i_part_a_program_association.yml b/models/core_warehouse/fct_student_title_i_part_a_program_association.yml index 3a517d24..fb6b3e90 100644 --- a/models/core_warehouse/fct_student_title_i_part_a_program_association.yml +++ b/models/core_warehouse/fct_student_title_i_part_a_program_association.yml @@ -15,7 +15,7 @@ models: enabled: "{{ var('src:program:title_i:enabled', True) }}" tests: - - dbt_utils.unique_combination_of_columns: + - edu_edfi_source.edu_unique_combination_of_columns: combination_of_columns: - k_student - k_program diff --git a/packages.yml b/packages.yml index 2841d9c2..23a86a70 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,4 @@ packages: - - package: edanalytics/edu_edfi_source - version: [">=0.5.1", "<0.6.0"] + # - package: edanalytics/edu_edfi_source + # version: [">=0.4.0", "<0.5.0"] + - local: ~/code/edu_edfi_source diff --git a/tests/config_tests/cfg_assessment_scores.sql b/tests/config_tests/cfg_assessment_scores.sql index 51535cec..fe527cb2 100644 --- a/tests/config_tests/cfg_assessment_scores.sql +++ b/tests/config_tests/cfg_assessment_scores.sql @@ -47,4 +47,6 @@ joined as ( order by assessment_identifier ) -select * from joined \ No newline at end of file +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/config_tests/cfg_attendance_event_codes.sql b/tests/config_tests/cfg_attendance_event_codes.sql index 83c06cd9..9d020ef9 100644 --- a/tests/config_tests/cfg_attendance_event_codes.sql +++ b/tests/config_tests/cfg_attendance_event_codes.sql @@ -20,4 +20,6 @@ joined as ( on stg_attendance_events.attendance_event_category = xwalk_attendance_events.attendance_event_descriptor where xwalk_attendance_events.is_absent is null ) -select * from joined \ No newline at end of file +select count(*) as failed_row_count, tenant_code, api_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/config_tests/cfg_behavior_types.sql b/tests/config_tests/cfg_behavior_types.sql index b4c8f89a..df984774 100644 --- a/tests/config_tests/cfg_behavior_types.sql +++ b/tests/config_tests/cfg_behavior_types.sql @@ -7,16 +7,25 @@ with fct_student_discipline_incident_behaviors as ( select * from {{ ref('fct_student_discipline_incident_behaviors') }} ), +dim_student as ( + select * from {{ ref('dim_student')}} +), xwalk_discipline_behaviors as ( select * from {{ ref('xwalk_discipline_behaviors') }} ), joined as ( select distinct fct_student_discipline_incident_behaviors.tenant_code, + dim_student.school_year, fct_student_discipline_incident_behaviors.behavior_type from fct_student_discipline_incident_behaviors left join xwalk_discipline_behaviors on fct_student_discipline_incident_behaviors.behavior_type = xwalk_discipline_behaviors.behavior_type + join dim_student + on fct_student_discipline_incident_behaviors.k_student = dim_student.k_student where xwalk_discipline_behaviors.severity_order is null + ) -select * from joined \ No newline at end of file +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/config_tests/cfg_calendar_event_codes.sql b/tests/config_tests/cfg_calendar_event_codes.sql index c8fed2fd..c88f6a38 100644 --- a/tests/config_tests/cfg_calendar_event_codes.sql +++ b/tests/config_tests/cfg_calendar_event_codes.sql @@ -26,4 +26,6 @@ joined as ( on stg_calendar_events.calendar_event = xwalk_calendar_events.calendar_event_descriptor where xwalk_calendar_events.is_school_day is null ) -select * from joined \ No newline at end of file +select count(*) as failed_row_count, tenant_code, api_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/config_tests/cfg_discipline_actions.sql b/tests/config_tests/cfg_discipline_actions.sql index 1213f78c..70f88ba0 100644 --- a/tests/config_tests/cfg_discipline_actions.sql +++ b/tests/config_tests/cfg_discipline_actions.sql @@ -13,10 +13,13 @@ xwalk_discipline_actions as ( joined as ( select distinct fct_student_discipline_action.tenant_code, + fct_student_discipline_action.school_year, fct_student_discipline_action.discipline_action from fct_student_discipline_action left join xwalk_discipline_actions on fct_student_discipline_action.discipline_action = xwalk_discipline_actions.discipline_action where xwalk_discipline_actions.severity_order is null ) -select * from joined \ No newline at end of file +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/config_tests/cfg_letter_grades.sql b/tests/config_tests/cfg_letter_grades.sql index 3acd1bba..20af4df4 100644 --- a/tests/config_tests/cfg_letter_grades.sql +++ b/tests/config_tests/cfg_letter_grades.sql @@ -20,4 +20,6 @@ joined as ( on lower(stg_grades.letter_grade_earned) = xwalk_letter_grades.letter_grade where xwalk_letter_grades.letter_grade is null ) -select * from joined \ No newline at end of file +select count(*) as failed_row_count, tenant_code, api_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/config_tests/cfg_objective_assessment_scores.sql b/tests/config_tests/cfg_objective_assessment_scores.sql index dc2ca121..6ab68a24 100644 --- a/tests/config_tests/cfg_objective_assessment_scores.sql +++ b/tests/config_tests/cfg_objective_assessment_scores.sql @@ -36,4 +36,6 @@ joined as ( where xwalk_objective_assessment_scores.objective_assessment_identification_code is null ) -select * from joined +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 diff --git a/tests/config_tests/cfg_section_attendance_event_codes.sql b/tests/config_tests/cfg_section_attendance_event_codes.sql index 2507ce96..1298049e 100644 --- a/tests/config_tests/cfg_section_attendance_event_codes.sql +++ b/tests/config_tests/cfg_section_attendance_event_codes.sql @@ -20,4 +20,6 @@ joined as ( on stg_section_attendance_events.attendance_event_category = xwalk_attendance_events.attendance_event_descriptor where xwalk_attendance_events.is_absent is null ) -select * from joined \ No newline at end of file +select count(*) as failed_row_count, tenant_code, api_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/integrity_tests/all_attendance_dates_have_calendar_dates.sql b/tests/integrity_tests/all_attendance_dates_have_calendar_dates.sql index 2e474602..78bd4c77 100644 --- a/tests/integrity_tests/all_attendance_dates_have_calendar_dates.sql +++ b/tests/integrity_tests/all_attendance_dates_have_calendar_dates.sql @@ -23,19 +23,26 @@ dim_calendar_date as ( ), dim_student as ( select * from {{ ref('dim_student') }} +), +joined as ( + select + stg_stu_sch_attend.tenant_code, + stg_stu_sch_attend.api_year, + stg_stu_sch_attend.k_student, + stg_stu_sch_attend.k_school, + fct_stu_school_assoc.k_school_calendar, + stg_stu_sch_attend.attendance_event_date + from stg_stu_sch_attend + join dim_student + on stg_stu_sch_attend.k_student = dim_student.k_student + left join fct_stu_school_assoc + on stg_stu_sch_attend.k_student = fct_stu_school_assoc.k_student + and stg_stu_sch_attend.k_school = fct_stu_school_assoc.k_school + left join dim_calendar_date + on fct_stu_school_assoc.k_school_calendar = dim_calendar_date.k_school_calendar + and stg_stu_sch_attend.attendance_event_date = dim_calendar_date.calendar_date + where dim_calendar_date.k_calendar_date is null ) -select - stg_stu_sch_attend.k_student, - stg_stu_sch_attend.k_school, - fct_stu_school_assoc.k_school_calendar, - stg_stu_sch_attend.attendance_event_date -from stg_stu_sch_attend -join dim_student - on stg_stu_sch_attend.k_student = dim_student.k_student -left join fct_stu_school_assoc - on stg_stu_sch_attend.k_student = fct_stu_school_assoc.k_student - and stg_stu_sch_attend.k_school = fct_stu_school_assoc.k_school -left join dim_calendar_date - on fct_stu_school_assoc.k_school_calendar = dim_calendar_date.k_school_calendar - and stg_stu_sch_attend.attendance_event_date = dim_calendar_date.calendar_date -where dim_calendar_date.k_calendar_date is null \ No newline at end of file +select count(*) as failed_row_count, tenant_code, api_year from joined +group by all +having count(*) > 1 diff --git a/tests/integrity_tests/all_student_school_associations_have_calendars.sql b/tests/integrity_tests/all_student_school_associations_have_calendars.sql index 64bf39b3..7673de33 100644 --- a/tests/integrity_tests/all_student_school_associations_have_calendars.sql +++ b/tests/integrity_tests/all_student_school_associations_have_calendars.sql @@ -30,33 +30,39 @@ school_calendar_counts as ( count(*) as n_calendars_at_school from dim_calendar group by 1,2 -) -select - fct_stu_school_assoc.k_school, - dim_school.school_id, - fct_stu_school_assoc.school_year, - count_if(dim_calendar.k_school_calendar is null) as n_students_missing_calendar, - count(*) as n_total, - round(100*n_students_missing_calendar/n_total, 2) as pct_missing, - any_value(school_calendar_counts.n_calendars_at_school) as n_calendars_at_school, - n_students_missing_calendar || ' records (' || pct_missing || ' %) of student enrollments have ambiguous calendars: the enrollment has no calendar association, and the school has multiple calendars. Calendars are necessary to calculate attendance.' as audit_message -from fct_stu_school_assoc -join dim_school - on fct_stu_school_assoc.k_school = dim_school.k_school -join school_calendar_counts - on dim_school.k_school = school_calendar_counts.k_school - and fct_stu_school_assoc.school_year = school_calendar_counts.school_year -left join dim_calendar - on fct_stu_school_assoc.k_school_calendar = dim_calendar.k_school_calendar -join first_school_day - on dim_school.k_school = first_school_day.k_school - and fct_stu_school_assoc.school_year = first_school_day.school_year - -- subset to school overall first date - and first_school_day.k_school_calendar is null -where true --- exclude students who exited before first day of school -and (fct_stu_school_assoc.exit_withdraw_date > first_school_day.first_school_day -or fct_stu_school_assoc.exit_withdraw_date is null) +), +joined as ( + select + fct_stu_school_assoc.tenant_code, + fct_stu_school_assoc.k_school, + dim_school.school_id, + fct_stu_school_assoc.school_year, + count_if(dim_calendar.k_school_calendar is null) as n_students_missing_calendar, + count(*) as n_total, + round(100*n_students_missing_calendar/n_total, 2) as pct_missing, + any_value(school_calendar_counts.n_calendars_at_school) as n_calendars_at_school, + n_students_missing_calendar || ' records (' || pct_missing || ' %) of student enrollments have ambiguous calendars: the enrollment has no calendar association, and the school has multiple calendars. Calendars are necessary to calculate attendance.' as audit_message + from fct_stu_school_assoc + join dim_school + on fct_stu_school_assoc.k_school = dim_school.k_school + join school_calendar_counts + on dim_school.k_school = school_calendar_counts.k_school + and fct_stu_school_assoc.school_year = school_calendar_counts.school_year + left join dim_calendar + on fct_stu_school_assoc.k_school_calendar = dim_calendar.k_school_calendar + join first_school_day + on dim_school.k_school = first_school_day.k_school + and fct_stu_school_assoc.school_year = first_school_day.school_year + -- subset to school overall first date + and first_school_day.k_school_calendar is null + where true + -- exclude students who exited before first day of school + and (fct_stu_school_assoc.exit_withdraw_date > first_school_day.first_school_day + or fct_stu_school_assoc.exit_withdraw_date is null) -group by 1,2,3 -having n_students_missing_calendar != 0 \ No newline at end of file + group by 1,2,3,4 + having n_students_missing_calendar != 0 +) +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/integrity_tests/all_students_have_one_primary_contact.sql b/tests/integrity_tests/all_students_have_one_primary_contact.sql index ad036257..4941362f 100644 --- a/tests/integrity_tests/all_students_have_one_primary_contact.sql +++ b/tests/integrity_tests/all_students_have_one_primary_contact.sql @@ -14,4 +14,6 @@ with stu_with_multiple_primary_contacts as ( group by tenant_code, school_year, k_student having n_primary_contacts > 1 ) -select * from stu_with_multiple_primary_contacts \ No newline at end of file +select count(*) as failed_row_count, tenant_code, school_year from stu_with_multiple_primary_contacts +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/integrity_tests/enrollments_without_overlapping_sections.sql b/tests/integrity_tests/enrollments_without_overlapping_sections.sql index d17b040b..a71378eb 100644 --- a/tests/integrity_tests/enrollments_without_overlapping_sections.sql +++ b/tests/integrity_tests/enrollments_without_overlapping_sections.sql @@ -22,11 +22,7 @@ with sections_per_enrollment as ( select * from {{ ref('sections_per_enrollment') }} ) - -select k_school, - k_student, - school_year, - entry_date, - n_sections -from sections_per_enrollment +select count(*) as failed_row_count, tenant_code, school_year from sections_per_enrollment where n_sections = 0 +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/integrity_tests/missing_demographics.sql b/tests/integrity_tests/missing_demographics.sql index ce32035b..40316132 100644 --- a/tests/integrity_tests/missing_demographics.sql +++ b/tests/integrity_tests/missing_demographics.sql @@ -13,10 +13,15 @@ with stu_with_enroll as ( from {{ ref('stg_ef3__students') }} stu join {{ ref('stg_ef3__student_school_associations') }} ssa on stu.k_student = ssa.k_student +), +joined as ( + -- of these: which students are not in the dimension + select stu_with_enroll.* + from stu_with_enroll + left join {{ ref('dim_student') }} + on stu_with_enroll.k_student = dim_student.k_student + where dim_student.k_student is null ) --- of these: which students are not in the dimension -select stu_with_enroll.* -from stu_with_enroll -left join {{ ref('dim_student') }} - on stu_with_enroll.k_student = dim_student.k_student -where dim_student.k_student is null \ No newline at end of file +select count(*) as failed_row_count, tenant_code, api_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/integrity_tests/schools_with_enrollments_without_overlapping_sections.sql b/tests/integrity_tests/schools_with_enrollments_without_overlapping_sections.sql index d137359d..ec27ef23 100644 --- a/tests/integrity_tests/schools_with_enrollments_without_overlapping_sections.sql +++ b/tests/integrity_tests/schools_with_enrollments_without_overlapping_sections.sql @@ -22,13 +22,19 @@ populated properly. with sections_per_enrollment as ( select * from {{ ref('sections_per_enrollment') }} -) - -select k_school, +), +final as ( +select + tenant_code, + k_school, school_year, count(1) as n_enrollments, sum(case when n_sections = 0 then 1 else 0 end) as n_enrollments_without_sections, n_enrollments_without_sections / n_enrollments as p_enrollments_without_sections from sections_per_enrollment -group by 1, 2 +group by 1,2,3 having p_enrollments_without_sections > 0 +) +select count(*) as failed_row_count, tenant_code, school_year from final +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/integrity_tests/sections_without_staff.sql b/tests/integrity_tests/sections_without_staff.sql index dc80ae30..455362aa 100644 --- a/tests/integrity_tests/sections_without_staff.sql +++ b/tests/integrity_tests/sections_without_staff.sql @@ -22,10 +22,14 @@ with dim_course_section as ( ), fct_staff_section_association as ( select * from {{ ref("fct_staff_section_association") }} +), +joined as ( + select dim_course_section.* + from dim_course_section + left join fct_staff_section_association + on fct_staff_section_association.k_course_section = dim_course_section.k_course_section + where fct_staff_section_association.k_course_section is null ) - -select dim_course_section.* -from dim_course_section -left join fct_staff_section_association - on fct_staff_section_association.k_course_section = dim_course_section.k_course_section -where fct_staff_section_association.k_course_section is null +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/integrity_tests/sections_without_students.sql b/tests/integrity_tests/sections_without_students.sql index 522144d1..f6228862 100644 --- a/tests/integrity_tests/sections_without_students.sql +++ b/tests/integrity_tests/sections_without_students.sql @@ -20,10 +20,14 @@ with dim_course_section as ( ), fct_student_section_association as ( select * from {{ ref("fct_student_section_association") }} +), +joined as ( + select dim_course_section.* + from dim_course_section + left join fct_student_section_association + on fct_student_section_association.k_course_section = dim_course_section.k_course_section + where fct_student_section_association.k_course_section is null ) - -select dim_course_section.* -from dim_course_section -left join fct_student_section_association - on fct_student_section_association.k_course_section = dim_course_section.k_course_section -where fct_student_section_association.k_course_section is null +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/value_tests/attendance_event_duplicates.sql b/tests/value_tests/attendance_event_duplicates.sql index ac580382..ef839d1e 100644 --- a/tests/value_tests/attendance_event_duplicates.sql +++ b/tests/value_tests/attendance_event_duplicates.sql @@ -29,24 +29,31 @@ dim_school_calendar as ( ), dim_session as ( select * from {{ ref('dim_session') }} +), +joined as ( + select + fct.tenant_code, + fct.k_student, + fct.k_school, + dim_calendar_date.calendar_date, + dim_calendar_date.school_year, + dim_school_calendar.calendar_code, + fct.attendance_event_category, + fct.is_absent, + fct.attendance_event_category = min(fct.attendance_event_category) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as is_preferred_category_by_dedupe, + fct.k_session, + count(*) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_duplicates, + count(distinct fct.attendance_event_category) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_unique_attendance_event_categories, + count(distinct fct.is_absent) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_unique_is_absent_values, + count(distinct fct.k_session) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_unique_sessions + from fct_student_school_attendance_event fct + join dim_calendar_date + on fct.k_calendar_date = dim_calendar_date.k_calendar_date + join dim_school_calendar + on dim_calendar_date.k_school_calendar = dim_school_calendar.k_school_calendar + qualify 1 < count(*) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) + order by fct.k_student, fct.k_school, calendar_date, attendance_event_category ) -select - fct.k_student, - fct.k_school, - dim_calendar_date.calendar_date, - dim_school_calendar.calendar_code, - fct.attendance_event_category, - fct.is_absent, - fct.attendance_event_category = min(fct.attendance_event_category) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as is_preferred_category_by_dedupe, - fct.k_session, - count(*) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_duplicates, - count(distinct fct.attendance_event_category) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_unique_attendance_event_categories, - count(distinct fct.is_absent) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_unique_is_absent_values, - count(distinct fct.k_session) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) as n_unique_sessions -from fct_student_school_attendance_event fct -join dim_calendar_date - on fct.k_calendar_date = dim_calendar_date.k_calendar_date -join dim_school_calendar - on dim_calendar_date.k_school_calendar = dim_school_calendar.k_school_calendar -qualify 1 < count(*) over(partition by fct.k_student, fct.k_school, dim_calendar_date.calendar_date) -order by fct.k_student, fct.k_school, calendar_date, attendance_event_category \ No newline at end of file +select count(*) as failed_row_count, tenant_code, school_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/value_tests/attendance_freshness_test.sql b/tests/value_tests/attendance_freshness_test.sql index 1794fdb0..f12c0fc2 100644 --- a/tests/value_tests/attendance_freshness_test.sql +++ b/tests/value_tests/attendance_freshness_test.sql @@ -18,9 +18,14 @@ or if there are errors pushing data from source system to ODS. severity = 'warn' ) }} -select *, - datediff(DAY, max_date, current_date()) as days_since_last_attendance_event -from {{ ref('attendance_freshness') }} -where datediff(DAY, max_date, current_date()) > 7 --- try to avoid warnings when school is out - and month(current_date()) not in (7, 8) \ No newline at end of file +with filtered as ( + select *, + datediff(DAY, max_date, current_date()) as days_since_last_attendance_event + from {{ ref('attendance_freshness') }} + where datediff(DAY, max_date, current_date()) > 7 + -- try to avoid warnings when school is out + and month(current_date()) not in (7, 8) +) +select count(*) as failed_row_count, tenant_code, school_year from filtered +group by all +having count(*) > 1 diff --git a/tests/value_tests/diploma_record_duplicates.sql b/tests/value_tests/diploma_record_duplicates.sql index 7df74b50..1b7f563b 100644 --- a/tests/value_tests/diploma_record_duplicates.sql +++ b/tests/value_tests/diploma_record_duplicates.sql @@ -57,7 +57,7 @@ count_duplicates as ( on stu_academic_records.academic_term = xwalk_academic_terms.academic_term {% endif %} ) -select * -from count_duplicates +select count(*) as failed_row_count, tenant_code, api_year from count_duplicates where n_duplicates > 1 -order by tenant_code, k_student +group by all +having count(*) > 1 diff --git a/tests/value_tests/invalid_enrollments.sql b/tests/value_tests/invalid_enrollments.sql index 11bf2bd9..5eb92a64 100644 --- a/tests/value_tests/invalid_enrollments.sql +++ b/tests/value_tests/invalid_enrollments.sql @@ -29,25 +29,30 @@ first_school_day as ( select k_school_calendar, min(calendar_date) as calendar_date from {{ ref('dim_calendar_date') }} group by 1 +), +joined as ( + select + exit_withdraw_date < first_school_day.calendar_date as exit_before_first_day, + exit_withdraw_date < entry_date as exit_before_entry, + {% set excl_withdraw_codes = var('edu:enroll:exclude_withdraw_codes') %} + {% if excl_withdraw_codes | length -%} + {% if excl_withdraw_codes is string -%} + {% set excl_withdraw_codes = [excl_withdraw_codes] %} + {%- endif -%} + stg_stu_school.exit_withdraw_type in ( + '{{ excl_withdraw_codes | join("', '") }}' + ) as excluded_withdraw_code, + {% else %} + null as excluded_withdraw_code, + {% endif %} + stg_stu_school.* + from stg_stu_school + left join first_school_day + on stg_stu_school.k_school_calendar = first_school_day.k_school_calendar + where exit_before_first_day + or exit_before_entry + or excluded_withdraw_code ) -select - exit_withdraw_date < first_school_day.calendar_date as exit_before_first_day, - exit_withdraw_date < entry_date as exit_before_entry, - {% set excl_withdraw_codes = var('edu:enroll:exclude_withdraw_codes') %} - {% if excl_withdraw_codes | length -%} - {% if excl_withdraw_codes is string -%} - {% set excl_withdraw_codes = [excl_withdraw_codes] %} - {%- endif -%} - stg_stu_school.exit_withdraw_type in ( - '{{ excl_withdraw_codes | join("', '") }}' - ) as excluded_withdraw_code, - {% else %} - null as excluded_withdraw_code, - {% endif %} - stg_stu_school.* -from stg_stu_school -left join first_school_day - on stg_stu_school.k_school_calendar = first_school_day.k_school_calendar -where exit_before_first_day -or exit_before_entry -or excluded_withdraw_code \ No newline at end of file +select count(*) as failed_row_count, tenant_code, api_year from joined +group by all +having count(*) > 1 \ No newline at end of file diff --git a/tests/value_tests/overlapping_enrollments.sql b/tests/value_tests/overlapping_enrollments.sql index bc20cc2c..0cfd590c 100644 --- a/tests/value_tests/overlapping_enrollments.sql +++ b/tests/value_tests/overlapping_enrollments.sql @@ -31,6 +31,8 @@ fct_student_school_assoc as ( ), counted as ( select + fct_student_school_assoc.tenant_code, + fct_student_school_assoc.school_year, fct_student_school_assoc.k_school, fct_student_school_assoc.k_student, dim_calendar_date.calendar_date, @@ -41,14 +43,14 @@ counted as ( and dim_calendar_date.calendar_date between fct_student_school_assoc.entry_date and coalesce(fct_student_school_assoc.exit_withdraw_date, current_date()) - group by 1,2,3 + group by 1,2,3,4,5 having duplicate_days > 1 ), summarized as ( select - k_school, - k_student, - count(distinct calendar_date) + tenant_code, + school_year, + count(*) as failed_row_count from counted group by 1,2 ) diff --git a/tests/value_tests/student_attendance_total_days.sql b/tests/value_tests/student_attendance_total_days.sql index 90cbd37a..2705b07b 100644 --- a/tests/value_tests/student_attendance_total_days.sql +++ b/tests/value_tests/student_attendance_total_days.sql @@ -29,6 +29,8 @@ in the source system or ODS. with cumulative_attendance as ( select * from {{ ref('msr_student_cumulative_attendance') }} ) -select * +select count(*) as failed_row_count, tenant_code, school_year from cumulative_attendance -where days_enrolled > 185 \ No newline at end of file +where days_enrolled > 185 +group by tenant_code, school_year +having count(*) > 1 \ No newline at end of file