diff --git a/macros/accordion_columns.sql b/macros/accordion_columns.sql index 1ebfb9f9..91a3ddff 100644 --- a/macros/accordion_columns.sql +++ b/macros/accordion_columns.sql @@ -9,8 +9,11 @@ be included in a table. - coalesce_value: if this value is specified, a coalesce function will be added with this value as the second parameter (to be returned if the value in the column is null). If none (default), the coalesce function will not be added. + - add_trailing_comma: if this value is specified, a trailing comma will be added to the last + column. If not passed in, a trailing comma is added for backward compatibility -#} -{% macro accordion_columns(source_table, exclude_columns, source_alias=none, coalesce_value=none) %} + +{% macro accordion_columns(source_table, exclude_columns, source_alias=none, coalesce_value=none, add_trailing_comma=true) %} {%- if source_alias is none -%} {%- set source_alias = source_table -%} {%- endif -%} @@ -20,11 +23,11 @@ be included in a table. ) %} {%- if coalesce_value is none %} {%- for col in keep_cols %} - {{ source_alias }}.{{ col }}, + {{ source_alias }}.{{ col }}{% if not loop.last %},{% elif add_trailing_comma %},{% endif %} {%- endfor %} {%- else -%} {%- for col in keep_cols %} - coalesce({{ source_alias }}.{{ col }}, {{ coalesce_value }}) as {{ col }}, + coalesce({{ source_alias }}.{{ col }}, {{ coalesce_value }}) as {{ col }}{% if not loop.last %},{% elif add_trailing_comma %},{% endif %} {%- endfor %} {%- endif -%} -{% endmacro %} +{% endmacro %} \ No newline at end of file diff --git a/macros/custom_data_sources_macros.sql b/macros/custom_data_sources_macros.sql new file mode 100644 index 00000000..a0c2555f --- /dev/null +++ b/macros/custom_data_sources_macros.sql @@ -0,0 +1,106 @@ +{%- macro cds_depends_on(cds_model_config) -%} + {%- set cds = var(cds_model_config, {}) -%} + {%- if cds is mapping and cds|length -%} + {%- for source_name, _ in cds.items() -%} +-- depends_on: {{ ref(source_name) }} + {%- endfor -%} + {%- endif -%} +{%- endmacro -%} + +{%- macro add_cds_joins_v1(custom_data_sources, driving_alias, join_cols) -%} + {#- + Expected "old" config shape per source: + : { + : { where: }, + ... + } + + Only sources WITHOUT 'joins' key are considered "old" and handled here. + -#} + + {% if custom_data_sources is mapping and custom_data_sources|length %} + {% for source_name, source_config in custom_data_sources|dictsort %} + {% if 'joins' not in source_config %} + left join {{ ref(source_name) }} as {{ source_name }} + {% for col in join_cols %} + {% if loop.first %} + on {{ driving_alias }}.{{ col }} = {{ source_name }}.{{ col }} + {% else %} + and {{ driving_alias }}.{{ col }} = {{ source_name }}.{{ col }} + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {% endif %} +{%- endmacro -%} + +{%- macro add_cds_joins_v2(custom_data_sources) -%} + {#- + Expected "new" config shape per source: + : { + joins: [ "", "", ... ], + cds_additional_cols: { + : { as: , default: }, + ... + } + } + + Only sources WITH 'joins' key are handled here. + -#} + + {% if custom_data_sources is mapping and custom_data_sources|length %} + {% for source_name, source_config in custom_data_sources|dictsort %} + {% if 'joins' in source_config and source_config.joins %} + left join {{ ref(source_name) }} as {{ source_name }} + {% for join in source_config.joins %} + {% if loop.first %} + on {{ join }} + {% else %} + and {{ join }} + {% endif %} + {% endfor %} + {% endif %} + {% endfor %} + {% endif %} +{%- endmacro -%} + +{%- macro add_cds_columns(custom_data_sources) -%} + {#- + Old format per source: + : { + : { where: }, + ... + } + Emits: + , . as + + New format per source: + : { + joins: [...], + cds_additional_cols: { + : { as: , default: }, + ... + } + } + Emits: + , coalesce(., ) as + -#} + + {% if custom_data_sources is mapping and custom_data_sources|length %} + {% for source_name, source_config in custom_data_sources|dictsort %} + {% if 'joins' in source_config %} + {# New config pathway #} + {% if 'cds_additional_cols' in source_config and source_config.cds_additional_cols %} + {% for src_col_name, src_col_config in source_config.cds_additional_cols.items() %} + , coalesce({{ source_name }}.{{ src_col_name }}, {{ src_col_config.default }}) as {{ src_col_config.as }} + {% endfor %} + {% endif %} + {% else %} + {# Old config pathway #} + {% for indicator_name, indicator_config in source_config.items() %} + , {{ source_name }}.{{ indicator_config.where }} as {{ indicator_name }} + {% endfor %} + {% endif %} + {% endfor %} + {% endif %} +{%- endmacro -%} diff --git a/models/build/edfi_3/students/bld_ef3__student__disabilities.sql b/models/build/edfi_3/students/bld_ef3__student__disabilities.sql new file mode 100644 index 00000000..6f096762 --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student__disabilities.sql @@ -0,0 +1,36 @@ +-- Define all optional disability models here. +{% set stage_disability_relations = [] %} + +--Ed Org Disabilities +{% do stage_disability_relations.append(ref('stg_ef3__stu_ed_org__disabilities')) %} + +-- Special Education +{% if var('src:program:special_ed:enabled', True) %} + {% do stage_disability_relations.append(ref('stg_ef3__stu_spec_ed__disabilities')) %} +{% endif %} + +with stacked as ( + {{ dbt_utils.union_relations( + relations=stage_disability_relations + ) }} +), +formatted as ( + select + tenant_code, + api_year, + school_year, + k_student, + ed_org_id, + k_lea, + k_school, + k_program, + program_enroll_begin_date, + program_enroll_end_date, + disability_type, + disability_source_type, + disability_diagnosis, + order_of_disability, + v_designations + from stacked +) +select * from formatted \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql b/models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql new file mode 100644 index 00000000..6f541784 --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql @@ -0,0 +1,45 @@ +with student_disabilities as ( + select * from {{ ref('bld_ef3__student__disabilities') }} +), +xwalk_disability_designations as ( + select * from {{ ref('xwalk_disability_designations') }} +), +flattened as ( + select + tenant_code, + api_year, + school_year, + k_student, + ed_org_id, + k_lea, + k_school, + k_program, + disability_type, + {{ edu_edfi_source.extract_descriptor('designation.value:disabilityDesignationDescriptor::string') }} as disability_designation + from student_disabilities + {{ edu_edfi_source.json_flatten('v_designations', 'designation', outer=true) }} +), +pivoted as ( + select + tenant_code, + api_year, + school_year, + k_student, + ed_org_id, + k_lea, + k_school, + k_program, + disability_type + {%- if not is_empty_model('xwalk_disability_designations') -%}, + {{ ea_pivot( + column='indicator_name', + values=dbt_utils.get_column_values(ref('xwalk_disability_designations'), 'indicator_name'), + cast='boolean', + ) }} + {%- endif %} + from flattened + left outer join xwalk_disability_designations + on flattened.disability_designation = xwalk_disability_designations.disability_designation_descriptor + group by all +) +select * from pivoted \ No newline at end of file diff --git a/models/core_warehouse/dim_assessment.sql b/models/core_warehouse/dim_assessment.sql index 7540686a..57009de6 100644 --- a/models/core_warehouse/dim_assessment.sql +++ b/models/core_warehouse/dim_assessment.sql @@ -7,6 +7,9 @@ ) }} +{{ cds_depends_on('edu:assessment:custom_data_sources') }} +{% set custom_data_sources = var('edu:assessment:custom_data_sources', []) %} + with stg_assessments as ( select * from {{ ref('stg_ef3__assessments') }} ), @@ -42,6 +45,9 @@ formatted as ( assessment_scores.scores_array, assessment_pls.performance_levels_array, assessment_grades.grades_array + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_assessments -- making all of these left joins because none of these are actually required left join assessment_scores @@ -50,6 +56,10 @@ formatted as ( on stg_assessments.k_assessment = assessment_pls.k_assessment left join assessment_grades on stg_assessments.k_assessment = assessment_grades.k_assessment + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_assessments', join_cols=['k_assessment']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_assessment \ No newline at end of file diff --git a/models/core_warehouse/dim_calendar_date.sql b/models/core_warehouse/dim_calendar_date.sql index 2a3bf622..ad73182a 100644 --- a/models/core_warehouse/dim_calendar_date.sql +++ b/models/core_warehouse/dim_calendar_date.sql @@ -8,8 +8,9 @@ ] ) }} -{# Load custom data sources from var #} -{% set custom_data_sources = var("edu:calendar_date:custom_data_sources", []) %} + +{{ cds_depends_on('edu:calendar_date:custom_data_sources') }} +{% set custom_data_sources = var('edu:calendar_date:custom_data_sources', []) %} with stg_calendar_date as ( select * from {{ ref('stg_ef3__calendar_dates') }} @@ -105,24 +106,15 @@ week_calculation as ( else week_of_calendar_year + 52 - start_week_offset end as week_of_school_year - -- custom indicators - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- for indicator in custom_data_sources[source] -%} - , {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }} - {%- endfor -%} - {%- endfor -%} - {%- endif %} + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from augmented join week_offset on augmented.k_school_calendar = week_offset.k_school_calendar + -- custom data sources - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - left join {{ ref(source) }} - on augmented.k_calendar_date = {{ source }}.k_calendar_date - {% endfor %} - {%- endif %} + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='augmented', join_cols=['k_calendar_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from week_calculation order by tenant_code, k_school, calendar_date desc diff --git a/models/core_warehouse/dim_class_period.sql b/models/core_warehouse/dim_class_period.sql index a56095c6..7883b4d9 100644 --- a/models/core_warehouse/dim_class_period.sql +++ b/models/core_warehouse/dim_class_period.sql @@ -6,8 +6,8 @@ ] ) }} -{# Load custom data sources from var #} -{% set custom_data_sources = var("edu:class_period:custom_data_sources", []) %} +{{ cds_depends_on('edu:class_period:custom_data_sources') }} +{% set custom_data_sources = var('edu:class_period:custom_data_sources', []) %} with class_periods as ( select * from {{ ref('stg_ef3__class_periods') }} @@ -45,24 +45,15 @@ formatted as ( end as end_time, timediff(MINUTE, concat('2020-01-01 ', start_time)::timestamp, concat('2020-01-01 ', end_time)::timestamp) as period_duration - -- custom indicators - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- for indicator in custom_data_sources[source] -%} - , {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }} - {%- endfor -%} - {%- endfor -%} - {%- endif %} + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from class_periods join dim_school on class_periods.k_school = dim_school.k_school + -- custom data sources - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - left join {{ ref(source) }} - on class_periods.k_class_period = {{ source }}.k_class_period - {% endfor %} - {%- endif %} + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='class_periods', join_cols=['k_class_period']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select {{ edu_edfi_source.star('formatted', except=['meeting_time']) }} from formatted diff --git a/models/core_warehouse/dim_classroom.sql b/models/core_warehouse/dim_classroom.sql index f4371dfd..24ba56d7 100644 --- a/models/core_warehouse/dim_classroom.sql +++ b/models/core_warehouse/dim_classroom.sql @@ -8,6 +8,9 @@ ) }} +{{ cds_depends_on('edu:classroom:custom_data_sources') }} +{% set custom_data_sources = var('edu:classroom:custom_data_sources', []) %} + with locations as ( select * from {{ ref('stg_ef3__locations') }} ), @@ -22,9 +25,16 @@ formatted as ( locations.classroom_id_code, locations.maximum_seating, locations.optimum_seating + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from locations join dim_school on locations.k_school = dim_school.k_school + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='locations', join_cols=['k_location']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_school, k_classroom \ No newline at end of file diff --git a/models/core_warehouse/dim_cohort.sql b/models/core_warehouse/dim_cohort.sql index ba49d536..96292cb8 100644 --- a/models/core_warehouse/dim_cohort.sql +++ b/models/core_warehouse/dim_cohort.sql @@ -7,6 +7,9 @@ ) }} +{{ cds_depends_on('edu:cohort:custom_data_sources') }} +{% set custom_data_sources = var('edu:cohort:custom_data_sources', []) %} + with stg_cohorts as ( select * from {{ ref('stg_ef3__cohorts') }} ), @@ -23,7 +26,14 @@ formatted as ( stg_cohorts.cohort_description, stg_cohorts.cohort_scope, stg_cohorts.cohort_type + + -- custom data sources_columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_cohorts + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_cohorts', join_cols=['k_cohort']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, school_year desc, k_cohort \ No newline at end of file diff --git a/models/core_warehouse/dim_course.sql b/models/core_warehouse/dim_course.sql index bed782a1..5d96a638 100644 --- a/models/core_warehouse/dim_course.sql +++ b/models/core_warehouse/dim_course.sql @@ -7,8 +7,8 @@ ) }} -{# Load custom data sources from var #} -{% set custom_data_sources = var("edu:course:custom_data_sources", []) %} +{{ cds_depends_on('edu:course:custom_data_sources') }} +{% set custom_data_sources = var('edu:course:custom_data_sources', []) %} with stg_course as ( select * from {{ ref('stg_ef3__courses') }} @@ -49,18 +49,9 @@ formatted as ( stg_course.minimum_available_credit_conversion, stg_course.number_of_parts, stg_course.time_required_for_completion, - - -- custom indicators - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- for indicator in custom_data_sources[source] -%} - {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }}, - {%- endfor -%} - {%- endfor -%} - {%- endif %} - bld_ef3__course_subject.subject_array - + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_course left join bld_ef3__wide_ids_course @@ -69,12 +60,8 @@ formatted as ( on stg_course.k_course = bld_ef3__course_subject.k_course -- custom data sources - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - left join {{ ref(source) }} - on stg_course.k_course = {{ source }}.k_course - {% endfor %} - {%- endif %} + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_course', join_cols=['k_course']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, school_year desc, k_course diff --git a/models/core_warehouse/dim_course_section.sql b/models/core_warehouse/dim_course_section.sql index 05395c4e..3c5ccfc4 100644 --- a/models/core_warehouse/dim_course_section.sql +++ b/models/core_warehouse/dim_course_section.sql @@ -10,8 +10,8 @@ ) }} -{# Load custom data sources from var #} -{% set custom_data_sources = var("edu:course_section:custom_data_sources", []) %} +{{ cds_depends_on('edu:course_section:custom_data_sources') }} +{% set custom_data_sources = var('edu:course_section:custom_data_sources', []) %} with offering as ( select * from {{ ref('stg_ef3__course_offerings') }} @@ -63,16 +63,10 @@ joined as ( stg_ef3__sections.available_credits, stg_ef3__sections.available_credit_type, stg_ef3__sections.available_credit_conversion - + -- todo: add characteristic indicators - -- custom indicators - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- for indicator in custom_data_sources[source] -%} - , {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }} - {%- endfor -%} - {%- endfor -%} - {%- endif %} + -- custom data sources_columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_ef3__sections join offering on stg_ef3__sections.k_course_offering = offering.k_course_offering @@ -80,15 +74,10 @@ joined as ( on offering.k_course = dim_course.k_course left join section_chars on stg_ef3__sections.k_course_section = section_chars.k_course_section + -- custom data sources - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- if source != 'stg_ef3__sections' -%} - left join {{ ref(source) }} - on stg_ef3__sections.k_course_section = {{ source }}.k_course_section - {%- endif -%} - {% endfor %} - {%- endif %} + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_ef3__sections', join_cols=['k_course_section']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from joined order by tenant_code, k_school, k_course_section diff --git a/models/core_warehouse/dim_discipline_incident.sql b/models/core_warehouse/dim_discipline_incident.sql index 178e7259..52169eb4 100644 --- a/models/core_warehouse/dim_discipline_incident.sql +++ b/models/core_warehouse/dim_discipline_incident.sql @@ -8,6 +8,9 @@ ) }} +{{ cds_depends_on('edu:discipline_incident:custom_data_sources') }} +{% set custom_data_sources = var('edu:discipline_incident:custom_data_sources', []) %} + with stg_discipline_incidents as ( select * from {{ ref('stg_ef3__discipline_incidents') }} ), @@ -51,11 +54,18 @@ formatted as ( stg_discipline_incidents.reporter_description, stg_discipline_incidents.incident_location, behaviors.behavior_array + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_discipline_incidents -- behaviors are not required left join behaviors on stg_discipline_incidents.k_discipline_incident = behaviors.k_discipline_incident join dim_school on stg_discipline_incidents.k_school = dim_school.k_school + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_discipline_incidents', join_cols=['k_discipline_incident']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/dim_esc.sql b/models/core_warehouse/dim_esc.sql index 26022b8d..83d65d15 100644 --- a/models/core_warehouse/dim_esc.sql +++ b/models/core_warehouse/dim_esc.sql @@ -7,6 +7,9 @@ ) }} +{{ cds_depends_on('edu:esc:custom_data_sources') }} +{% set custom_data_sources = var('edu:esc:custom_data_sources', []) %} + with stg_esc as ( select * from {{ ref('stg_ef3__education_service_centers') }} ), @@ -40,9 +43,16 @@ formatted as ( choose_address.county_fips_code, choose_address.latitude, choose_address.longitude + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_esc left join choose_address on stg_esc.k_esc = choose_address.k_esc + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_esc', join_cols=['k_esc']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_esc \ No newline at end of file diff --git a/models/core_warehouse/dim_grading_period.sql b/models/core_warehouse/dim_grading_period.sql index 78beabe9..f164fa23 100644 --- a/models/core_warehouse/dim_grading_period.sql +++ b/models/core_warehouse/dim_grading_period.sql @@ -8,6 +8,8 @@ ) }} +{{ cds_depends_on('edu:grading_period:custom_data_sources') }} +{% set custom_data_sources = var('edu:grading_period:custom_data_sources', []) %} with stg_grading_periods as ( select * from {{ ref('stg_ef3__grading_periods') }} @@ -26,9 +28,16 @@ formatted as ( stg_grading_periods.begin_date, stg_grading_periods.end_date, stg_grading_periods.total_instructional_days + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_grading_periods join dim_school on stg_grading_periods.k_school = dim_school.k_school + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_grading_periods', join_cols=['k_grading_period']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_school, k_grading_period \ No newline at end of file diff --git a/models/core_warehouse/dim_graduation_plan.sql b/models/core_warehouse/dim_graduation_plan.sql index 05a38b53..ade4db11 100644 --- a/models/core_warehouse/dim_graduation_plan.sql +++ b/models/core_warehouse/dim_graduation_plan.sql @@ -7,6 +7,8 @@ ) }} +{{ cds_depends_on('edu:graduation_plan:custom_data_sources') }} +{% set custom_data_sources = var('edu:graduation_plan:custom_data_sources', []) %} with stg_graduation_plans as ( select * from {{ ref('stg_ef3__graduation_plans') }} @@ -33,8 +35,13 @@ formatted as ( {{ edu_edfi_source.extract_extension(model_name='stg_ef3__graduation_plans', flatten=False) }} - + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_graduation_plans + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_graduation_plans', join_cols=['k_graduation_plan']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_graduation_plan \ No newline at end of file diff --git a/models/core_warehouse/dim_lea.sql b/models/core_warehouse/dim_lea.sql index 7e627471..33779fac 100644 --- a/models/core_warehouse/dim_lea.sql +++ b/models/core_warehouse/dim_lea.sql @@ -7,6 +7,9 @@ ) }} +{{ cds_depends_on('edu:lea:custom_data_sources') }} +{% set custom_data_sources = var('edu:lea:custom_data_sources', []) %} + with stg_lea as ( select * from {{ ref('stg_ef3__local_education_agencies') }} ), @@ -50,12 +53,19 @@ formatted as ( choose_address.county_fips_code, choose_address.latitude, choose_address.longitude + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_lea left join choose_address on stg_lea.k_lea = choose_address.k_lea join tenant_lea_ownership on stg_lea.tenant_code = tenant_lea_ownership.tenant_code - and stg_lea.lea_id = tenant_lea_ownership.lea_id + and stg_lea.lea_id = cast(tenant_lea_ownership.lea_id as int) + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_lea', join_cols=['k_lea']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_lea diff --git a/models/core_warehouse/dim_learning_standard.sql b/models/core_warehouse/dim_learning_standard.sql index e5351973..5b7ca0ee 100644 --- a/models/core_warehouse/dim_learning_standard.sql +++ b/models/core_warehouse/dim_learning_standard.sql @@ -7,6 +7,8 @@ ) }} +{{ cds_depends_on('edu:learning_standard:custom_data_sources') }} +{% set custom_data_sources = var('edu:learning_standard:custom_data_sources', []) %} with stg_learning_standards as ( select * from {{ ref('stg_ef3__learning_standards') }} @@ -30,7 +32,14 @@ formatted as ( stg_learning_standards.v_grade_levels, stg_learning_standards.v_learning_standard_identification_codes, stg_learning_standards.v_content_standard + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_learning_standards + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_learning_standards', join_cols=['k_learning_standard']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_learning_standard \ No newline at end of file diff --git a/models/core_warehouse/dim_network.sql b/models/core_warehouse/dim_network.sql index c9f31141..528b98e5 100644 --- a/models/core_warehouse/dim_network.sql +++ b/models/core_warehouse/dim_network.sql @@ -7,6 +7,8 @@ ) }} +{{ cds_depends_on('edu:network:custom_data_sources') }} +{% set custom_data_sources = var('edu:network:custom_data_sources', []) %} with stg_networks as ( select * from {{ ref('stg_ef3__education_organization_networks') }} @@ -21,7 +23,14 @@ formatted as ( stg_networks.operational_status, stg_networks.network_short_name, stg_networks.website + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_networks + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_networks', join_cols=['k_network']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_network \ No newline at end of file diff --git a/models/core_warehouse/dim_objective_assessment.sql b/models/core_warehouse/dim_objective_assessment.sql index a293b556..5464ef1a 100644 --- a/models/core_warehouse/dim_objective_assessment.sql +++ b/models/core_warehouse/dim_objective_assessment.sql @@ -8,6 +8,9 @@ ) }} +{{ cds_depends_on('edu:objective_assessment:custom_data_sources') }} +{% set custom_data_sources = var('edu:objective_assessment:custom_data_sources', []) %} + with stg_obj_assessments as ( select * from {{ ref('stg_ef3__objective_assessments') }} ), @@ -33,12 +36,19 @@ formatted as ( stg_obj_assessments.academic_subject, obj_assessment_scores.scores_array, obj_assessment_pls.performance_levels_array + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_obj_assessments -- making all of these left joins because none of these are actually required left join obj_assessment_scores on stg_obj_assessments.k_objective_assessment = obj_assessment_scores.k_objective_assessment left join obj_assessment_pls on stg_obj_assessments.k_objective_assessment = obj_assessment_pls.k_objective_assessment + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_obj_assessments', join_cols=['k_objective_assessment']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_objective_assessment \ No newline at end of file diff --git a/models/core_warehouse/dim_parent.sql b/models/core_warehouse/dim_parent.sql index eb5f1e25..a380ffae 100644 --- a/models/core_warehouse/dim_parent.sql +++ b/models/core_warehouse/dim_parent.sql @@ -7,6 +7,9 @@ ) }} +{{ cds_depends_on('edu:parent:custom_data_sources') }} +{% set custom_data_sources = var('edu:parent:custom_data_sources', []) %} + with stg_parent as ( -- parents were renamed to contacts in Data Standard v5.0 -- the contacts staging tables contain both parent and contact records @@ -57,6 +60,9 @@ formatted as ( source_alias='parent_emails_wide' ) }} choose_address.full_address + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_parent left join parent_phones_wide on stg_parent.k_contact = parent_phones_wide.k_parent --k_contact has been renamed back to k_parent in the build models @@ -64,6 +70,10 @@ formatted as ( on stg_parent.k_contact = parent_emails_wide.k_parent left join choose_address on stg_parent.k_contact = choose_address.k_contact + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_parent', join_cols=['k_contact']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_parent \ No newline at end of file diff --git a/models/core_warehouse/dim_program.sql b/models/core_warehouse/dim_program.sql index da107f9b..f3899c2c 100644 --- a/models/core_warehouse/dim_program.sql +++ b/models/core_warehouse/dim_program.sql @@ -7,6 +7,9 @@ ) }} +{{ cds_depends_on('edu:program:custom_data_sources') }} +{% set custom_data_sources = var('edu:program:custom_data_sources', []) %} + with stg_programs as ( select * from {{ ref('stg_ef3__programs') }} ), @@ -23,7 +26,14 @@ formatted as ( stg_programs.program_id, stg_programs.program_name, stg_programs.program_type + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_programs + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_programs', join_cols=['k_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/dim_school.sql b/models/core_warehouse/dim_school.sql index 1d5b14fa..002ee3b6 100644 --- a/models/core_warehouse/dim_school.sql +++ b/models/core_warehouse/dim_school.sql @@ -20,8 +20,9 @@ ] ) }} -{# Load custom data sources from var #} -{% set custom_data_sources = var("edu:schools:custom_data_sources", []) %} + +{{ cds_depends_on('edu:schools:custom_data_sources') }} +{% set custom_data_sources = var('edu:schools:custom_data_sources', []) %} with stg_school as ( select * from {{ ref('stg_ef3__schools') }} @@ -98,14 +99,8 @@ formatted as ( choose_address.latitude, choose_address.longitude - -- custom data sources - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- for indicator in custom_data_sources[source] -%} - , {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }} - {%- endfor -%} - {%- endfor -%} - {%- endif %} + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_school join dim_lea on stg_school.k_lea = dim_lea.k_lea @@ -115,13 +110,10 @@ formatted as ( on stg_school.k_school = choose_address.k_school left join bld_network_associations on stg_school.k_school = bld_network_associations.k_school + -- custom data sources - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - left join {{ ref(source) }} - on stg_school.k_school = {{ source }}.k_school - {% endfor %} - {%- endif %} + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_school', join_cols=['k_school']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_school diff --git a/models/core_warehouse/dim_school_calendar.sql b/models/core_warehouse/dim_school_calendar.sql index 08794f54..a81d7bf6 100644 --- a/models/core_warehouse/dim_school_calendar.sql +++ b/models/core_warehouse/dim_school_calendar.sql @@ -8,6 +8,9 @@ ) }} +{{ cds_depends_on('edu:school_calendar:custom_data_sources') }} +{% set custom_data_sources = var('edu:school_calendar:custom_data_sources', []) %} + with stg_calendar as ( select * from {{ ref('stg_ef3__calendars') }} ), @@ -33,11 +36,18 @@ formatted as ( stg_calendar.calendar_code, stg_calendar.calendar_type, aggregated_grades.applicable_grade_levels_array + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_calendar join dim_school on stg_calendar.k_school = dim_school.k_school left join aggregated_grades on stg_calendar.k_school_calendar = aggregated_grades.k_school_calendar + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_calendar', join_cols=['k_school_calendar']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_school, k_school_calendar \ No newline at end of file diff --git a/models/core_warehouse/dim_session.sql b/models/core_warehouse/dim_session.sql index 9b33866f..5f36f313 100644 --- a/models/core_warehouse/dim_session.sql +++ b/models/core_warehouse/dim_session.sql @@ -8,6 +8,9 @@ ) }} +{{ cds_depends_on('edu:session:custom_data_sources') }} +{% set custom_data_sources = var('edu:session:custom_data_sources', []) %} + with stg_sessions as ( select * from {{ ref('stg_ef3__sessions') }} ), @@ -25,9 +28,16 @@ formatted as ( stg_sessions.session_end_date, stg_sessions.total_instructional_days, stg_sessions.academic_term + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_sessions join dim_school on stg_sessions.k_school = dim_school.k_school + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_sessions', join_cols=['k_session']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_school, session_begin_date desc \ No newline at end of file diff --git a/models/core_warehouse/dim_staff.sql b/models/core_warehouse/dim_staff.sql index 7352820d..1e1cf401 100644 --- a/models/core_warehouse/dim_staff.sql +++ b/models/core_warehouse/dim_staff.sql @@ -6,8 +6,9 @@ ] ) }} -{# Load custom data sources from var #} -{% set custom_data_sources = var("edu:staff:custom_data_sources", []) %} + +{{ cds_depends_on('edu:staff:custom_data_sources') }} +{% set custom_data_sources = var('edu:staff:custom_data_sources', []) %} with stg_staff as ( select * from {{ ref('stg_ef3__staffs') }} @@ -56,14 +57,8 @@ formatted as ( stg_staff.years_of_prior_professional_experience, stg_staff.years_of_prior_teaching_experience - -- custom indicators - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- for indicator in custom_data_sources[source] -%} - , {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }} - {%- endfor -%} - {%- endfor -%} - {%- endif %} + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_staff left join bld_ef3__wide_ids_staff on stg_staff.k_staff = bld_ef3__wide_ids_staff.k_staff @@ -71,13 +66,10 @@ formatted as ( on stg_staff.k_staff = choose_email.k_staff left join staff_race_ethnicity on stg_staff.k_staff = staff_race_ethnicity.k_staff + -- custom data sources - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - left join {{ ref(source) }} - on stg_staff.k_staff = {{ source }}.k_staff - {% endfor %} - {%- endif %} + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_staff', join_cols=['k_staff']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_staff diff --git a/models/core_warehouse/dim_student.sql b/models/core_warehouse/dim_student.sql index b1cb258b..d2343c34 100644 --- a/models/core_warehouse/dim_student.sql +++ b/models/core_warehouse/dim_student.sql @@ -7,9 +7,8 @@ ) }} -{# Load custom data sources from var #} -{% set custom_data_sources = var("edu:stu_demos:custom_data_sources") %} - +{{ cds_depends_on('edu:stu_demos:custom_data_sources') }} +{% set custom_data_sources = var('edu:stu_demos:custom_data_sources', []) %} {# If edu var has been configured to make demos immutable, set join var to `k_student_xyear` bc demos are unique by xyear #} {# otherwise, use k_student bc demos are unique by student+year #} @@ -241,15 +240,6 @@ formatted as ( {%- endfor -%} {%- endif %} - -- custom indicators - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - {%- for indicator in custom_data_sources[source] -%} - {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }}, - {%- endfor -%} - {%- endfor -%} - {%- endif %} - -- other name types {% if other_name_types is not none and other_name_types | length -%} {%- for type in other_name_types -%} @@ -265,6 +255,9 @@ formatted as ( stu_cohort_year.cohort_year_array, stu_immutable_demos.safe_display_name + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} + from stg_student join stu_demos @@ -330,13 +323,8 @@ formatted as ( -- custom data sources -- Note, dbt test "custom_demo_sources_are_unique_on_k_student" is configured to fail if any not unique by k_student - {% if custom_data_sources is not none and custom_data_sources | length -%} - {%- for source in custom_data_sources -%} - left join {{ ref(source) }} - on stu_demos.k_student = {{ source }}.k_student - {% endfor %} - {%- endif %} - + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stu_demos', join_cols=['k_student']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/dim_subgroup.sql b/models/core_warehouse/dim_subgroup.sql index 53560306..4af55da6 100644 --- a/models/core_warehouse/dim_subgroup.sql +++ b/models/core_warehouse/dim_subgroup.sql @@ -8,6 +8,9 @@ ) }} +{{ cds_depends_on('edu:subgroup:custom_data_sources') }} +{% set custom_data_sources = var('edu:subgroup:custom_data_sources', []) %} + with dim_student as ( select * from {{ ref('dim_student') }} ), @@ -90,7 +93,14 @@ keyed as ( joined_with_display_names.subgroup_category_display_name, joined_with_display_names.subgroup_value, joined_with_display_names.subgroup_value_display_name + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from joined_with_display_names + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='joined_with_display_names', join_cols=['subgroup_category', 'subgroup_value']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from keyed diff --git a/models/core_warehouse/fct_course_transcripts.sql b/models/core_warehouse/fct_course_transcripts.sql index 50da54a0..9caa6cd6 100644 --- a/models/core_warehouse/fct_course_transcripts.sql +++ b/models/core_warehouse/fct_course_transcripts.sql @@ -12,6 +12,9 @@ ) }} +{{ cds_depends_on('edu:course_transcripts:custom_data_sources') }} +{% set custom_data_sources = var('edu:course_transcripts:custom_data_sources', []) %} + with course_transcripts as ( select * from {{ ref('stg_ef3__course_transcripts') }} ), @@ -53,8 +56,15 @@ formatted as ( course_transcripts.v_credit_categories {# add any extension columns configured from stg_ef3__course_transcripts #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__course_transcripts', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from course_transcripts join fct_student_academic_record on course_transcripts.k_student_academic_record = fct_student_academic_record.k_student_academic_record + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='course_transcripts', join_cols=['k_course', 'k_student_academic_record', 'course_attempt_result']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_staff_school_association.sql b/models/core_warehouse/fct_staff_school_association.sql index d37cf0fd..850400dd 100644 --- a/models/core_warehouse/fct_staff_school_association.sql +++ b/models/core_warehouse/fct_staff_school_association.sql @@ -9,6 +9,9 @@ ) }} +{{ cds_depends_on('edu:staff_school_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:staff_school_association:custom_data_sources', []) %} + with stg_staff_school as ( select * from {{ ref('stg_ef3__staff_school_associations') }} ), @@ -104,5 +107,17 @@ check_active as ( true, false ) as is_active_assignment from formatted +), +cds_cols as ( + select + ca.* + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} + from check_active ca + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='ca', join_cols=['k_staff_school_association']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) -select * from check_active \ No newline at end of file +select * from cds_cols \ No newline at end of file diff --git a/models/core_warehouse/fct_staff_section_association.sql b/models/core_warehouse/fct_staff_section_association.sql index 9a8d2be8..623e4ceb 100644 --- a/models/core_warehouse/fct_staff_section_association.sql +++ b/models/core_warehouse/fct_staff_section_association.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:staff_section_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:staff_section_association:custom_data_sources', []) %} + with stg_staff_section as ( select * from {{ ref('stg_ef3__staff_section_associations') }} ), @@ -47,10 +50,17 @@ formatted as ( ) as is_active_assignment {# add any extension columns configured from stg_ef3__staff_section_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__staff_section_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_staff_section join dim_staff on stg_staff_section.k_staff = dim_staff.k_staff join dim_course_section on stg_staff_section.k_course_section = dim_course_section.k_course_section + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_staff_section', join_cols=['k_staff', 'k_course_section', 'begin_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_academic_record.sql b/models/core_warehouse/fct_student_academic_record.sql index 52d295f9..a2cd1e48 100644 --- a/models/core_warehouse/fct_student_academic_record.sql +++ b/models/core_warehouse/fct_student_academic_record.sql @@ -8,6 +8,9 @@ ) }} +{{ cds_depends_on('edu:student_academic_record:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_academic_record:custom_data_sources', []) %} + with stg_academic_record as ( select * from {{ ref('stg_ef3__student_academic_records') }} ), @@ -46,11 +49,18 @@ formatted as ( stg_academic_record.session_attempted_credit_conversion {# add any extension columns configured from stg_ef3__student_academic_records #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_academic_records', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_academic_record left join dim_school on stg_academic_record.k_school = dim_school.k_school left join dim_student on stg_academic_record.k_student_xyear = dim_student.k_student_xyear and stg_academic_record.school_year = dim_student.school_year + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_academic_record', join_cols=['k_student_academic_record']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_assessment.sql b/models/core_warehouse/fct_student_assessment.sql index c4204cd4..2fcad6a1 100644 --- a/models/core_warehouse/fct_student_assessment.sql +++ b/models/core_warehouse/fct_student_assessment.sql @@ -9,6 +9,9 @@ ) }} +{{ cds_depends_on('edu:student_assessment:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_assessment:custom_data_sources', []) %} + with student_assessments_long_results as ( select * from {{ ref('bld_ef3__student_assessments_long_results') }} ), @@ -93,6 +96,12 @@ student_assessments_wide as ( select student_assessments_wide.*, v_other_results + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} + from student_assessments_wide left join object_agg_other_results on student_assessments_wide.k_student_assessment = object_agg_other_results.k_student_assessment +-- custom data sources +{{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='student_assessments_wide', join_cols=['k_student_assessment']) }} +{{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} diff --git a/models/core_warehouse/fct_student_cohort_association.sql b/models/core_warehouse/fct_student_cohort_association.sql index 44a7a469..b701ce13 100644 --- a/models/core_warehouse/fct_student_cohort_association.sql +++ b/models/core_warehouse/fct_student_cohort_association.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:student_cohort_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_cohort_association:custom_data_sources', []) %} + with stage as ( select * from {{ ref('stg_ef3__student_cohort_associations') }} ), @@ -48,13 +51,18 @@ formatted as ( ) as is_active_cohort_association {# add any extension columns configured from stg_ef3__student_cohort_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_cohort_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage - inner join dim_student on stage.k_student = dim_student.k_student - inner join dim_cohort on stage.k_cohort = dim_cohort.k_cohort + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student', 'k_cohort', 'cohort_begin_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_cte_program_associations.sql b/models/core_warehouse/fct_student_cte_program_associations.sql index 51bfc257..b1132d80 100644 --- a/models/core_warehouse/fct_student_cte_program_associations.sql +++ b/models/core_warehouse/fct_student_cte_program_associations.sql @@ -2,17 +2,22 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}" ] ) }} +{{ cds_depends_on('edu:student_cte_program_associations:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_cte_program_associations:custom_data_sources', []) %} + with stage as ( select * from {{ ref('stg_ef3__student_cte_program_associations') }} ), @@ -27,11 +32,14 @@ dim_program as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, + stage.ed_org_id, + stage.tenant_code, dim_program.school_year, stage.program_enroll_begin_date, @@ -44,6 +52,9 @@ formatted as ( stage.reason_exited {# add any extension columns configured from stg_ef3__student_cte_program_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_cte_program_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -51,6 +62,10 @@ formatted as ( inner join dim_program on stage.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_cte_program_associations.yml b/models/core_warehouse/fct_student_cte_program_associations.yml index 798f72e2..cfa5c939 100644 --- a/models/core_warehouse/fct_student_cte_program_associations.yml +++ b/models/core_warehouse/fct_student_cte_program_associations.yml @@ -7,8 +7,7 @@ models: This fact table contains student cte program enrollment information. ##### Primary Key: - `k_student, k_student_xyear, k_program, program_enroll_begin_date` -- There - is one record per student, year, program enrol begin date, and cte program. + `k_student_program` -- There is one record per student, year, ed_org_id, program enroll begin date, and cte program. config: tags: ['cte'] @@ -20,13 +19,17 @@ models: combination_of_columns: - k_student - k_program + - ed_org_id + - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/fct_student_daily_attendance.sql b/models/core_warehouse/fct_student_daily_attendance.sql index 47caea6c..bc356d23 100644 --- a/models/core_warehouse/fct_student_daily_attendance.sql +++ b/models/core_warehouse/fct_student_daily_attendance.sql @@ -12,6 +12,9 @@ ) }} +{{ cds_depends_on('edu:student_daily_attendance:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_daily_attendance:custom_data_sources', []) %} + with fct_student_school_att as ( select * from {{ ref(var("edu:attendance:daily_attendance_source", 'fct_student_school_attendance_event')) }} ), @@ -231,7 +234,7 @@ cumulatives as ( cumulative_days_enrolled >= {{ var('edu:attendance:chronic_absence_min_days') }} as meets_enrollment_threshold, {{ msr_chronic_absentee('cumulative_attendance_rate', 'cumulative_days_enrolled') }} as is_chronic_absentee, excusal_status_streaks.event_duration, - excusal_status_streaks.school_attendance_duration, + excusal_status_streaks.school_attendance_duration from excusal_status_streaks ), metric_labels as ( @@ -245,10 +248,17 @@ metric_labels as ( when meets_enrollment_threshold then metric_absentee_categories.level_label else null end as absentee_category_label + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from cumulatives left join metric_absentee_categories on cumulative_attendance_rate > metric_absentee_categories.threshold_lower and cumulative_attendance_rate <= metric_absentee_categories.threshold_upper + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='cumulatives', join_cols=['k_student', 'k_school', 'calendar_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from metric_labels diff --git a/models/core_warehouse/fct_student_diploma.sql b/models/core_warehouse/fct_student_diploma.sql index b086a140..765ab711 100644 --- a/models/core_warehouse/fct_student_diploma.sql +++ b/models/core_warehouse/fct_student_diploma.sql @@ -1,5 +1,8 @@ {% set xwalk_academic_term_enabled = var('edu:xwalk_academic_terms:enabled', False) %} +{{ cds_depends_on('edu:student_diploma:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_diploma:custom_data_sources', []) %} + with stg_diplomas as ( select * from {{ ref('stg_ef3__student_academic_records__diplomas') }} ), @@ -53,11 +56,18 @@ joined_with_xwalk as ( {% else %} NULL as sort_index {% endif %} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from joined_diploma {% if xwalk_academic_term_enabled %} left join xwalk_academic_terms on joined_diploma.academic_term = xwalk_academic_terms.academic_term {% endif %} + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='joined_diploma', join_cols=['k_student', 'school_year', 'k_lea', 'k_school', 'diploma_type', 'diploma_award_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ), dedupe_diplomas as ( {{ diff --git a/models/core_warehouse/fct_student_disability.sql b/models/core_warehouse/fct_student_disability.sql new file mode 100644 index 00000000..d94b651d --- /dev/null +++ b/models/core_warehouse/fct_student_disability.sql @@ -0,0 +1,87 @@ +{{ + config( + post_hook=[ + "alter table {{ this }} alter column k_student_disability set not null", + "alter table {{ this }} alter column k_student set not null", + "alter table {{ this }} alter column disability_type set not null", + "alter table {{ this }} add primary key (k_student_disability)", + "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", + "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", + "alter table {{ this }} add constraint fk_{{ this.name }}_lea foreign key (k_lea) references {{ ref('dim_lea') }}", + "alter table {{ this }} add constraint fk_{{ this.name }}_school foreign key (k_school) references {{ ref('dim_school') }}", + ] + ) +}} + +{{ cds_depends_on('edu:student_disability:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_disability:custom_data_sources', []) %} + +with student_disabilities as ( + select * from {{ ref('bld_ef3__student__disabilities') }} +), +dim_student as ( + select * from {{ ref('dim_student') }} +), +student_disability_designations as ( + select * from {{ ref('bld_ef3__student__wide_disability_designations') }} +), +formatted as ( + select + {{ dbt_utils.generate_surrogate_key( + ['sd.tenant_code', + 'sd.school_year', + 'sd.k_student', + 'sd.k_lea', + 'sd.k_school', + 'sd.k_program', + 'sd.ed_org_id', + 'sd.program_enroll_begin_date',] + ) }} as k_student_disability, + sd.k_student, + stu.k_student_xyear, + sd.k_lea, + sd.k_school, + sd.k_program, + sd.ed_org_id, + sd.program_enroll_begin_date, + sd.program_enroll_end_date, + sd.tenant_code, + sd.api_year, + sd.school_year, + sd.disability_type, + sd.disability_source_type, + sd.disability_diagnosis, + sd.order_of_disability, + -- disability designations + {{ accordion_columns( + source_table='bld_ef3__student__wide_disability_designations', + exclude_columns=['tenant_code', 'api_year', 'school_year', 'k_student', 'ed_org_id', 'k_lea', 'k_school', 'k_program', 'disability_type'], + source_alias='disability_designations', + add_trailing_comma=false + ) }} + from student_disabilities sd + join dim_student stu + on stu.k_student = sd.k_student + left join student_disability_designations disability_designations + on sd.k_student = disability_designations.k_student + and (sd.k_lea = disability_designations.k_lea or (sd.k_lea is null and disability_designations.k_lea is null)) + and (sd.k_school = disability_designations.k_school or (sd.k_school is null and disability_designations.k_school is null)) + and (sd.k_program = disability_designations.k_program or (sd.k_program is null and disability_designations.k_program is null)) + and sd.ed_org_id = disability_designations.ed_org_id + and sd.tenant_code = disability_designations.tenant_code + and sd.school_year = disability_designations.school_year + and sd.disability_type = disability_designations.disability_type +), +cds_cols as ( + select + f.* + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} + from formatted f + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='f', join_cols=['k_student_disability']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} +) +select * from cds_cols \ No newline at end of file diff --git a/models/core_warehouse/fct_student_disability.yml b/models/core_warehouse/fct_student_disability.yml new file mode 100644 index 00000000..02b2500a --- /dev/null +++ b/models/core_warehouse/fct_student_disability.yml @@ -0,0 +1,86 @@ +version: 3 + +models: + - name: fct_student_disability + description: > + ##### Overview: + This fact table provides student disabilities, which could be assigned from multiple sources. + It references any or all of the following models: + - [stg_ef3__stu_ed_org__disabilities](#!/model/model.edu_edfi_source.stg_ef3__stu_ed_org__disabilities) + - [stg_ef3__stu_spec_ed__disabilities](#!/model/model.edu_edfi_source.stg_ef3__stu_spec_ed__disabilities) + + ##### Primary Key: + `k_student_disability` -- + There is one record per student, year, ed org / program + + ##### Important business rules: + - This table is at two different grains since both Ed Orgs and Special Ed Programs can have disabilities listed. This is why this table has + a surrogate key being made up of the two grains that fit in this table. If k_program is null then this is an ed org disability. + If k_program is not null then it is a program disability. + - `program_enroll_begin_date` is included in the unique key, because a student may be associated with the same program at multiple times, + and those associations may have different disabilities. When joining this table to `fct_student_{program_type}_program_association`, + always include `program_enroll_begin_date` in the join (see example query below). + + ##### Example Use Cases: + 1. Join Disabilities to Student Special Education data to find students' Special Ed disabilities + they received as part of that program: + + ``` + SELECT + assoc.k_student, + assoc.school_year, + assoc.k_program, + dim_program.program_type, + dim_program.program_id, + dim_program.program_name, + assoc.program_enroll_begin_date, + assoc.program_enroll_end_date, + dis.disability_type, + dis.disability_source_type, + dis.disability_diagnosis, + dis.order_of_disability + FROM analytics.prod_wh.fct_student_special_education_program_association assoc + JOIN analytics.prod_wh.dim_program + ON assoc.k_program = dim_program.k_program + -- left join because some programs may not have disabilities + LEFT join analytics.prod_wh.fct_student_disability dis + ON assoc.k_student = dis.k_student + AND assoc.k_program = dis.k_program + AND assoc.program_enroll_begin_date = dis.program_enroll_begin_date; + ``` + + + config: + tags: ['special_ed'] + enabled: > + {%- set var_values = [] -%} + {%- for variable in [ + 'src:program:special_ed:enabled', + ] -%} + {%- do var_values.append(var(variable, none)) -%} + {%- endfor -%} + + {%- if True in var_values -%} + {{ True | as_bool }} + {%- elif False in var_values -%} + {{ False | as_bool }} + {%- else -%} + {{ True | as_bool }} + {%- endif -%} + + columns: + - name: k_student_disability, + - name: k_student + - name: k_student_xyear + - name: k_lea + - name: k_school + - name: k_program + - name: program_enroll_begin_date + - name: program_enroll_end_date + - name: tenant_code + - name: api_year + - name: school_year + - name: disability_type + - name: disability_source_type + - name: disability_diagnosis + - name: order_of_disability \ No newline at end of file diff --git a/models/core_warehouse/fct_student_discipline_actions.sql b/models/core_warehouse/fct_student_discipline_actions.sql index 90456739..5cd50b96 100644 --- a/models/core_warehouse/fct_student_discipline_actions.sql +++ b/models/core_warehouse/fct_student_discipline_actions.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:discipline_actions:custom_data_sources') }} +{% set custom_data_sources = var('edu:discipline_actions:custom_data_sources', []) %} + with stg_discipline_actions as ( select * from {{ ref('stg_ef3__discipline_actions') }} ), @@ -83,6 +86,9 @@ formatted as ( bld_discipline_incident_associations.k_student_discipline_incident_behavior_array {# add any extension columns configured from stg_ef3__discipline_actions #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__discipline_actions', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_discipline_actions join dim_student on stg_discipline_actions.k_student = dim_student.k_student @@ -100,6 +106,11 @@ formatted as ( on stg_discipline_actions.k_student = agg_staff_keys.k_student and stg_discipline_actions.discipline_action_id = agg_staff_keys.discipline_action_id and stg_discipline_actions.discipline_date = agg_staff_keys.discipline_date + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_discipline_actions', join_cols=['k_student', 'discipline_date', 'discipline_action_id']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} + {{ edu_edfi_source.json_flatten('v_disciplines') }} -- brule: one or the other school must be populated where (assignment_school_id is not null or responsibility_school_id is not null) @@ -132,3 +143,4 @@ join_descriptor_interpretation as ( on formatted.discipline_action = xwalk_discipline_actions.discipline_action ) select * from join_descriptor_interpretation + diff --git a/models/core_warehouse/fct_student_discipline_actions_summary.sql b/models/core_warehouse/fct_student_discipline_actions_summary.sql index 292c8f9b..1949992f 100644 --- a/models/core_warehouse/fct_student_discipline_actions_summary.sql +++ b/models/core_warehouse/fct_student_discipline_actions_summary.sql @@ -9,6 +9,9 @@ ) }} +{{ cds_depends_on('edu:student_discipline_actions_summary:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_discipline_actions_summary:custom_data_sources', []) %} + with stu_discipline_incident_behaviors_actions as ( select * from {{ ref('stg_ef3__discipline_actions__student_discipline_incident_behaviors') }} ), @@ -72,6 +75,9 @@ formatted as ( {{ edu_edfi_source.extract_extension(model_name='stg_ef3__discipline_actions__student_discipline_incident_behaviors', flatten=False) }} {# add any extension columns configured from stg_ef3__discipline_actions__disciplines #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__discipline_actions__disciplines', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from fct_student_discipline_actions left join stu_discipline_incident_behaviors_actions on fct_student_discipline_actions.k_student = stu_discipline_incident_behaviors_actions.k_student @@ -95,6 +101,11 @@ formatted as ( and fct_student_discipline_actions.k_student_xyear = behaviors_array.k_student_xyear and fct_student_discipline_actions.discipline_action_id = behaviors_array.discipline_action_id and fct_student_discipline_actions.discipline_date = behaviors_array.discipline_date + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='fct_student_discipline_actions', join_cols=['k_student', 'k_discipline_actions_event']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} + -- in order to keep the grain of k_student and k_discipline_actions_event, we want to keep this subset -- even if we do not have the severity orders defined qualify 1 = row_number() over (partition by fct_student_discipline_actions.k_student, fct_student_discipline_actions.k_discipline_actions_event order by fct_student_discipline_actions.severity_order desc nulls last, fct_student_discipline_incident_behaviors.severity_order desc nulls last) diff --git a/models/core_warehouse/fct_student_discipline_incident_behaviors.sql b/models/core_warehouse/fct_student_discipline_incident_behaviors.sql index 31695073..cdb4cd71 100644 --- a/models/core_warehouse/fct_student_discipline_incident_behaviors.sql +++ b/models/core_warehouse/fct_student_discipline_incident_behaviors.sql @@ -13,6 +13,9 @@ ) }} +{{ cds_depends_on('edu:student_discipline_incident_behaviors:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_discipline_incident_behaviors:custom_data_sources', []) %} + with stg_stu_discipline_incident_behaviors as ( select * from {{ ref('stg_ef3__student_discipline_incident_behavior_associations') }} ), @@ -75,6 +78,9 @@ formatted as ( participation_codes.participation_codes_array {# add any extension columns configured from stg_ef3__student_discipline_incident_behavior_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_discipline_incident_behavior_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_stu_discipline_incident_behaviors left join participation_codes on stg_stu_discipline_incident_behaviors.k_student = participation_codes.k_student @@ -85,6 +91,10 @@ formatted as ( join dim_discipline_incident on stg_stu_discipline_incident_behaviors.k_discipline_incident = dim_discipline_incident.k_discipline_incident left join xwalk_discipline_behaviors on stg_stu_discipline_incident_behaviors.behavior_type = xwalk_discipline_behaviors.behavior_type + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_stu_discipline_incident_behaviors', join_cols=['k_student', 'k_discipline_incident', 'behavior_type']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_discipline_incident_non_offenders.sql b/models/core_warehouse/fct_student_discipline_incident_non_offenders.sql index 1c8acf07..92d3be32 100644 --- a/models/core_warehouse/fct_student_discipline_incident_non_offenders.sql +++ b/models/core_warehouse/fct_student_discipline_incident_non_offenders.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:student_discipline_incident_non_offenders:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_discipline_incident_non_offenders:custom_data_sources', []) %} + with stg_stu_discipline_incident_non_offenders as ( select * from {{ ref('stg_ef3__student_discipline_incident_non_offender_associations') }} ), @@ -47,6 +50,9 @@ formatted as ( participation_codes.participation_codes_array {# add any extension columns configured from stg_ef3__student_discipline_incident_non_offender_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_discipline_incident_non_offender_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_stu_discipline_incident_non_offenders join participation_codes on stg_stu_discipline_incident_non_offenders.k_student = participation_codes.k_student @@ -55,6 +61,10 @@ formatted as ( join dim_student on stg_stu_discipline_incident_non_offenders.k_student = dim_student.k_student join dim_school on stg_stu_discipline_incident_non_offenders.k_school = dim_school.k_school join dim_discipline_incident on stg_stu_discipline_incident_non_offenders.k_discipline_incident = dim_discipline_incident.k_discipline_incident + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_stu_discipline_incident_non_offenders', join_cols=['k_student', 'k_discipline_incident']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_discipline_incident_summary.sql b/models/core_warehouse/fct_student_discipline_incident_summary.sql index f9ecc4d6..f62ee8f3 100644 --- a/models/core_warehouse/fct_student_discipline_incident_summary.sql +++ b/models/core_warehouse/fct_student_discipline_incident_summary.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:student_discipline_incident_summary:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_discipline_incident_summary:custom_data_sources', []) %} + with stu_discipline_incident_behaviors_actions as ( select * from {{ ref('stg_ef3__discipline_actions__student_discipline_incident_behaviors') }} ), @@ -67,6 +70,9 @@ formatted as ( {{ edu_edfi_source.extract_extension(model_name='stg_ef3__discipline_actions__student_discipline_incident_behaviors', flatten=False) }} {# add any extension columns configured from stg_ef3__discipline_actions__disciplines #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__discipline_actions__disciplines', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from fct_student_discipline_incident_behaviors left join stu_discipline_incident_behaviors_actions on fct_student_discipline_incident_behaviors.k_student = stu_discipline_incident_behaviors_actions.k_student @@ -89,6 +95,11 @@ formatted as ( on fct_student_discipline_incident_behaviors.k_student = actions_array.k_student and fct_student_discipline_incident_behaviors.k_student_xyear = actions_array.k_student_xyear and fct_student_discipline_incident_behaviors.incident_id = actions_array.incident_id + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='fct_student_discipline_incident_behaviors', join_cols=['k_student', 'k_discipline_incident']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} + -- in order to keep the grain of k_student and k_discipline_incident, we want to keep this subset -- even if we do not have the severity orders defined qualify 1 = row_number() over (partition by fct_student_discipline_incident_behaviors.k_student, fct_student_discipline_incident_behaviors.k_discipline_incident order by fct_student_discipline_actions.severity_order desc nulls last, fct_student_discipline_incident_behaviors.severity_order desc nulls last) diff --git a/models/core_warehouse/fct_student_gpa.sql b/models/core_warehouse/fct_student_gpa.sql index 5935a132..c46f0855 100644 --- a/models/core_warehouse/fct_student_gpa.sql +++ b/models/core_warehouse/fct_student_gpa.sql @@ -10,6 +10,9 @@ ) }} +{{ cds_depends_on('edu:student_gpa:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_gpa:custom_data_sources', []) %} + with combined_gpas as ( select * from {{ ref('bld_ef3__combine_gpas') }} ), @@ -30,9 +33,16 @@ formatted as ( combined_gpas.gpa_value, combined_gpas.is_cumulative, combined_gpas.max_gpa_value + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from combined_gpas join academic_record on combined_gpas.k_student_academic_record = academic_record.k_student_academic_record + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='combined_gpas', join_cols=['k_student_academic_record', 'gpa_type']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted order by tenant_code, k_student \ No newline at end of file diff --git a/models/core_warehouse/fct_student_grades.sql b/models/core_warehouse/fct_student_grades.sql index 8b3ea75c..1acc9d8b 100644 --- a/models/core_warehouse/fct_student_grades.sql +++ b/models/core_warehouse/fct_student_grades.sql @@ -15,6 +15,9 @@ ) }} +{{ cds_depends_on('edu:student_grades:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_grades:custom_data_sources', []) %} + with stg_grades as ( select * from {{ ref('stg_ef3__grades') }} ), @@ -51,6 +54,9 @@ formatted as ( letter_grade_xwalk.grade_sort_index {# add any extension columns configured from stg_ef3__grades #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__grades', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_grades join dim_student on stg_grades.k_student = dim_student.k_student @@ -62,5 +68,9 @@ formatted as ( on stg_grades.k_course_section = dim_course_section.k_course_section left join letter_grade_xwalk on lower(stg_grades.letter_grade_earned) = letter_grade_xwalk.letter_grade + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_grades', join_cols=['k_grading_period', 'k_student', 'k_school', 'k_course_section', 'grade_type']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_homeless_program_association.sql b/models/core_warehouse/fct_student_homeless_program_association.sql index a76b101e..2d7d2f36 100644 --- a/models/core_warehouse/fct_student_homeless_program_association.sql +++ b/models/core_warehouse/fct_student_homeless_program_association.sql @@ -1,17 +1,22 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_homeless_program_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_homeless_program_association:custom_data_sources', []) %} + with stage as ( select * from {{ ref('stg_ef3__student_homeless_program_associations') }} ), @@ -26,12 +31,14 @@ dim_program as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, - + stage.ed_org_id, + stage.tenant_code, dim_program.school_year, stage.program_enroll_begin_date, @@ -49,6 +56,9 @@ formatted as ( stage.reason_exited {# add any extension columns configured from stg_ef3__student_homeless_program_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_homeless_program_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -56,6 +66,10 @@ formatted as ( inner join dim_program on stage.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_homeless_program_association.yml b/models/core_warehouse/fct_student_homeless_program_association.yml index db1b879b..010e4dd7 100644 --- a/models/core_warehouse/fct_student_homeless_program_association.yml +++ b/models/core_warehouse/fct_student_homeless_program_association.yml @@ -7,8 +7,7 @@ models: This fact table contains student homeless program enrollment information. ##### Primary Key: - `k_student, k_student_xyear, k_program, program_enroll_begin_date` -- There is one - record per student, year, program enrol begin date, and homeless program. + `k_student_program` -- There is one record per student, year, ed_org_id, program enroll begin date, and homeless program. config: tags: ['homeless'] @@ -20,13 +19,17 @@ models: combination_of_columns: - k_student - k_program + - ed_org_id + - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/fct_student_language_instruction_program_association.sql b/models/core_warehouse/fct_student_language_instruction_program_association.sql index ac1c5a53..0b61e35c 100644 --- a/models/core_warehouse/fct_student_language_instruction_program_association.sql +++ b/models/core_warehouse/fct_student_language_instruction_program_association.sql @@ -1,17 +1,22 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_language_instruction_program_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_language_instruction_program_association:custom_data_sources', []) %} + with stage as ( select * from {{ ref('stg_ef3__student_language_instruction_program_associations') }} ), @@ -26,11 +31,13 @@ dim_program as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, + stage.ed_org_id, stage.tenant_code, dim_program.school_year, @@ -48,6 +55,9 @@ formatted as ( stage.reason_exited {# add any extension columns configured from stg_ef3__student_language_instruction_program_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_language_instruction_program_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -55,6 +65,10 @@ formatted as ( inner join dim_program on stage.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_language_instruction_program_association.yml b/models/core_warehouse/fct_student_language_instruction_program_association.yml index f032d01a..6d874efb 100644 --- a/models/core_warehouse/fct_student_language_instruction_program_association.yml +++ b/models/core_warehouse/fct_student_language_instruction_program_association.yml @@ -7,8 +7,7 @@ models: This fact table contains student language instruction program enrollment information. ##### Primary Key: - `k_student, k_student_xyear, k_program, program_enroll_begin_date` -- There is one - record per student, year, program enrol begin date, and language instruction program. + `k_student_program` -- There is one record per student, year, ed_org_id, program enroll begin date, and language instruction program. config: tags: ['language_instruction'] @@ -20,15 +19,17 @@ models: combination_of_columns: - k_student - k_program - - k_student_xyear + - ed_org_id - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/fct_student_learning_standard_grades.sql b/models/core_warehouse/fct_student_learning_standard_grades.sql index 6b1b62d0..0bdad2bf 100644 --- a/models/core_warehouse/fct_student_learning_standard_grades.sql +++ b/models/core_warehouse/fct_student_learning_standard_grades.sql @@ -17,6 +17,9 @@ ) }} +{{ cds_depends_on('edu:student_learning_standard_grades:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_learning_standard_grades:custom_data_sources', []) %} + with stg_grades_learning_standards as ( select * from {{ ref('stg_ef3__grades__learning_standards') }} ), @@ -50,6 +53,9 @@ formatted as ( stg_grades_learning_standards.learning_standard_numeric_grade_earned {# add any extension columns configured from stg_ef3__grades__learning_standards #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__grades__learning_standards', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_grades_learning_standards join dim_learning_standard on stg_grades_learning_standards.k_learning_standard = dim_learning_standard.k_learning_standard @@ -61,5 +67,9 @@ formatted as ( on stg_grades_learning_standards.k_grading_period = dim_grading_period.k_grading_period join dim_course_section on stg_grades_learning_standards.k_course_section = dim_course_section.k_course_section + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_grades_learning_standards', join_cols=['k_grading_period', 'k_student', 'k_school', 'k_course_section', 'grade_type', 'k_learning_standard']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_migrant_education_program_associations.sql b/models/core_warehouse/fct_student_migrant_education_program_associations.sql index 9429df44..80818300 100644 --- a/models/core_warehouse/fct_student_migrant_education_program_associations.sql +++ b/models/core_warehouse/fct_student_migrant_education_program_associations.sql @@ -1,17 +1,21 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_migrant_education_program_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_migrant_education_program_association:custom_data_sources', []) %} with stage as ( select * from {{ ref('stg_ef3__student_migrant_education_program_associations') }} @@ -27,11 +31,13 @@ dim_program as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, + stage.ed_org_id, stage.tenant_code, dim_program.school_year, stage.program_enroll_begin_date, @@ -49,6 +55,8 @@ formatted as ( {# add any extension columns configured from stg_ef3__student_migrant_education_program_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_migrant_education_program_associations', flatten=False) }} + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -56,6 +64,10 @@ formatted as ( inner join dim_program on stage.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_migrant_education_program_associations.yml b/models/core_warehouse/fct_student_migrant_education_program_associations.yml index ef123092..f900d1b5 100644 --- a/models/core_warehouse/fct_student_migrant_education_program_associations.yml +++ b/models/core_warehouse/fct_student_migrant_education_program_associations.yml @@ -7,8 +7,7 @@ models: This fact table contains student migrant education program enrollment information. ##### Primary Key: - `k_student, k_student_xyear, k_program, program_enroll_begin_date` -- There is one - record per student, year, program enrol begin date, and migrant education program. + `k_student_program` -- There is one record per student, year, ed_org_id, program enroll begin date, and migrant education program. config: tags: ['migrant_education'] @@ -20,13 +19,17 @@ models: combination_of_columns: - k_student - k_program + - ed_org_id + - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/fct_student_objective_assessment.sql b/models/core_warehouse/fct_student_objective_assessment.sql index da8181ed..8d773d83 100644 --- a/models/core_warehouse/fct_student_objective_assessment.sql +++ b/models/core_warehouse/fct_student_objective_assessment.sql @@ -10,6 +10,9 @@ ) }} +{{ cds_depends_on('edu:student_objective_assessment:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_objective_assessment:custom_data_sources', []) %} + with student_obj_assessments_long_results as ( select * from {{ ref('bld_ef3__student_objective_assessments_long_results') }} ), @@ -93,6 +96,13 @@ student_obj_assessments_wide as ( select student_obj_assessments_wide.*, v_other_results + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from student_obj_assessments_wide left join object_agg_other_results on student_obj_assessments_wide.k_student_objective_assessment = object_agg_other_results.k_student_objective_assessment + +-- custom data sources +{{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='student_obj_assessments_wide', join_cols=['k_student_objective_assessment']) }} +{{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} \ No newline at end of file diff --git a/models/core_warehouse/fct_student_parent_association.sql b/models/core_warehouse/fct_student_parent_association.sql index 09d267c6..c95c9018 100644 --- a/models/core_warehouse/fct_student_parent_association.sql +++ b/models/core_warehouse/fct_student_parent_association.sql @@ -10,6 +10,9 @@ ) }} +{{ cds_depends_on('edu:student_parent_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_parent_association:custom_data_sources', []) %} + with stg_stu_parent as ( -- parents were renamed to contacts in Data Standard v5.0 -- the contacts staging tables contain both parent and contact records @@ -43,6 +46,9 @@ formatted as ( stg_stu_parent.is_legal_guardian {# add any extension columns configured from stg_ef3__student_contact_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_contact_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_stu_parent -- subset to only the stu/parent records associated with the most recent student records join most_recent_k_student @@ -52,5 +58,9 @@ formatted as ( on stg_stu_parent.k_student_xyear = dim_student.k_student_xyear join dim_parent on stg_stu_parent.k_contact = dim_parent.k_parent + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_stu_parent', join_cols=['k_student', 'k_contact']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_program_association.sql b/models/core_warehouse/fct_student_program_association.sql index a34cf6b3..8f6c529f 100644 --- a/models/core_warehouse/fct_student_program_association.sql +++ b/models/core_warehouse/fct_student_program_association.sql @@ -1,17 +1,22 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_program_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_program_association:custom_data_sources', []) %} + with stage as ( select * from {{ ref('stg_ef3__student_program_associations') }} ), @@ -26,11 +31,13 @@ dim_program as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, + stage.ed_org_id, stage.tenant_code, dim_program.school_year, @@ -45,6 +52,9 @@ formatted as ( stage.reason_exited {# add any extension columns configured from stg_ef3__student_program_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_program_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -52,6 +62,10 @@ formatted as ( inner join dim_program on stage.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_program_association.yml b/models/core_warehouse/fct_student_program_association.yml index 7d830b30..7f2413ab 100644 --- a/models/core_warehouse/fct_student_program_association.yml +++ b/models/core_warehouse/fct_student_program_association.yml @@ -5,7 +5,7 @@ models: description: > Student program enrollment information. - *Primary Key:* `k_student, k_program` + *Primary Key:* `k_student_program, program_enroll_begin_date, participation_status` config: tags: ['core'] @@ -15,14 +15,16 @@ models: combination_of_columns: - k_student - k_program - - k_student_xyear + - ed_org_id - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/fct_student_program_participation_status.sql b/models/core_warehouse/fct_student_program_participation_status.sql new file mode 100644 index 00000000..69f23db2 --- /dev/null +++ b/models/core_warehouse/fct_student_program_participation_status.sql @@ -0,0 +1,111 @@ +{{ + config( + post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", + "alter table {{ this }} alter column k_student set not null", + "alter table {{ this }} alter column k_program set not null", + "alter table {{ this }} alter column program_enroll_begin_date set not null", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} alter column participation_status set not null", + "alter table {{ this }} alter column status_begin_date set not null", + "alter table {{ this }} add primary key (k_student_program, participation_status, status_begin_date)", + "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", + "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", + ] + ) +}} + +{{ cds_depends_on('edu:student_program_participation_status:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_program_participation_status:custom_data_sources', []) %} + +-- Define all optional program service models here. +{% set stage_program_relations = [] %} + +--Generic Program Assoc +{% do stage_program_relations.append(ref('stg_ef3__stu_program__program_participation_statuses')) %} + +-- Special Education +{% if var('src:program:special_ed:enabled', True) %} + {% do stage_program_relations.append(ref('stg_ef3__stu_spec_ed__program_participation_statuses')) %} +{% endif %} + +-- Language Instruction +{% if var('src:program:language_instruction:enabled', True) %} + {% do stage_program_relations.append(ref('stg_ef3__stu_lang_instr__program_participation_statuses')) %} +{% endif %} + +-- Homeless +{% if var('src:program:homeless:enabled', True) %} + {% do stage_program_relations.append(ref('stg_ef3__stu_homeless__program_participation_statuses')) %} +{% endif %} + +-- Title I Part A +{% if var('src:program:title_i:enabled', True) %} + {% do stage_program_relations.append(ref('stg_ef3__stu_title_i_part_a__program_participation_statuses')) %} +{% endif %} + +-- CTE +{% if var('src:program:cte:enabled', True) %} + {% do stage_program_relations.append(ref('stg_ef3__stu_cte__program_participation_statuses')) %} +{% endif %} + +-- Migrant Education +{% if var('src:program:migrant_education:enabled', True) %} + {% do stage_program_relations.append(ref('stg_ef3__stu_migrant_edu__program_participation_statuses')) %} +{% endif %} + +-- Food Service +{% if var('src:program:food_service:enabled', True) %} + {% do stage_program_relations.append(ref('stg_ef3__stu_school_food_service__program_services')) %} +{% endif %} + +with dim_student as ( + select * from {{ ref('dim_student') }} +), + +dim_program as ( + select * from {{ ref('dim_program') }} +), + +stacked as ( + {{ dbt_utils.union_relations( + + relations=stage_program_relations + + ) }} +), + +{# -- append the name of each `stage_program_relations` into a new list, so we can pass that list to `extract_extension` below -- #} +{% set relation_names = [] -%} +{%- for relation in stage_program_relations -%} + {%- do relation_names.append(relation.name) -%} +{%- endfor -%} + +subset as ( + select + stacked.k_student_program, + stacked.k_student, + stacked.k_student_xyear, + stacked.k_program, + stacked.tenant_code, + stacked.ed_org_id, + stacked.program_enroll_begin_date, + stacked.participation_status, + stacked.status_begin_date, + stacked.status_end_date, + stacked.designated_by + {# add any extension columns configured from all stage_program_relations #} + {{ edu_edfi_source.extract_extension(model_name=relation_names, flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} + from stacked + join dim_program + on stacked.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stacked', join_cols=['k_student_program', 'participation_status', 'status_begin_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} +) + +select * from subset diff --git a/models/core_warehouse/fct_student_program_participation_status.yml b/models/core_warehouse/fct_student_program_participation_status.yml new file mode 100644 index 00000000..55829413 --- /dev/null +++ b/models/core_warehouse/fct_student_program_participation_status.yml @@ -0,0 +1,61 @@ +version: 1 + +models: + - name: fct_student_program_participation_status + description: > + ##### Overview: + This fact table provides student program participation statuses, received as part of a program enrollment. + It references any or all of the following models: + - [stg_ef3__stu_spec_ed__program_participation_statuses](#!/model/model.edu_edfi_source.stg_ef3__stu_spec_ed__program_participation_statuses) + - [stg_ef3__stu_lang_instr__program_participation_statuses](#!/model/model.edu_edfi_source.stg_ef3__stu_lang_instr__program_participation_statuses) + - [stg_ef3__stu_homeless__program_participation_statuses](#!/model/model.edu_edfi_source.stg_ef3__stu_homeless__program_participation_statuses) + - [stg_ef3__stu_title_i__program_participation_statuses](#!/model/model.edu_edfi_source.stg_ef3__stu_title_i__program_participation_statuses) + - [stg_ef3__stu_cte__program_participation_statuses](#!/model/model.edu_edfi_source.stg_ef3__stu_cte__program_participation_statuses) + - [stg_ef3__stu_migrant_edu__program_participation_statusess](#!/model/model.edu_edfi_source.stg_ef3__stu_migrant_edu__program_participation_statuses) + - [stg_ef3__stu_school_food_service__program_participation_statuses](#!/model/model.edu_edfi_source.stg_ef3__stu_school_food_service__program_participation_statuses) + - [stg_ef3__stu_program__program_participation_statuses](#!/model/model.edu_edfi_source.stg_ef3__stu_program__program_participation_statuses) + + ##### Primary Key: + `k_student_program, participation_status, status_begin_date` -- + There is one record per student, year, program, ed org, program start date, program participation status, and status begin date. + + ##### Important business rules: + - `program_enroll_begin_date` is included in the unique key, because a student may be associated with the same program at multiple times, + and those associations may have different program participation statuses. When joining this table to `fct_student_{program_type}_program_association`, + always include `program_enroll_begin_date` in the join. + + config: + tags: ['special_ed', 'homeless', 'language_instruction', 'title_i', 'cte', 'migrant_education', 'food_service'] + enabled: > + {%- set var_values = [] -%} + {%- for variable in [ + 'src:program:special_ed:enabled', + 'src:program:homeless:enabled', + 'src:program:language_instruction:enabled', + 'src:program:title_i:enabled', + 'src:program:cte:enabled', + 'src:program:migrant_education:enabled', + 'src:program:food_service:enabled' + ] -%} + {%- do var_values.append(var(variable, none)) -%} + {%- endfor -%} + + {%- if True in var_values -%} + {{ True | as_bool }} + {%- elif False in var_values -%} + {{ False | as_bool }} + {%- else -%} + {{ True | as_bool }} + {%- endif -%} + + columns: + - name: k_student_program + - name: k_student + - name: k_program + - name: tenant_code + - name: ed_org_id + - name: program_enroll_begin_date + - name: participation_status + - name: status_begin_date + - name: status_end_date + - name: designated_by diff --git a/models/core_warehouse/fct_student_program_service.sql b/models/core_warehouse/fct_student_program_service.sql index e0794617..3c636d71 100644 --- a/models/core_warehouse/fct_student_program_service.sql +++ b/models/core_warehouse/fct_student_program_service.sql @@ -1,21 +1,28 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", + "alter table {{ this }} alter column ed_org_id set not null", "alter table {{ this }} alter column program_service set not null", - "alter table {{ this }} add primary key (k_student, k_program, program_enroll_begin_date, program_service)", + "alter table {{ this }} add primary key (k_student_program, program_service)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_program_service:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_program_service:custom_data_sources', []) %} -- Define all optional program service models here. {% set stage_program_relations = [] %} +--Generic Program Assoc +{% do stage_program_relations.append(ref('stg_ef3__stu_program__program_services')) %} + -- Special Education {% if var('src:program:special_ed:enabled', True) %} {% do stage_program_relations.append(ref('stg_ef3__stu_spec_ed__program_services')) %} @@ -74,23 +81,36 @@ stacked as ( {%- endfor -%} subset as ( - select - stacked.k_student, - stacked.k_student_xyear, - stacked.k_program, - stacked.tenant_code, - stacked.program_enroll_begin_date, - stacked.program_service, - stacked.primary_indicator, - stacked.v_providers, - stacked.service_begin_date, - stacked.service_end_date - {# add any extension columns configured from all stage_program_relations #} - {{ edu_edfi_source.extract_extension(model_name=relation_names, flatten=False) }} - - from stacked - join dim_program - on stacked.k_program = dim_program.k_program + select + stacked.k_student_program, + stacked.k_student, + stacked.k_student_xyear, + stacked.k_program, + stacked.tenant_code, + stacked.ed_org_id, + stacked.program_enroll_begin_date, + stacked.program_service, + stacked.primary_indicator, + {% if var('src:program:special_ed:enabled', True) %} + stacked.v_providers, + {% endif %} + {% if var('src:program:cte:enabled', True) %} + stacked.cip_code, + {% endif %} + stacked.service_begin_date, + stacked.service_end_date + {# add any extension columns configured from all stage_program_relations #} + {{ edu_edfi_source.extract_extension(model_name=relation_names, flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} + from stacked + join dim_program + on stacked.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stacked', join_cols=['k_student_program', 'program_service']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from subset diff --git a/models/core_warehouse/fct_student_program_service.yml b/models/core_warehouse/fct_student_program_service.yml index 7c2aa774..6ea2eab8 100644 --- a/models/core_warehouse/fct_student_program_service.yml +++ b/models/core_warehouse/fct_student_program_service.yml @@ -84,12 +84,15 @@ models: combination_of_columns: - k_student - k_program + - ed_org_id - program_enroll_begin_date - program_service columns: + - name: k_student_program - name: k_student - name: k_program - name: tenant_code + - name: ed_org_id - name: program_enroll_begin_date - name: program_service - name: primary_indicator diff --git a/models/core_warehouse/fct_student_school_association.sql b/models/core_warehouse/fct_student_school_association.sql index c47f3c99..b22b4b9b 100644 --- a/models/core_warehouse/fct_student_school_association.sql +++ b/models/core_warehouse/fct_student_school_association.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:student_school_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_school_association:custom_data_sources', []) %} + with stg_stu_school as ( select * from {{ ref('stg_ef3__student_school_associations') }} ), @@ -94,6 +97,9 @@ formatted as ( ) as is_latest_annual_entry {# add any extension columns configured from stg_ef3__student_school_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_school_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_stu_school join dim_student on stg_stu_school.k_student = dim_student.k_student @@ -110,6 +116,10 @@ formatted as ( and equal_null(dim_school_calendar.k_school_calendar, bld_school_calendar_windows.k_school_calendar) left join xwalk_grade_levels on stg_stu_school.entry_grade_level = xwalk_grade_levels.grade_level + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_stu_school', join_cols=['k_student', 'k_school', 'entry_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} where true {% if var('edu:enroll:exclude_exit_before_first_day', True) -%} -- exclude students who exited before the first school day diff --git a/models/core_warehouse/fct_student_school_attendance_event.sql b/models/core_warehouse/fct_student_school_attendance_event.sql index 89ed0a6b..ef982279 100644 --- a/models/core_warehouse/fct_student_school_attendance_event.sql +++ b/models/core_warehouse/fct_student_school_attendance_event.sql @@ -13,6 +13,9 @@ ) }} +{{ cds_depends_on('edu:student_school_attendance_event:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_school_attendance_event:custom_data_sources', []) %} + with stg_stu_sch_attend as ( select * from {{ ref('stg_ef3__student_school_attendance_events') }} ), @@ -97,14 +100,14 @@ deduped as ( ), formatted as ( select - k_student, + deduped.k_student, k_student_xyear, - k_school, - k_calendar_date, - k_session, + deduped.k_school, + deduped.k_calendar_date, + deduped.k_session, tenant_code, calendar_date, - attendance_event_category, + deduped.attendance_event_category, attendance_event_reason, is_absent, attendance_excusal_status, @@ -115,6 +118,13 @@ formatted as ( educational_environment {# add any extension columns configured from stg_ef3__student_school_attendance_events #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_school_attendance_events', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from deduped + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='deduped', join_cols=['k_student', 'k_school', 'k_session', 'attendance_event_category', 'k_calendar_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_school_food_service_program_associations.sql b/models/core_warehouse/fct_student_school_food_service_program_associations.sql index 980389aa..b9296f7f 100644 --- a/models/core_warehouse/fct_student_school_food_service_program_associations.sql +++ b/models/core_warehouse/fct_student_school_food_service_program_associations.sql @@ -1,17 +1,21 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_school_food_service_program_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_school_food_service_program_association:custom_data_sources', []) %} with stage as ( select * from {{ ref('stg_ef3__student_school_food_service_program_association') }} @@ -27,11 +31,14 @@ dim_program as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, + stage.ed_org_id, + stage.tenant_code, dim_program.school_year, stage.program_enroll_begin_date, @@ -41,6 +48,9 @@ formatted as ( stage.served_outside_of_regular_session {# add any extension columns configured from stg_ef3__student_school_food_service_program_association #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_school_food_service_program_association', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -48,6 +58,10 @@ formatted as ( inner join dim_program on stage.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_school_food_service_program_associations.yml b/models/core_warehouse/fct_student_school_food_service_program_associations.yml index 76dfcc9a..f1221ca7 100644 --- a/models/core_warehouse/fct_student_school_food_service_program_associations.yml +++ b/models/core_warehouse/fct_student_school_food_service_program_associations.yml @@ -7,8 +7,7 @@ models: This fact table contains student food service program enrollment information. ##### Primary Key: - `k_student, k_student_xyear, k_program, program_enroll_begin_date` -- There - is one record per student, year, program enrol begin date, and food service program. + `k_student_program` -- There is one record per student, year, ed_org_id, program enroll begin date, and food service program. config: tags: ['food_service'] @@ -20,13 +19,17 @@ models: combination_of_columns: - k_student - k_program + - ed_org_id + - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/fct_student_section_association.sql b/models/core_warehouse/fct_student_section_association.sql index 544e4b22..8d3eddfb 100644 --- a/models/core_warehouse/fct_student_section_association.sql +++ b/models/core_warehouse/fct_student_section_association.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:student_section_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_section_association:custom_data_sources', []) %} + with stg_stu_section as ( select * from {{ ref('stg_ef3__student_section_associations') }} ), @@ -48,10 +51,17 @@ formatted as ( stg_stu_section.repeat_identifier {# add any extension columns configured from stg_ef3__student_section_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_section_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_stu_section join dim_student on stg_stu_section.k_student = dim_student.k_student join dim_course_section on stg_stu_section.k_course_section = dim_course_section.k_course_section + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_stu_section', join_cols=['k_student', 'k_course_section', 'begin_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted \ No newline at end of file diff --git a/models/core_warehouse/fct_student_section_attendance_event.sql b/models/core_warehouse/fct_student_section_attendance_event.sql index bd921f39..a12ff2b8 100644 --- a/models/core_warehouse/fct_student_section_attendance_event.sql +++ b/models/core_warehouse/fct_student_section_attendance_event.sql @@ -12,6 +12,9 @@ ) }} +{{ cds_depends_on('edu:student_section_attendance_event:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_section_attendance_event:custom_data_sources', []) %} + with stg_stu_section_attendance as ( select * from {{ ref('stg_ef3__student_section_attendance_events') }} ), @@ -42,6 +45,9 @@ formatted as ( stg_stu_section_attendance.educational_environment {# add any extension columns configured from stg_ef3__student_section_attendance_events #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_section_attendance_events', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stg_stu_section_attendance join dim_student on stg_stu_section_attendance.k_student = dim_student.k_student @@ -49,5 +55,9 @@ formatted as ( on stg_stu_section_attendance.k_course_section = dim_course_section.k_course_section join xwalk_att_events on stg_stu_section_attendance.attendance_event_category = xwalk_att_events.attendance_event_descriptor + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_stu_section_attendance', join_cols=['k_student', 'k_course_section', 'attendance_event_category', 'attendance_event_date']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_special_education_program_association.sql b/models/core_warehouse/fct_student_special_education_program_association.sql index 34602598..3e518605 100644 --- a/models/core_warehouse/fct_student_special_education_program_association.sql +++ b/models/core_warehouse/fct_student_special_education_program_association.sql @@ -1,17 +1,21 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_special_education_program_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_special_education_program_association:custom_data_sources', []) %} with stage as ( select * from {{ ref('stg_ef3__student_special_education_program_associations') }} @@ -35,11 +39,13 @@ bld_primary_disability as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, + stage.ed_org_id, stage.tenant_code, dim_program.school_year, stage.program_enroll_begin_date, @@ -64,6 +70,9 @@ formatted as ( bld_primary_disability.disability_type as primary_disability_type {# add any extension columns configured from stg_ef3__student_special_education_program_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_special_education_program_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -82,6 +91,10 @@ formatted as ( on stage.k_student = bld_primary_disability.k_student and stage.k_program = bld_primary_disability.k_program and stage.program_enroll_begin_date = bld_primary_disability.program_enroll_begin_date + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_special_education_program_association.yml b/models/core_warehouse/fct_student_special_education_program_association.yml index 19fa1a7b..599d7755 100644 --- a/models/core_warehouse/fct_student_special_education_program_association.yml +++ b/models/core_warehouse/fct_student_special_education_program_association.yml @@ -7,8 +7,7 @@ models: This fact table contains student special education program enrollment information. ##### Primary Key: - `k_student, k_student_xyear, k_program, program_enroll_begin_date` -- There is one - record per student, year, program enrol begin date, and special education program + `k_student_program` -- There is one record per student, year, ed_org_id, program enroll begin date, and special education program config: tags: ['special_ed'] @@ -19,13 +18,16 @@ models: combination_of_columns: - k_student - k_program + - ed_org_id - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/fct_student_subgroup.sql b/models/core_warehouse/fct_student_subgroup.sql index 6bd45fb2..fcd349f4 100644 --- a/models/core_warehouse/fct_student_subgroup.sql +++ b/models/core_warehouse/fct_student_subgroup.sql @@ -9,6 +9,9 @@ ) }} +{{ cds_depends_on('edu:student_subgroup:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_subgroup:custom_data_sources', []) %} + with dim_student as ( select * from {{ ref('dim_student') }} ), @@ -77,12 +80,19 @@ keyed as ( dim_subgroup.k_subgroup, stu_long_subgroup.tenant_code, stu_long_subgroup.school_year + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stu_long_subgroup_with_all stu_long_subgroup -- todo: use dbt_utils.generate_surrogate_key() instead? -- lower() is because unpivot makes values capitalized join dim_subgroup on lower(stu_long_subgroup.subgroup_category) = lower(dim_subgroup.subgroup_category) and lower(stu_long_subgroup.subgroup_value) = lower(dim_subgroup.subgroup_value) + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stu_long_subgroup_with_all', join_cols=['k_student', 'subgroup_category', 'subgroup_value']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from keyed diff --git a/models/core_warehouse/fct_student_title_i_part_a_program_association.sql b/models/core_warehouse/fct_student_title_i_part_a_program_association.sql index c3538401..3a07bcf4 100644 --- a/models/core_warehouse/fct_student_title_i_part_a_program_association.sql +++ b/models/core_warehouse/fct_student_title_i_part_a_program_association.sql @@ -1,17 +1,22 @@ {{ config( post_hook=[ + "alter table {{ this }} alter column k_student_program set not null", "alter table {{ this }} alter column k_student set not null", "alter table {{ this }} alter column k_student_xyear set not null", "alter table {{ this }} alter column k_program set not null", "alter table {{ this }} alter column program_enroll_begin_date set not null", - "alter table {{ this }} add primary key (k_student, k_student_xyear, k_program, program_enroll_begin_date)", + "alter table {{ this }} alter column ed_org_id set not null", + "alter table {{ this }} add primary key (k_student_program)", "alter table {{ this }} add constraint fk_{{ this.name }}_student foreign key (k_student) references {{ ref('dim_student') }}", "alter table {{ this }} add constraint fk_{{ this.name }}_program foreign key (k_program) references {{ ref('dim_program') }}", ] ) }} +{{ cds_depends_on('edu:student_title_i_part_a_program_association:custom_data_sources') }} +{% set custom_data_sources = var('edu:student_title_i_part_a_program_association:custom_data_sources', []) %} + with stage as ( select * from {{ ref('stg_ef3__student_title_i_part_a_program_associations') }} ), @@ -26,11 +31,13 @@ dim_program as ( formatted as ( select + stage.k_student_program, dim_student.k_student, dim_student.k_student_xyear, dim_program.k_program, dim_program.k_lea, dim_program.k_school, + stage.ed_org_id, stage.tenant_code, dim_program.school_year, stage.program_enroll_begin_date, @@ -44,6 +51,9 @@ formatted as ( stage.reason_exited {# add any extension columns configured from stg_ef3__student_title_i_part_a_program_associations #} {{ edu_edfi_source.extract_extension(model_name='stg_ef3__student_title_i_part_a_program_associations', flatten=False) }} + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from stage inner join dim_student @@ -51,6 +61,10 @@ formatted as ( inner join dim_program on stage.k_program = dim_program.k_program + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stage', join_cols=['k_student_program']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from formatted diff --git a/models/core_warehouse/fct_student_title_i_part_a_program_association.yml b/models/core_warehouse/fct_student_title_i_part_a_program_association.yml index a0ebea79..28528227 100644 --- a/models/core_warehouse/fct_student_title_i_part_a_program_association.yml +++ b/models/core_warehouse/fct_student_title_i_part_a_program_association.yml @@ -7,8 +7,7 @@ models: This fact table contains student Title I Part A program enrollment information. ##### Primary Key: - k_student, k_student_xyear, k_program, program_enroll_begin_date` -- There is one - record per student, year, program enrol begin date, and Title I Part A program. + k_student_program` -- There is one record per student, year, ed_org_id, program enroll begin date, and Title I Part A program. config: tags: ['title_i'] @@ -20,15 +19,17 @@ models: combination_of_columns: - k_student - k_program - - k_student_xyear + - ed_org_id - program_enroll_begin_date columns: + - name: k_student_program - name: k_student - name: k_student_xyear - name: k_program - name: k_lea - name: k_school + - name: ed_org_id - name: tenant_code - name: school_year - name: program_enroll_begin_date diff --git a/models/core_warehouse/msr_student_cumulative_attendance.sql b/models/core_warehouse/msr_student_cumulative_attendance.sql index d054d541..ea88325f 100644 --- a/models/core_warehouse/msr_student_cumulative_attendance.sql +++ b/models/core_warehouse/msr_student_cumulative_attendance.sql @@ -11,6 +11,9 @@ ) }} +{{ cds_depends_on('edu:msr_student_cumulative_attendance:custom_data_sources') }} +{% set custom_data_sources = var('edu:msr_student_cumulative_attendance:custom_data_sources', []) %} + with stu_daily_attendance as ( select * from {{ ref('fct_student_daily_attendance') }} ), @@ -49,9 +52,16 @@ metric_labels as ( when meets_enrollment_threshold then metric_absentee_categories.level_label else null end as absentee_category_label + + -- custom data sources columns + {{ add_cds_columns(custom_data_sources=custom_data_sources) }} from aggregated left join metric_absentee_categories on attendance_rate > metric_absentee_categories.threshold_lower and attendance_rate <= metric_absentee_categories.threshold_upper + + -- custom data sources + {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='aggregated', join_cols=['k_student', 'k_school', 'school_year']) }} + {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }} ) select * from metric_labels