Feature/: TDOE CDS enhanced and everywhere

macros/accordion_columns.sql

@@ -9,8 +9,11 @@ be included in a table.
   - coalesce_value: if this value is specified, a coalesce function will be added with
     this value as the second parameter (to be returned if the value in the column is null).
     If none (default), the coalesce function will not be added.
+  - add_trailing_comma: if this value is specified, a trailing comma will be added to the last
+    column. If not passed in, a trailing comma is added for backward compatibility
 -#}
-{% macro accordion_columns(source_table, exclude_columns, source_alias=none, coalesce_value=none) %}
+
+{% macro accordion_columns(source_table, exclude_columns, source_alias=none, coalesce_value=none, add_trailing_comma=true) %}
   {%- if source_alias is none -%}
    {%- set source_alias = source_table -%} 
   {%- endif -%}
@@ -20,11 +23,11 @@ be included in a table.
     ) %}
   {%- if coalesce_value is none %}
     {%- for col in keep_cols %}
-      {{ source_alias }}.{{ col }},
+      {{ source_alias }}.{{ col }}{% if not loop.last %},{% elif add_trailing_comma %},{% endif %}
     {%- endfor %}
   {%- else -%}
     {%- for col in keep_cols %}
-      coalesce({{ source_alias }}.{{ col }}, {{ coalesce_value }}) as {{ col }},
+      coalesce({{ source_alias }}.{{ col }}, {{ coalesce_value }}) as {{ col }}{% if not loop.last %},{% elif add_trailing_comma %},{% endif %}
     {%- endfor %}  
   {%- endif -%}
-{% endmacro %}
+{% endmacro %}

macros/custom_data_sources_macros.sql

@@ -0,0 +1,106 @@
+{%- macro cds_depends_on(cds_model_config) -%}
+    {%- set cds = var(cds_model_config, {}) -%}
+        {%- if cds is mapping and cds|length -%}
+            {%- for source_name, _ in cds.items() -%}
+-- depends_on: {{ ref(source_name) }}
+            {%- endfor -%}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro add_cds_joins_v1(custom_data_sources, driving_alias, join_cols) -%}
+    {#-
+      Expected "old" config shape per source:
+        <source_name>: {
+          <indicator_name>: { where: <col_or_expr> },
+          ...
+        }
+
+      Only sources WITHOUT 'joins' key are considered "old" and handled here.
+    -#}
+
+    {% if custom_data_sources is mapping and custom_data_sources|length %}
+        {% for source_name, source_config in custom_data_sources|dictsort %}
+            {% if 'joins' not in source_config %}
+                left join {{ ref(source_name) }} as {{ source_name }}
+                {% for col in join_cols %}
+                    {% if loop.first %}
+                        on {{ driving_alias }}.{{ col }} = {{ source_name }}.{{ col }}
+                    {% else %}
+                        and {{ driving_alias }}.{{ col }} = {{ source_name }}.{{ col }}
+                    {% endif %}
+                {% endfor %}
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+{%- endmacro -%}
+
+{%- macro add_cds_joins_v2(custom_data_sources) -%}
+    {#-
+      Expected "new" config shape per source:
+        <source_name>: {
+          joins: [ "<expr1>", "<expr2>", ... ],
+          cds_additional_cols: {
+            <src_col_name>: { as: <alias>, default: <literal_or_expr> },
+            ...
+          }
+        }
+
+      Only sources WITH 'joins' key are handled here.
+    -#}
+
+    {% if custom_data_sources is mapping and custom_data_sources|length %}
+        {% for source_name, source_config in custom_data_sources|dictsort %}
+            {% if 'joins' in source_config and source_config.joins %}
+                left join {{ ref(source_name) }} as {{ source_name }}
+                {% for join in source_config.joins %}
+                    {% if loop.first %}
+                        on {{ join }}
+                    {% else %}
+                        and {{ join }}
+                    {% endif %}
+                {% endfor %}
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+{%- endmacro -%}
+
+{%- macro add_cds_columns(custom_data_sources) -%}
+    {#-
+      Old format per source:
+        <source_name>: {
+          <indicator_name>: { where: <col_or_expr> },
+          ...
+        }
+      Emits:
+        , <source_name>.<where> as <indicator_name>
+
+      New format per source:
+        <source_name>: {
+          joins: [...],
+          cds_additional_cols: {
+            <src_col_name>: { as: <alias>, default: <literal_or_expr> },
+            ...
+          }
+        }
+      Emits:
+        , coalesce(<source_name>.<src_col_name>, <default>) as <alias>
+    -#}
+
+    {% if custom_data_sources is mapping and custom_data_sources|length %}
+        {% for source_name, source_config in custom_data_sources|dictsort %}
+            {% if 'joins' in source_config %}
+                {# New config pathway #}
+                {% if 'cds_additional_cols' in source_config and source_config.cds_additional_cols %}
+                    {% for src_col_name, src_col_config in source_config.cds_additional_cols.items() %}
+                        , coalesce({{ source_name }}.{{ src_col_name }}, {{ src_col_config.default }}) as {{ src_col_config.as }}
+                    {% endfor %}
+                {% endif %}
+            {% else %}
+                {# Old config pathway #}
+                {% for indicator_name, indicator_config in source_config.items() %}
+                    , {{ source_name }}.{{ indicator_config.where }} as {{ indicator_name }}
+                {% endfor %}
+            {% endif %}
+        {% endfor %}
+    {% endif %}
+{%- endmacro -%}

models/build/edfi_3/students/bld_ef3__student__disabilities.sql

@@ -0,0 +1,36 @@
+-- Define all optional disability models here.
+{% set stage_disability_relations = [] %}
+
+--Ed Org Disabilities
+{% do stage_disability_relations.append(ref('stg_ef3__stu_ed_org__disabilities')) %}
+
+-- Special Education
+{% if var('src:program:special_ed:enabled', True) %}
+    {% do stage_disability_relations.append(ref('stg_ef3__stu_spec_ed__disabilities')) %}
+{% endif %}
+
+with stacked as (
+    {{ dbt_utils.union_relations(
+        relations=stage_disability_relations
+    ) }}
+),
+formatted as (
+    select 
+        tenant_code,
+        api_year,
+        school_year,
+        k_student,
+        ed_org_id,
+        k_lea,
+        k_school,
+        k_program,
+        program_enroll_begin_date,
+        program_enroll_end_date,
+        disability_type,
+        disability_source_type,
+        disability_diagnosis,
+        order_of_disability,
+        v_designations
+    from stacked
+)
+select * from formatted

models/build/edfi_3/students/bld_ef3__student__wide_disability_designations.sql

@@ -0,0 +1,45 @@
+with student_disabilities as (
+    select * from {{ ref('bld_ef3__student__disabilities') }}
+),
+xwalk_disability_designations as (
+    select * from {{ ref('xwalk_disability_designations') }}
+),
+flattened as (
+    select 
+        tenant_code,
+        api_year,
+        school_year,
+        k_student,
+        ed_org_id,
+        k_lea,
+        k_school,
+        k_program,
+        disability_type,
+        {{ edu_edfi_source.extract_descriptor('designation.value:disabilityDesignationDescriptor::string') }} as disability_designation
+    from student_disabilities
+        {{ edu_edfi_source.json_flatten('v_designations', 'designation', outer=true) }}
+),
+pivoted as (
+    select 
+        tenant_code,
+        api_year,
+        school_year,
+        k_student,
+        ed_org_id,
+        k_lea,
+        k_school,
+        k_program,
+        disability_type
+        {%- if not is_empty_model('xwalk_disability_designations') -%},
+            {{ ea_pivot(
+                    column='indicator_name',
+                    values=dbt_utils.get_column_values(ref('xwalk_disability_designations'), 'indicator_name'),
+                    cast='boolean',
+            ) }}
+        {%- endif %}
+    from flattened
+    left outer join xwalk_disability_designations 
+        on flattened.disability_designation = xwalk_disability_designations.disability_designation_descriptor
+    group by all
+)
+select * from pivoted

models/core_warehouse/dim_assessment.sql

@@ -7,6 +7,9 @@
   )
 }}
 
+{{ cds_depends_on('edu:assessment:custom_data_sources') }}
+{% set custom_data_sources = var('edu:assessment:custom_data_sources', []) %}
+
 with stg_assessments as (
     select * from {{ ref('stg_ef3__assessments') }}
 ),
@@ -42,6 +45,9 @@ formatted as (
         assessment_scores.scores_array,
         assessment_pls.performance_levels_array,
         assessment_grades.grades_array
+
+        -- custom data sources columns
+        {{ add_cds_columns(custom_data_sources=custom_data_sources) }}
     from stg_assessments
     -- making all of these left joins because none of these are actually required
     left join assessment_scores 
@@ -50,6 +56,10 @@ formatted as (
         on stg_assessments.k_assessment = assessment_pls.k_assessment
     left join assessment_grades
         on stg_assessments.k_assessment = assessment_grades.k_assessment
+
+    -- custom data sources
+    {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_assessments', join_cols=['k_assessment']) }}
+    {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }}
 )
 select * from formatted
 order by tenant_code, k_assessment

models/core_warehouse/dim_calendar_date.sql

@@ -8,8 +8,9 @@
     ]
   )
 }}
-{# Load custom data sources from var #}
-{% set custom_data_sources = var("edu:calendar_date:custom_data_sources", []) %}
+
+{{ cds_depends_on('edu:calendar_date:custom_data_sources') }}
+{% set custom_data_sources = var('edu:calendar_date:custom_data_sources', []) %}
 
 with stg_calendar_date as (
     select * from {{ ref('stg_ef3__calendar_dates') }}
@@ -105,24 +106,15 @@ week_calculation as (
             else week_of_calendar_year + 52 - start_week_offset
         end as week_of_school_year
 
-        -- custom indicators
-        {% if custom_data_sources is not none and custom_data_sources | length -%}
-          {%- for source in custom_data_sources -%}
-            {%- for indicator in custom_data_sources[source] -%}
-              , {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }}
-            {%- endfor -%}
-          {%- endfor -%}
-        {%- endif %}
+        -- custom data sources columns
+        {{ add_cds_columns(custom_data_sources=custom_data_sources) }}
     from augmented
     join week_offset
         on augmented.k_school_calendar = week_offset.k_school_calendar
+
     -- custom data sources
-    {% if custom_data_sources is not none and custom_data_sources | length -%}
-      {%- for source in custom_data_sources -%}
-        left join {{ ref(source) }}
-          on augmented.k_calendar_date = {{ source }}.k_calendar_date
-      {% endfor %}
-    {%- endif %}
+    {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='augmented', join_cols=['k_calendar_date']) }}
+    {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }}
 )
 select * from week_calculation
 order by tenant_code, k_school, calendar_date desc

models/core_warehouse/dim_class_period.sql

@@ -6,8 +6,8 @@
     ]
   )
 }}
-{# Load custom data sources from var #}
-{% set custom_data_sources = var("edu:class_period:custom_data_sources", []) %}
+{{ cds_depends_on('edu:class_period:custom_data_sources') }}
+{% set custom_data_sources = var('edu:class_period:custom_data_sources', []) %}
 
 with class_periods as (
     select * from {{ ref('stg_ef3__class_periods') }}
@@ -45,24 +45,15 @@ formatted as (
         end as end_time,
         timediff(MINUTE, concat('2020-01-01 ', start_time)::timestamp, concat('2020-01-01 ', end_time)::timestamp) as period_duration
 
-        -- custom indicators
-        {% if custom_data_sources is not none and custom_data_sources | length -%}
-          {%- for source in custom_data_sources -%}
-            {%- for indicator in custom_data_sources[source] -%}
-              , {{ custom_data_sources[source][indicator]['where'] }} as {{ indicator }}
-            {%- endfor -%}
-          {%- endfor -%}
-        {%- endif %}
+        -- custom data sources columns
+        {{ add_cds_columns(custom_data_sources=custom_data_sources) }}
     from class_periods
     join dim_school
         on class_periods.k_school = dim_school.k_school
+
     -- custom data sources
-    {% if custom_data_sources is not none and custom_data_sources | length -%}
-      {%- for source in custom_data_sources -%}
-        left join {{ ref(source) }}
-          on class_periods.k_class_period = {{ source }}.k_class_period
-      {% endfor %}
-    {%- endif %}
+    {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='class_periods', join_cols=['k_class_period']) }}
+    {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }}
 )
 select {{ edu_edfi_source.star('formatted', except=['meeting_time']) }}
 from formatted

models/core_warehouse/dim_classroom.sql

@@ -8,6 +8,9 @@
   )
 }}
 
+{{ cds_depends_on('edu:classroom:custom_data_sources') }}
+{% set custom_data_sources = var('edu:classroom:custom_data_sources', []) %}
+
 with locations as (
     select * from {{ ref('stg_ef3__locations') }}
 ),
@@ -22,9 +25,16 @@ formatted as (
         locations.classroom_id_code,
         locations.maximum_seating,
         locations.optimum_seating
+
+        -- custom data sources columns
+        {{ add_cds_columns(custom_data_sources=custom_data_sources) }}
     from locations
     join dim_school
         on locations.k_school = dim_school.k_school
+
+    -- custom data sources
+    {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='locations', join_cols=['k_location']) }}
+    {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }}
 )
 select * from formatted
 order by tenant_code, k_school, k_classroom

models/core_warehouse/dim_cohort.sql

@@ -7,6 +7,9 @@
   )
 }}
 
+{{ cds_depends_on('edu:cohort:custom_data_sources') }}
+{% set custom_data_sources = var('edu:cohort:custom_data_sources', []) %}
+
 with stg_cohorts as (
     select * from {{ ref('stg_ef3__cohorts') }}
 ),
@@ -23,7 +26,14 @@ formatted as (
         stg_cohorts.cohort_description,
         stg_cohorts.cohort_scope,
         stg_cohorts.cohort_type
+
+        -- custom data sources_columns
+        {{ add_cds_columns(custom_data_sources=custom_data_sources) }}
     from stg_cohorts
+
+    -- custom data sources
+    {{ add_cds_joins_v1(custom_data_sources=custom_data_sources, driving_alias='stg_cohorts', join_cols=['k_cohort']) }}
+    {{ add_cds_joins_v2(custom_data_sources=custom_data_sources) }}
 )
 select * from formatted
 order by tenant_code, school_year desc, k_cohort