diff --git a/CHANGELOG.md b/CHANGELOG.md index 285f5ad3..9e629fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased ## New features +- Update base/stg models for SEDM `student_ieps` and `iep_events` +- Add base/stg models for new SEDM `student_iep_disability_collections` ## Under the hood ## Fixes diff --git a/macros/gen_skey.sql b/macros/gen_skey.sql index 8958ed7a..c529e434 100644 --- a/macros/gen_skey.sql +++ b/macros/gen_skey.sql @@ -344,18 +344,18 @@ }, 'k_idea_event': { 'reference_name': 'idea_event_reference', - 'col_list': ['ed_org_id', - 'idea_event_id', - 'student_unique_id', - 'idea_event'], + 'col_list': ['educationOrganizationId', + 'ideaEvent', + 'ideaEventId', + 'studentUniqueId'], 'annualize': True }, 'k_student_iep': { 'reference_name': 'student_iep_reference', - 'col_list': ['iep_finalized_date', - 'ed_org_id', - 'student_iep_association_id', - 'student_unique_id'], + 'col_list': ['educationOrganizationId', + 'iepFinalizedDate', + 'studentIEPAssociationId', + 'studentUniqueId'], 'annualize': True }, 'k_student_iep_service_prescription': { diff --git a/models/staging/sedm/base/_sedm__base.yml b/models/staging/sedm/base/_sedm__base.yml index e012faaa..39f3d760 100644 --- a/models/staging/sedm/base/_sedm__base.yml +++ b/models/staging/sedm/base/_sedm__base.yml @@ -5,6 +5,10 @@ models: config: tags: ['sedm'] enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: base_sedm__student_iep_disability_collections + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" - name: base_sedm__idea_events config: tags: ['sedm'] diff --git a/models/staging/sedm/base/base_sedm__idea_events.sql b/models/staging/sedm/base/base_sedm__idea_events.sql index d2579e28..4134c225 100644 --- a/models/staging/sedm/base/base_sedm__idea_events.sql +++ b/models/staging/sedm/base/base_sedm__idea_events.sql @@ -21,8 +21,8 @@ renamed as ( v:educationOrganizationReference:link:rel::string as ed_org_type, {{ extract_descriptor('v:ideaEventDescriptor::string') }} as idea_event, -- value columns - v:eventBeginDate::date as event_begin_date, - v:eventEndDate::date as event_end_date, + v:beginDate::date as event_begin_date, + v:endDate::date as event_end_date, v:eventNarrative::string as event_narrative, -- descriptors {{ extract_descriptor('v:eventReasonDescriptor::string') }} as event_reason, diff --git a/models/staging/sedm/base/base_sedm__student_iep_disability_collections.sql b/models/staging/sedm/base/base_sedm__student_iep_disability_collections.sql new file mode 100644 index 00000000..8cc7fdad --- /dev/null +++ b/models/staging/sedm/base/base_sedm__student_iep_disability_collections.sql @@ -0,0 +1,29 @@ +with disabilities as ( + {{ source_edfi3('student_iep_disability_collections')}} +), +renamed as ( + select + tenant_code, + api_year, + pull_timestamp, + last_modified_timestamp, + file_row_number, + filename, + is_deleted, + v:id::string as record_guid, + ods_version, + data_model_version, + v:studentIEPReference:educationOrganizationId::int as ed_org_id, + v:studentIEPReference:iepFinalizedDate::date as iep_finalized_date, + v:studentIEPReference:studentIEPAssociationID::string as student_iep_association_id, + v:studentIEPReference:studentUniqueId::string as student_unique_id, + -- references + v:studentIEPReference as student_iep_reference, + -- lists + v:disabilities as v_disabilities, + + -- edfi extensions + v:_ext as v_ext + from disabilities +) +select * from renamed \ No newline at end of file diff --git a/models/staging/sedm/stage/_sedm_stage.yml b/models/staging/sedm/stage/_sedm_stage.yml index cff66b27..e63beaad 100644 --- a/models/staging/sedm/stage/_sedm_stage.yml +++ b/models/staging/sedm/stage/_sedm_stage.yml @@ -11,4 +11,8 @@ models: config: tags: ['sedm'] enabled: "{{ var('src:domain:sedm:enabled', False) }}" + - name: stg_sedm__student_iep_disability_collections + config: + tags: ['sedm'] + enabled: "{{ var('src:domain:sedm:enabled', False) }}" diff --git a/models/staging/sedm/stage/stg_sedm__student_iep_disability_collections.sql b/models/staging/sedm/stage/stg_sedm__student_iep_disability_collections.sql new file mode 100644 index 00000000..a0a742d7 --- /dev/null +++ b/models/staging/sedm/stage/stg_sedm__student_iep_disability_collections.sql @@ -0,0 +1,45 @@ +with base as ( + select * from {{ ref('base_sedm__student_iep_disability_collections') }} +), +keyed as ( + select + {{ dbt_utils.generate_surrogate_key( + [ + 'tenant_code', + 'api_year', + 'ed_org_id', + 'iep_finalized_date', + 'lower(student_iep_association_id)', + 'lower(student_unique_id)' + ] + ) }} as k_student_iep, + api_year as school_year, + base.*, + {{ extract_extension(model_name=this.name, flatten=True) }} + from base +), +deduped as ( + {{ + dbt_utils.deduplicate( + relation='keyed', + partition_by='k_student_iep, ed_org_id', + order_by='last_modified_timestamp desc, pull_timestamp desc' + ) + }} +), +flattened as ( + select + tenant_code, + school_year, + k_student_iep, + ed_org_id, + student_iep_association_id, + {{ extract_descriptor('value:disabilityDescriptor::string') }} as disability_descriptor, + {{ extract_descriptor('value:disabilityDeterminationSourceTypeDescriptor::string') }} as disability_determination_source_type_descriptor, + {{ extract_descriptor('value:disabilityDiagnosis::string') }} as disability_diagnosis, + {{ extract_descriptor('value:orderOfDisability::int') }} as order_of_disability + from deduped + {{ json_flatten('v_disabilities') }} + where not is_deleted +) +select * from flattened \ No newline at end of file diff --git a/models/staging/sedm/stage/stg_sedm__student_ieps.sql b/models/staging/sedm/stage/stg_sedm__student_ieps.sql index 6e57ad7d..3741d33d 100644 --- a/models/staging/sedm/stage/stg_sedm__student_ieps.sql +++ b/models/staging/sedm/stage/stg_sedm__student_ieps.sql @@ -8,9 +8,9 @@ keyed as ( [ 'tenant_code', 'api_year', - 'lower(student_iep_association_id)', 'ed_org_id', 'iep_finalized_date', + 'lower(student_iep_association_id)', 'lower(student_unique_id)' ] ) }} as k_student_iep,