From 9b462d9ab5edfd40262b4790a3929b8e6edb36d6 Mon Sep 17 00:00:00 2001 From: sleblanc23 Date: Wed, 29 Jan 2025 10:05:19 -0600 Subject: [PATCH] add second dedupe --- .../stage/stg_ef3__staff_school_associations.sql | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/models/staging/edfi_3/stage/stg_ef3__staff_school_associations.sql b/models/staging/edfi_3/stage/stg_ef3__staff_school_associations.sql index 10dcd73b..337d96a6 100644 --- a/models/staging/edfi_3/stage/stg_ef3__staff_school_associations.sql +++ b/models/staging/edfi_3/stage/stg_ef3__staff_school_associations.sql @@ -16,13 +16,24 @@ keyed as ( from base_staff_school_assoc ), -deduped as ( +-- The first dedupe step removes duplicates within api_years that occur when the optional school_year field is updated +deduped_api_year as ( {{ dbt_utils.deduplicate( relation='keyed', + partition_by='k_staff, k_school, program_assignment, api_year', + order_by='pull_timestamp desc' + ) + }} +-- The second dedupe step removes duplicates across api_years +), +deduped_school_year as ( + {{ + dbt_utils.deduplicate( + relation='deduped_api_year', partition_by='k_staff, k_school, program_assignment, school_year', order_by='pull_timestamp desc' ) }} ) -select * from deduped \ No newline at end of file +select * from deduped_school_year \ No newline at end of file