From 89069d6b34c20c53d0d72310492de97c02d5208c Mon Sep 17 00:00:00 2001 From: keenzarate213 Date: Wed, 28 Feb 2024 02:28:10 -0600 Subject: [PATCH 1/8] create fct contact and add language in dim student --- models/core_warehouse/dim_student.sql | 13 ++++++- models/core_warehouse/dim_subgroup.sql | 4 +- .../fct_student_contact_information.sql | 39 +++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 models/core_warehouse/fct_student_contact_information.sql diff --git a/models/core_warehouse/dim_student.sql b/models/core_warehouse/dim_student.sql index 66bc8b1d..b933f8b4 100644 --- a/models/core_warehouse/dim_student.sql +++ b/models/core_warehouse/dim_student.sql @@ -50,6 +50,13 @@ stu_grade as ( select * from {{ ref('bld_ef3__stu_grade_level') }} ), +choose_language as ( + {{ row_pluck(ref('stg_ef3__stu_ed_org__languages'), + key='k_student', + column='language_use', + preferred='Home language') }} +), + -- student programs {% if var('src:program:special_ed:enabled', True) %} stu_special_ed as ( @@ -188,8 +195,8 @@ formatted as ( stg_student.api_year = max(stg_student.api_year) over(partition by stg_student.k_student_xyear) as is_latest_record, stu_immutable_demos.race_array, - stu_immutable_demos.safe_display_name - + stu_immutable_demos.safe_display_name, + choose_language.code_value as home_language from stg_student join stu_demos @@ -212,6 +219,8 @@ formatted as ( left join stu_grade on stu_demos.k_student = stu_grade.k_student and stg_student.api_year = stu_grade.school_year + left join choose_language + on stg_student.k_student = choose_language.k_student -- student programs {% if var('src:program:special_ed:enabled', True) %} diff --git a/models/core_warehouse/dim_subgroup.sql b/models/core_warehouse/dim_subgroup.sql index ef4ac0f0..cdbd9ba2 100644 --- a/models/core_warehouse/dim_subgroup.sql +++ b/models/core_warehouse/dim_subgroup.sql @@ -32,7 +32,9 @@ stu_long_subgroup as ( 'display_name', 'birth_date', 'race_array', - 'safe_display_name' + 'safe_display_name', + 'full_address', + 'home_language' ], remove = stu_id_cols, field_name='subgroup_category', diff --git a/models/core_warehouse/fct_student_contact_information.sql b/models/core_warehouse/fct_student_contact_information.sql new file mode 100644 index 00000000..828ced25 --- /dev/null +++ b/models/core_warehouse/fct_student_contact_information.sql @@ -0,0 +1,39 @@ +with stg_stu_address as ( + select * from {{ ref('stg_ef3__stu_ed_org__addresses') }} +), +stg_stu_emails as ( + select * + from {{ ref('stg_ef3__stu_ed_org__emails') }} +), +stg_stu_phones as ( + select * + from {{ ref('stg_ef3__stu_ed_org__telephones') }} +), +dim_student as ( + select * from {{ ref('dim_student') }} +) +select + dim_student.k_student, + dim_student.k_student_xyear, + dim_student.tenant_code, + dim_student.school_year, + address_type, + street_address, + city, + name_of_county, + state_code, + postal_code, + building_site_number, + locale, + congressional_district, + county_fips_code, + latitude, + longitude, + email_type, + email_address, + phone_number_type, + phone_number +from dim_student +left join stg_stu_address on dim_student.k_student = stg_stu_address.k_student +left join stg_stu_emails on dim_student.k_student = stg_stu_emails.k_student +left join stg_stu_phones on dim_student.k_student = stg_stu_phones.k_student From 7d25b18c6b624422dac07ef48a007df862649a86 Mon Sep 17 00:00:00 2001 From: keenzarate213 Date: Wed, 28 Feb 2024 02:29:23 -0600 Subject: [PATCH 2/8] remove full address --- models/core_warehouse/dim_subgroup.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/models/core_warehouse/dim_subgroup.sql b/models/core_warehouse/dim_subgroup.sql index cdbd9ba2..67870716 100644 --- a/models/core_warehouse/dim_subgroup.sql +++ b/models/core_warehouse/dim_subgroup.sql @@ -33,7 +33,6 @@ stu_long_subgroup as ( 'birth_date', 'race_array', 'safe_display_name', - 'full_address', 'home_language' ], remove = stu_id_cols, From d9880f2eaa8f12ec86fc0c2b050e42b354c0dd04 Mon Sep 17 00:00:00 2001 From: keenzarate213 Date: Thu, 7 Mar 2024 14:11:14 -0600 Subject: [PATCH 3/8] make a couple updates make language configurable, widen contact data before joining with dim student in fct_stu_contact_info --- dbt_project.yml | 4 ++ .../edfi_3/students/_edfi_3__students.yml | 3 ++ .../students/bld_ef3__choose_stu_language.sql | 15 ++++++ .../bld_ef3__student_wide_address.sql | 27 ++++++++++ .../students/bld_ef3__student_wide_emails.sql | 27 ++++++++++ .../bld_ef3__student_wide_phone_numbers.sql | 27 ++++++++++ models/core_warehouse/dim_student.sql | 17 +++--- models/core_warehouse/dim_subgroup.sql | 6 ++- .../fct_student_contact_information.sql | 53 +++++++++---------- .../core_warehouse/fct_student_subgroup.sql | 6 ++- 10 files changed, 145 insertions(+), 40 deletions(-) create mode 100644 models/build/edfi_3/students/bld_ef3__choose_stu_language.sql create mode 100644 models/build/edfi_3/students/bld_ef3__student_wide_address.sql create mode 100644 models/build/edfi_3/students/bld_ef3__student_wide_emails.sql create mode 100644 models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql diff --git a/dbt_project.yml b/dbt_project.yml index fa1f5950..0fa1a483 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -36,6 +36,10 @@ vars: 'edu:stu_demos:hispanic_latino_code': Latinx 'edu:stu_demos:race_unknown_code': Unknown + # preferred student language use and column name + 'edu:stu_language:language_use': Home language + 'edu:stu_language:language_use_col_name': home_language + # custom intersections of other demographic groupings 'edu:stu_demos:intersection_groups': Null # custom data sources for student characteristics. must contain and be unique by `k_student` diff --git a/models/build/edfi_3/students/_edfi_3__students.yml b/models/build/edfi_3/students/_edfi_3__students.yml index 8d221bba..42f4aab4 100644 --- a/models/build/edfi_3/students/_edfi_3__students.yml +++ b/models/build/edfi_3/students/_edfi_3__students.yml @@ -61,3 +61,6 @@ models: - name: bld_ef3__wide_ids_student config: tags: ['core'] + - name: bld_ef3__choose_stu_languages + config: + tags: ['core'] diff --git a/models/build/edfi_3/students/bld_ef3__choose_stu_language.sql b/models/build/edfi_3/students/bld_ef3__choose_stu_language.sql new file mode 100644 index 00000000..e081b8c9 --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__choose_stu_language.sql @@ -0,0 +1,15 @@ +with stu_language as ( + select * from {{ ref('stg_ef3__stu_ed_org__languages') }} +), +choose_language as ( + -- pick a single language use, + -- defaults to home language + {{ + row_pluck('stu_language', + key='k_student', + column='language_use', + preferred=var('edu:stu_language:language_use') + ) + }} +) +select * from choose_language \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student_wide_address.sql b/models/build/edfi_3/students/bld_ef3__student_wide_address.sql new file mode 100644 index 00000000..ea609605 --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student_wide_address.sql @@ -0,0 +1,27 @@ +with stg_student_address as ( + select * from {{ ref('stg_ef3__stu_ed_org__addresses') }} +), +address_types as ( + select * from {{ ref('xwalk_student_address_types') }} +), +address_wide as ( + select + k_student, + tenant_code + {%- if not is_empty_model('xwalk_student_address_types') -%}, + {{ dbt_utils.pivot( + 'normalized_address_type', + dbt_utils.get_column_values(ref('xwalk_student_address_types'), 'normalized_address_type'), + agg='max', + then_value='street_address', + else_value='null', + suffix='_address', + quote_identifiers = False + ) }} + {%- endif %} + from stg_student_address + join address_types + on stg_student_address.address_type = address_types.original_address_type + group by k_student, tenant_code +) +select * from address_wide \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student_wide_emails.sql b/models/build/edfi_3/students/bld_ef3__student_wide_emails.sql new file mode 100644 index 00000000..9fa22f04 --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student_wide_emails.sql @@ -0,0 +1,27 @@ +with stg_student_emails as ( + select * from {{ ref('stg_ef3__stu_ed_org__emails') }} +), +email_types as ( + select * from {{ ref('xwalk_student_email_types') }} +), +emails_wide as ( + select + k_student, + tenant_code + {%- if not is_empty_model('xwalk_student_email_types') -%}, + {{ dbt_utils.pivot( + 'normalized_email_type', + dbt_utils.get_column_values(ref('xwalk_student_email_types'), 'normalized_email_type'), + agg='max', + then_value='email_address', + else_value='null', + suffix='_email_address', + quote_identifiers = False + ) }} + {%- endif %} + from stg_student_emails + join email_types + on stg_student_emails.email_type = email_types.original_email_type + group by k_student, tenant_code +) +select * from emails_wide \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql b/models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql new file mode 100644 index 00000000..cdda3212 --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql @@ -0,0 +1,27 @@ +with stg_student_phones as ( + select * from {{ ref('stg_ef3__stu_ed_org__telephones') }} +), +phone_number_types as ( + select * from {{ ref('xwalk_student_phone_number_types') }} +), +phones_wide as ( + select + k_student, + tenant_code + {%- if not is_empty_model('xwalk_student_phone_number_types') -%}, + {{ dbt_utils.pivot( + 'normalized_phone_number_type', + dbt_utils.get_column_values(ref('xwalk_student_phone_number_types'), 'normalized_phone_number_type'), + agg='max', + then_value='phone_number', + else_value='null', + suffix='_phone_number', + quote_identifiers = False + ) }} + {%- endif %} + from stg_student_phones + join phone_number_types + on stg_student_phones.phone_number_type = phone_number_types.original_phone_number_type + group by k_student, tenant_code +) +select * from phones_wide \ No newline at end of file diff --git a/models/core_warehouse/dim_student.sql b/models/core_warehouse/dim_student.sql index b933f8b4..8e43142b 100644 --- a/models/core_warehouse/dim_student.sql +++ b/models/core_warehouse/dim_student.sql @@ -24,6 +24,8 @@ {% set custom_language_instruction_program_agg_indicators = var('edu:language_instruction:custom_program_agg_indicators', None) %} {% set custom_title_i_program_agg_indicators = var('edu:title_i:custom_program_agg_indicators', None) %} +{# Load customizable column name for language use, defaults as home_language #} +{% set language_use_name = var('edu:stu_language:language_use_col_name') %} with stg_student as ( select * from {{ ref('stg_ef3__students') }} @@ -49,14 +51,9 @@ stu_programs as ( stu_grade as ( select * from {{ ref('bld_ef3__stu_grade_level') }} ), - -choose_language as ( - {{ row_pluck(ref('stg_ef3__stu_ed_org__languages'), - key='k_student', - column='language_use', - preferred='Home language') }} +stu_language as ( + select * from {{ ref('bld_ef3__choose_stu_language')}} ), - -- student programs {% if var('src:program:special_ed:enabled', True) %} stu_special_ed as ( @@ -196,7 +193,7 @@ formatted as ( stu_immutable_demos.race_array, stu_immutable_demos.safe_display_name, - choose_language.code_value as home_language + stu_language.code_value as {{ language_use_name }} from stg_student join stu_demos @@ -219,8 +216,8 @@ formatted as ( left join stu_grade on stu_demos.k_student = stu_grade.k_student and stg_student.api_year = stu_grade.school_year - left join choose_language - on stg_student.k_student = choose_language.k_student + left join stu_language + on stg_student.k_student = stu_language.k_student -- student programs {% if var('src:program:special_ed:enabled', True) %} diff --git a/models/core_warehouse/dim_subgroup.sql b/models/core_warehouse/dim_subgroup.sql index 67870716..b42a83f5 100644 --- a/models/core_warehouse/dim_subgroup.sql +++ b/models/core_warehouse/dim_subgroup.sql @@ -3,6 +3,10 @@ tags=['bypass_rls'] ) }} + +{# bring in the customizable column name for language use from dim_student, defaults as home_language #} +{% set language_use_name = var('edu:stu_language:language_use_col_name') %} + with dim_student as ( select * from {{ ref('dim_student') }} ), @@ -33,7 +37,7 @@ stu_long_subgroup as ( 'birth_date', 'race_array', 'safe_display_name', - 'home_language' + language_use_name ], remove = stu_id_cols, field_name='subgroup_category', diff --git a/models/core_warehouse/fct_student_contact_information.sql b/models/core_warehouse/fct_student_contact_information.sql index 828ced25..b0077c0c 100644 --- a/models/core_warehouse/fct_student_contact_information.sql +++ b/models/core_warehouse/fct_student_contact_information.sql @@ -1,13 +1,11 @@ -with stg_stu_address as ( - select * from {{ ref('stg_ef3__stu_ed_org__addresses') }} +with stu_phone_wide as ( + select * from {{ ref('bld_ef3__student_wide_phone_numbers') }} ), -stg_stu_emails as ( - select * - from {{ ref('stg_ef3__stu_ed_org__emails') }} -), -stg_stu_phones as ( - select * - from {{ ref('stg_ef3__stu_ed_org__telephones') }} +stu_emails_wide as ( + select * from {{ ref('bld_ef3__student_wide_emails') }} +), +stu_address_wide as ( + select * from {{ ref('bld_ef3__student_wide_address') }} ), dim_student as ( select * from {{ ref('dim_student') }} @@ -17,23 +15,22 @@ select dim_student.k_student_xyear, dim_student.tenant_code, dim_student.school_year, - address_type, - street_address, - city, - name_of_county, - state_code, - postal_code, - building_site_number, - locale, - congressional_district, - county_fips_code, - latitude, - longitude, - email_type, - email_address, - phone_number_type, - phone_number + {{ accordion_columns( + source_table='bld_ef3__student_wide_phone_numbers', + exclude_columns=["k_student", "tenant_code"], + source_alias='stu_phone_wide' + ) }} + {{ accordion_columns( + source_table='bld_ef3__student_wide_emails', + exclude_columns=["k_student", "tenant_code"], + source_alias='stu_emails_wide' + ) }} + {{ accordion_columns( + source_table='bld_ef3__student_wide_address', + exclude_columns=["k_student", "tenant_code"], + source_alias='stu_address_wide' + ) }} from dim_student -left join stg_stu_address on dim_student.k_student = stg_stu_address.k_student -left join stg_stu_emails on dim_student.k_student = stg_stu_emails.k_student -left join stg_stu_phones on dim_student.k_student = stg_stu_phones.k_student +left join stu_phone_wide on dim_student.k_student = stu_phone_wide.k_student +left join stu_emails_wide on dim_student.k_student = stu_emails_wide.k_student +left join stu_address_wide on dim_student.k_student = stu_address_wide.k_student diff --git a/models/core_warehouse/fct_student_subgroup.sql b/models/core_warehouse/fct_student_subgroup.sql index 3099d22a..0c322441 100644 --- a/models/core_warehouse/fct_student_subgroup.sql +++ b/models/core_warehouse/fct_student_subgroup.sql @@ -1,3 +1,6 @@ +{# bring in the customizable column name for language use from dim_student, defaults as home_language #} +{% set language_use_name = var('edu:stu_language:language_use_col_name') %} + with dim_student as ( select * from {{ ref('dim_student') }} ), @@ -24,7 +27,8 @@ stu_long_subgroup as ( 'display_name', 'birth_date', 'race_array', - 'safe_display_name' + 'safe_display_name', + language_use_name ], remove = stu_id_cols, field_name='subgroup_category', From 1239237bbc82503fa45533210aa93f978c80e790 Mon Sep 17 00:00:00 2001 From: keenzarate213 Date: Fri, 15 Mar 2024 13:40:32 -0500 Subject: [PATCH 4/8] remove the need of xwalk adjust default order_by in get_column_values, switch bld language to wide instead of choosing preference, and adjust yml to list language to allow multiple --- dbt_project.yml | 7 ++++--- .../edfi_3/students/_edfi_3__students.yml | 14 ++++++++++++- .../students/bld_ef3__choose_stu_language.sql | 15 ------------- ...ql => bld_ef3__student_wide_addresses.sql} | 11 +++------- .../students/bld_ef3__student_wide_emails.sql | 15 +++++-------- .../bld_ef3__student_wide_languages.sql | 21 +++++++++++++++++++ .../bld_ef3__student_wide_phone_numbers.sql | 13 ++++-------- models/core_warehouse/dim_student.sql | 12 +++++++---- models/core_warehouse/dim_subgroup.sql | 2 +- .../fct_student_contact_information.sql | 4 ++-- .../core_warehouse/fct_student_subgroup.sql | 2 +- 11 files changed, 62 insertions(+), 54 deletions(-) delete mode 100644 models/build/edfi_3/students/bld_ef3__choose_stu_language.sql rename models/build/edfi_3/students/{bld_ef3__student_wide_address.sql => bld_ef3__student_wide_addresses.sql} (53%) create mode 100644 models/build/edfi_3/students/bld_ef3__student_wide_languages.sql diff --git a/dbt_project.yml b/dbt_project.yml index 0fa1a483..f1905581 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -36,9 +36,10 @@ vars: 'edu:stu_demos:hispanic_latino_code': Latinx 'edu:stu_demos:race_unknown_code': Unknown - # preferred student language use and column name - 'edu:stu_language:language_use': Home language - 'edu:stu_language:language_use_col_name': home_language + # default language use home language + 'edu:stu_language:language_uses': + - home_language + #- correspondence_language # custom intersections of other demographic groupings 'edu:stu_demos:intersection_groups': Null diff --git a/models/build/edfi_3/students/_edfi_3__students.yml b/models/build/edfi_3/students/_edfi_3__students.yml index 42f4aab4..15177eaf 100644 --- a/models/build/edfi_3/students/_edfi_3__students.yml +++ b/models/build/edfi_3/students/_edfi_3__students.yml @@ -61,6 +61,18 @@ models: - name: bld_ef3__wide_ids_student config: tags: ['core'] - - name: bld_ef3__choose_stu_languages + - name: bld_ef3__student_wide_addresses config: tags: ['core'] + - name: bld_ef3__student_wide_emails + config: + tags: ['core'] + - name: bld_ef3__student_wide_languages + config: + tags: ['core'] + - name: bld_ef3__student_wide_phone_numbers + config: + tags: ['core'] + + + diff --git a/models/build/edfi_3/students/bld_ef3__choose_stu_language.sql b/models/build/edfi_3/students/bld_ef3__choose_stu_language.sql deleted file mode 100644 index e081b8c9..00000000 --- a/models/build/edfi_3/students/bld_ef3__choose_stu_language.sql +++ /dev/null @@ -1,15 +0,0 @@ -with stu_language as ( - select * from {{ ref('stg_ef3__stu_ed_org__languages') }} -), -choose_language as ( - -- pick a single language use, - -- defaults to home language - {{ - row_pluck('stu_language', - key='k_student', - column='language_use', - preferred=var('edu:stu_language:language_use') - ) - }} -) -select * from choose_language \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student_wide_address.sql b/models/build/edfi_3/students/bld_ef3__student_wide_addresses.sql similarity index 53% rename from models/build/edfi_3/students/bld_ef3__student_wide_address.sql rename to models/build/edfi_3/students/bld_ef3__student_wide_addresses.sql index ea609605..1592402d 100644 --- a/models/build/edfi_3/students/bld_ef3__student_wide_address.sql +++ b/models/build/edfi_3/students/bld_ef3__student_wide_addresses.sql @@ -1,17 +1,14 @@ with stg_student_address as ( select * from {{ ref('stg_ef3__stu_ed_org__addresses') }} ), -address_types as ( - select * from {{ ref('xwalk_student_address_types') }} -), address_wide as ( select k_student, tenant_code - {%- if not is_empty_model('xwalk_student_address_types') -%}, + {%- if not is_empty_model('stg_ef3__stu_ed_org__addresses') -%}, {{ dbt_utils.pivot( - 'normalized_address_type', - dbt_utils.get_column_values(ref('xwalk_student_address_types'), 'normalized_address_type'), + 'address_type', + dbt_utils.get_column_values(ref('stg_ef3__stu_ed_org__addresses'), 'address_type', order_by = 'address_type'), agg='max', then_value='street_address', else_value='null', @@ -20,8 +17,6 @@ address_wide as ( ) }} {%- endif %} from stg_student_address - join address_types - on stg_student_address.address_type = address_types.original_address_type group by k_student, tenant_code ) select * from address_wide \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student_wide_emails.sql b/models/build/edfi_3/students/bld_ef3__student_wide_emails.sql index 9fa22f04..cba5ed39 100644 --- a/models/build/edfi_3/students/bld_ef3__student_wide_emails.sql +++ b/models/build/edfi_3/students/bld_ef3__student_wide_emails.sql @@ -1,17 +1,14 @@ with stg_student_emails as ( select * from {{ ref('stg_ef3__stu_ed_org__emails') }} ), -email_types as ( - select * from {{ ref('xwalk_student_email_types') }} -), -emails_wide as ( +email_wide as ( select k_student, tenant_code - {%- if not is_empty_model('xwalk_student_email_types') -%}, + {%- if not is_empty_model('stg_ef3__stu_ed_org__emails') -%}, {{ dbt_utils.pivot( - 'normalized_email_type', - dbt_utils.get_column_values(ref('xwalk_student_email_types'), 'normalized_email_type'), + 'email_type', + dbt_utils.get_column_values(ref('stg_ef3__stu_ed_org__emails'), 'email_type', order_by = 'email_type'), agg='max', then_value='email_address', else_value='null', @@ -20,8 +17,6 @@ emails_wide as ( ) }} {%- endif %} from stg_student_emails - join email_types - on stg_student_emails.email_type = email_types.original_email_type group by k_student, tenant_code ) -select * from emails_wide \ No newline at end of file +select * from email_wide \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student_wide_languages.sql b/models/build/edfi_3/students/bld_ef3__student_wide_languages.sql new file mode 100644 index 00000000..eabba78b --- /dev/null +++ b/models/build/edfi_3/students/bld_ef3__student_wide_languages.sql @@ -0,0 +1,21 @@ +with stg_student_languages as ( + select * from {{ ref('stg_ef3__stu_ed_org__languages') }} +), +language_wide as ( + select + k_student, + tenant_code + {%- if not is_empty_model('stg_ef3__stu_ed_org__languages') -%}, + {{ dbt_utils.pivot( + 'language_use', + dbt_utils.get_column_values(ref('stg_ef3__stu_ed_org__languages'), 'language_use', order_by = 'language_use'), + agg='max', + then_value='code_value', + else_value='null', + quote_identifiers = False + ) }} + {%- endif %} + from stg_student_languages + group by k_student, tenant_code +) +select * from language_wide \ No newline at end of file diff --git a/models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql b/models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql index cdda3212..5e780fc4 100644 --- a/models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql +++ b/models/build/edfi_3/students/bld_ef3__student_wide_phone_numbers.sql @@ -1,17 +1,14 @@ with stg_student_phones as ( select * from {{ ref('stg_ef3__stu_ed_org__telephones') }} ), -phone_number_types as ( - select * from {{ ref('xwalk_student_phone_number_types') }} -), phones_wide as ( select k_student, tenant_code - {%- if not is_empty_model('xwalk_student_phone_number_types') -%}, + {%- if not is_empty_model('stg_ef3__stu_ed_org__telephones') -%}, {{ dbt_utils.pivot( - 'normalized_phone_number_type', - dbt_utils.get_column_values(ref('xwalk_student_phone_number_types'), 'normalized_phone_number_type'), + 'phone_number_type', + dbt_utils.get_column_values(ref('stg_ef3__stu_ed_org__telephones'), 'phone_number_type', order_by = 'phone_number_type'), agg='max', then_value='phone_number', else_value='null', @@ -20,8 +17,6 @@ phones_wide as ( ) }} {%- endif %} from stg_student_phones - join phone_number_types - on stg_student_phones.phone_number_type = phone_number_types.original_phone_number_type group by k_student, tenant_code ) -select * from phones_wide \ No newline at end of file +select * from phones_wide diff --git a/models/core_warehouse/dim_student.sql b/models/core_warehouse/dim_student.sql index 8e43142b..264d3ae1 100644 --- a/models/core_warehouse/dim_student.sql +++ b/models/core_warehouse/dim_student.sql @@ -25,7 +25,7 @@ {% set custom_title_i_program_agg_indicators = var('edu:title_i:custom_program_agg_indicators', None) %} {# Load customizable column name for language use, defaults as home_language #} -{% set language_use_name = var('edu:stu_language:language_use_col_name') %} +{% set language_use_name = var('edu:stu_language:language_uses') %} with stg_student as ( select * from {{ ref('stg_ef3__students') }} @@ -52,7 +52,7 @@ stu_grade as ( select * from {{ ref('bld_ef3__stu_grade_level') }} ), stu_language as ( - select * from {{ ref('bld_ef3__choose_stu_language')}} + select * from {{ ref('bld_ef3__student_wide_languages')}} ), -- student programs {% if var('src:program:special_ed:enabled', True) %} @@ -192,8 +192,12 @@ formatted as ( stg_student.api_year = max(stg_student.api_year) over(partition by stg_student.k_student_xyear) as is_latest_record, stu_immutable_demos.race_array, - stu_immutable_demos.safe_display_name, - stu_language.code_value as {{ language_use_name }} + stu_immutable_demos.safe_display_name + {%- if language_use_name is not none and language_use_name | length -%} + {%- for language_use in language_use_name -%} + , stu_language.{{ language_use }} as {{ language_use }} + {%- endfor -%} + {%- endif %} from stg_student join stu_demos diff --git a/models/core_warehouse/dim_subgroup.sql b/models/core_warehouse/dim_subgroup.sql index b42a83f5..4d24b3a7 100644 --- a/models/core_warehouse/dim_subgroup.sql +++ b/models/core_warehouse/dim_subgroup.sql @@ -5,7 +5,7 @@ }} {# bring in the customizable column name for language use from dim_student, defaults as home_language #} -{% set language_use_name = var('edu:stu_language:language_use_col_name') %} +{% set language_use_name = var('edu:stu_language:language_uses') %} with dim_student as ( select * from {{ ref('dim_student') }} diff --git a/models/core_warehouse/fct_student_contact_information.sql b/models/core_warehouse/fct_student_contact_information.sql index b0077c0c..cfa61a93 100644 --- a/models/core_warehouse/fct_student_contact_information.sql +++ b/models/core_warehouse/fct_student_contact_information.sql @@ -5,7 +5,7 @@ stu_emails_wide as ( select * from {{ ref('bld_ef3__student_wide_emails') }} ), stu_address_wide as ( - select * from {{ ref('bld_ef3__student_wide_address') }} + select * from {{ ref('bld_ef3__student_wide_addresses') }} ), dim_student as ( select * from {{ ref('dim_student') }} @@ -26,7 +26,7 @@ select source_alias='stu_emails_wide' ) }} {{ accordion_columns( - source_table='bld_ef3__student_wide_address', + source_table='bld_ef3__student_wide_addresses', exclude_columns=["k_student", "tenant_code"], source_alias='stu_address_wide' ) }} diff --git a/models/core_warehouse/fct_student_subgroup.sql b/models/core_warehouse/fct_student_subgroup.sql index 0c322441..fddfa66f 100644 --- a/models/core_warehouse/fct_student_subgroup.sql +++ b/models/core_warehouse/fct_student_subgroup.sql @@ -1,5 +1,5 @@ {# bring in the customizable column name for language use from dim_student, defaults as home_language #} -{% set language_use_name = var('edu:stu_language:language_use_col_name') %} +{% set language_use_name = var('edu:stu_language:language_uses') %} with dim_student as ( select * from {{ ref('dim_student') }} From 2cc9fc7332b73a9a5245b3d176d04d18e3fa9e49 Mon Sep 17 00:00:00 2001 From: rlittle08 Date: Wed, 20 Mar 2024 15:34:42 -0500 Subject: [PATCH 5/8] fix concat in dim_subgroup, rename var, handle empty var --- dbt_project.yml | 5 ++--- models/core_warehouse/dim_student.sql | 6 +++--- models/core_warehouse/dim_subgroup.sql | 7 +++---- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index f1905581..41f18f71 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -37,10 +37,9 @@ vars: 'edu:stu_demos:race_unknown_code': Unknown # default language use home language - 'edu:stu_language:language_uses': + 'edu:stu_language:language_use_types': - home_language - #- correspondence_language - + # custom intersections of other demographic groupings 'edu:stu_demos:intersection_groups': Null # custom data sources for student characteristics. must contain and be unique by `k_student` diff --git a/models/core_warehouse/dim_student.sql b/models/core_warehouse/dim_student.sql index 264d3ae1..ae3d2acb 100644 --- a/models/core_warehouse/dim_student.sql +++ b/models/core_warehouse/dim_student.sql @@ -25,7 +25,7 @@ {% set custom_title_i_program_agg_indicators = var('edu:title_i:custom_program_agg_indicators', None) %} {# Load customizable column name for language use, defaults as home_language #} -{% set language_use_name = var('edu:stu_language:language_uses') %} +{% set language_use_types = var('edu:stu_language:language_use_types', None) %} with stg_student as ( select * from {{ ref('stg_ef3__students') }} @@ -193,8 +193,8 @@ formatted as ( stu_immutable_demos.race_array, stu_immutable_demos.safe_display_name - {%- if language_use_name is not none and language_use_name | length -%} - {%- for language_use in language_use_name -%} + {%- if language_use_types is not none and language_use_types | length -%} + {%- for language_use in language_use_types -%} , stu_language.{{ language_use }} as {{ language_use }} {%- endfor -%} {%- endif %} diff --git a/models/core_warehouse/dim_subgroup.sql b/models/core_warehouse/dim_subgroup.sql index 4d24b3a7..561ca73e 100644 --- a/models/core_warehouse/dim_subgroup.sql +++ b/models/core_warehouse/dim_subgroup.sql @@ -5,7 +5,7 @@ }} {# bring in the customizable column name for language use from dim_student, defaults as home_language #} -{% set language_use_name = var('edu:stu_language:language_uses') %} +{% set language_use_types = var('edu:stu_language:language_use_types', []) %} with dim_student as ( select * from {{ ref('dim_student') }} @@ -36,9 +36,8 @@ stu_long_subgroup as ( 'display_name', 'birth_date', 'race_array', - 'safe_display_name', - language_use_name - ], + 'safe_display_name' + ] + language_use_types, remove = stu_id_cols, field_name='subgroup_category', value_name='subgroup_value' From 58a4381f19361d72ec6fe9e9b1045eb5766df642 Mon Sep 17 00:00:00 2001 From: keenzarate213 Date: Thu, 21 Mar 2024 11:51:28 -0500 Subject: [PATCH 6/8] add additional fields include zip code, latitude, long, city, etc --- .../fct_student_contact_information.sql | 28 +++++++++++++++++-- .../core_warehouse/fct_student_subgroup.sql | 7 ++--- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/models/core_warehouse/fct_student_contact_information.sql b/models/core_warehouse/fct_student_contact_information.sql index cfa61a93..0e7e52b1 100644 --- a/models/core_warehouse/fct_student_contact_information.sql +++ b/models/core_warehouse/fct_student_contact_information.sql @@ -1,3 +1,7 @@ +{# configure address preferred #} +{% set preferred = 'Physical' %} + + with stu_phone_wide as ( select * from {{ ref('bld_ef3__student_wide_phone_numbers') }} ), @@ -9,6 +13,13 @@ stu_address_wide as ( ), dim_student as ( select * from {{ ref('dim_student') }} +), +choose_address as ( + {{ row_pluck(ref('stg_ef3__stu_ed_org__addresses'), + key='k_student', + column='address_type', + preferred=preferred, + where='address_end_date is null') }} ) select dim_student.k_student, @@ -29,8 +40,19 @@ select source_table='bld_ef3__student_wide_addresses', exclude_columns=["k_student", "tenant_code"], source_alias='stu_address_wide' - ) }} + ) }} + choose_address.city as {{preferred}}_address_city, + choose_address.name_of_county as {{preferred}}_address_name_of_county, + choose_address.state_code as {{preferred}}_address_state_code, + choose_address.postal_code as {{preferred}}_address_postal_code, + choose_address.building_site_number as {{preferred}}_address_building_site_number, + choose_address.locale as {{preferred}}_address_locale, + choose_address.congressional_district as {{preferred}}_address_congressional_district, + choose_address.county_fips_code as {{preferred}}_address_county_fips_code, + choose_address.latitude as {{preferred}}_address_latitude, + choose_address.longitude as {{preferred}}_address_longitude from dim_student -left join stu_phone_wide on dim_student.k_student = stu_phone_wide.k_student -left join stu_emails_wide on dim_student.k_student = stu_emails_wide.k_student +left join stu_phone_wide on dim_student.k_student = stu_phone_wide.k_student +left join stu_emails_wide on dim_student.k_student = stu_emails_wide.k_student left join stu_address_wide on dim_student.k_student = stu_address_wide.k_student +left join choose_address on stu_address_wide.k_student = choose_address.k_student diff --git a/models/core_warehouse/fct_student_subgroup.sql b/models/core_warehouse/fct_student_subgroup.sql index fddfa66f..cd2a3540 100644 --- a/models/core_warehouse/fct_student_subgroup.sql +++ b/models/core_warehouse/fct_student_subgroup.sql @@ -1,5 +1,5 @@ {# bring in the customizable column name for language use from dim_student, defaults as home_language #} -{% set language_use_name = var('edu:stu_language:language_uses') %} +{% set language_use_types = var('edu:stu_language:language_use_types', []) %} with dim_student as ( select * from {{ ref('dim_student') }} @@ -27,9 +27,8 @@ stu_long_subgroup as ( 'display_name', 'birth_date', 'race_array', - 'safe_display_name', - language_use_name - ], + 'safe_display_name' + ] + language_use_types, remove = stu_id_cols, field_name='subgroup_category', value_name='subgroup_value' From 84453bf6343b28ebbce11339ff6251770adc082b Mon Sep 17 00:00:00 2001 From: rlittle08 Date: Mon, 25 Mar 2024 17:26:06 -0500 Subject: [PATCH 7/8] remove default language (may break), add docs, add lang to contact --- dbt_project.yml | 4 ---- models/core_warehouse/dim_student.sql | 15 ++++++++----- .../fct_student_contact_information.sql | 11 +++++++++- .../fct_student_contact_information.yml | 22 +++++++++++++++++++ 4 files changed, 41 insertions(+), 11 deletions(-) create mode 100644 models/core_warehouse/fct_student_contact_information.yml diff --git a/dbt_project.yml b/dbt_project.yml index 41f18f71..fa1f5950 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -36,10 +36,6 @@ vars: 'edu:stu_demos:hispanic_latino_code': Latinx 'edu:stu_demos:race_unknown_code': Unknown - # default language use home language - 'edu:stu_language:language_use_types': - - home_language - # custom intersections of other demographic groupings 'edu:stu_demos:intersection_groups': Null # custom data sources for student characteristics. must contain and be unique by `k_student` diff --git a/models/core_warehouse/dim_student.sql b/models/core_warehouse/dim_student.sql index ae3d2acb..ceddf6ff 100644 --- a/models/core_warehouse/dim_student.sql +++ b/models/core_warehouse/dim_student.sql @@ -25,7 +25,8 @@ {% set custom_title_i_program_agg_indicators = var('edu:title_i:custom_program_agg_indicators', None) %} {# Load customizable column name for language use, defaults as home_language #} -{% set language_use_types = var('edu:stu_language:language_use_types', None) %} +{% set language_use_types = var('edu:stu_demos:language_use_types', None) %} +{% set address_types = var('edu:stu_demos:address_types', None) %} with stg_student as ( select * from {{ ref('stg_ef3__students') }} @@ -188,16 +189,18 @@ formatted as ( {%- endfor -%} {%- endif %} + -- student languages + {% if language_use_types is not none and language_use_types | length -%} + {%- for language_use in language_use_types -%} + stu_language.{{ language_use }}, + {%- endfor -%} + {%- endif %} + -- add indicator of most recent demographic entry stg_student.api_year = max(stg_student.api_year) over(partition by stg_student.k_student_xyear) as is_latest_record, stu_immutable_demos.race_array, stu_immutable_demos.safe_display_name - {%- if language_use_types is not none and language_use_types | length -%} - {%- for language_use in language_use_types -%} - , stu_language.{{ language_use }} as {{ language_use }} - {%- endfor -%} - {%- endif %} from stg_student join stu_demos diff --git a/models/core_warehouse/fct_student_contact_information.sql b/models/core_warehouse/fct_student_contact_information.sql index 0e7e52b1..c0a0abd2 100644 --- a/models/core_warehouse/fct_student_contact_information.sql +++ b/models/core_warehouse/fct_student_contact_information.sql @@ -11,6 +11,9 @@ stu_emails_wide as ( stu_address_wide as ( select * from {{ ref('bld_ef3__student_wide_addresses') }} ), +stu_language_wide as ( + select * from {{ ref('bld_ef3__student_wide_languages') }} +), dim_student as ( select * from {{ ref('dim_student') }} ), @@ -36,6 +39,11 @@ select exclude_columns=["k_student", "tenant_code"], source_alias='stu_emails_wide' ) }} + {{ accordion_columns( + source_table='bld_ef3__student_wide_languages', + exclude_columns=["k_student", "tenant_code"], + source_alias='stu_language_wide' + ) }} {{ accordion_columns( source_table='bld_ef3__student_wide_addresses', exclude_columns=["k_student", "tenant_code"], @@ -54,5 +62,6 @@ select from dim_student left join stu_phone_wide on dim_student.k_student = stu_phone_wide.k_student left join stu_emails_wide on dim_student.k_student = stu_emails_wide.k_student +left join stu_language_wide on dim_student.k_student = stu_language_wide.k_student left join stu_address_wide on dim_student.k_student = stu_address_wide.k_student -left join choose_address on stu_address_wide.k_student = choose_address.k_student +left join choose_address on stu_address_wide.k_student = choose_address.k_student \ No newline at end of file diff --git a/models/core_warehouse/fct_student_contact_information.yml b/models/core_warehouse/fct_student_contact_information.yml new file mode 100644 index 00000000..beaf1fbb --- /dev/null +++ b/models/core_warehouse/fct_student_contact_information.yml @@ -0,0 +1,22 @@ +version: 2 + +models: + - name: fct_student_contact_information + description: > + ##### Overview: + Student contact information, including address, email, phone, and language. + + ##### Primary Key: + `k_student` + + config: + tags: ['core'] + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - k_student + columns: + - name: k_student + - name: k_student_xyear + - name: tenant_code + - name: school_year \ No newline at end of file From 845ea0add28a8de449a5942a7273600eb1a8a25c Mon Sep 17 00:00:00 2001 From: rlittle08 Date: Mon, 25 Mar 2024 17:30:26 -0500 Subject: [PATCH 8/8] rm address types from dim stu --- models/core_warehouse/dim_student.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/core_warehouse/dim_student.sql b/models/core_warehouse/dim_student.sql index ceddf6ff..bb016486 100644 --- a/models/core_warehouse/dim_student.sql +++ b/models/core_warehouse/dim_student.sql @@ -26,8 +26,7 @@ {# Load customizable column name for language use, defaults as home_language #} {% set language_use_types = var('edu:stu_demos:language_use_types', None) %} -{% set address_types = var('edu:stu_demos:address_types', None) %} - + with stg_student as ( select * from {{ ref('stg_ef3__students') }} ),