From 6008c37c922f5610d54078b868b485903f5984b0 Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Wed, 23 Jul 2025 18:09:54 -0500 Subject: [PATCH 1/4] Create dim_candidate, dim_performance_evaluation, and fct_candidate_assessment --- models/tpdm_warehouse/dim_candidate.sql | 48 ++++++++++++++++++ .../dim_performance_evaluation.sql | 25 ++++++++++ .../fct_candidate_assessment.sql | 49 +++++++++++++++++++ 3 files changed, 122 insertions(+) create mode 100644 models/tpdm_warehouse/dim_candidate.sql create mode 100644 models/tpdm_warehouse/dim_performance_evaluation.sql create mode 100644 models/tpdm_warehouse/fct_candidate_assessment.sql diff --git a/models/tpdm_warehouse/dim_candidate.sql b/models/tpdm_warehouse/dim_candidate.sql new file mode 100644 index 00000000..26b06270 --- /dev/null +++ b/models/tpdm_warehouse/dim_candidate.sql @@ -0,0 +1,48 @@ +with stg_candidates as ( + select * from {{ ref('stg_tpdm__candidates') }} +), + +formatted as ( + select + k_candidate, + k_person, + tenant_code, + api_year, + candidate_id, + person_id, + first_name, + last_name, + middle_name, + maiden_name, + generation_code_suffix, + personal_title_prefix, + preferred_first_name, + preferred_last_name, + birth_city, + birth_date, + birth_international_province, + date_entered_us, + displacement_status, + is_economic_disadvantaged, + is_first_generation_student, + has_hispanic_latino_ethnicity, + is_multiple_birth, + gender, + sex, + birth_sex, + birth_state, + birth_country, + english_language_exam, + lep_code, + v_addresses, + v_disabilities, + v_emails, + v_languages, + v_other_names, + v_personal_identification_documents, + v_races, + v_telephones + from stg_candidates +) + +select * from formatted \ No newline at end of file diff --git a/models/tpdm_warehouse/dim_performance_evaluation.sql b/models/tpdm_warehouse/dim_performance_evaluation.sql new file mode 100644 index 00000000..7a67da25 --- /dev/null +++ b/models/tpdm_warehouse/dim_performance_evaluation.sql @@ -0,0 +1,25 @@ +with stg_performance_evaluations as ( + select * from {{ ref('stg_tpdm__performance_evaluations') }} +), + +stg_performance_evaluation_ratings as ( + select * from {{ ref('stg_tpdm__performance_evaluation_ratings') }} +), + +formatted as ( + select + stg_performance_evaluation_ratings.k_performance_evaluation, + stg_performance_evaluations.ed_org_id, + stg_performance_evaluations.performance_evaluation_title, + stg_performance_evaluations.performance_evaluation_type, + stg_performance_evaluations.school_year, + stg_performance_evaluations.academic_term, + stg_performance_evaluations.performance_evaluation_description, + stg_performance_evaluations.academic_subject, + stg_performance_evaluations.v_grade_levels, + stg_performance_evaluations.v_rating_levels + from stg_performance_evaluation_ratings + join stg_performance_evaluations + on stg_performance_evaluation_ratings.k_performance_evaluation = stg_performance_evaluations.k_performance_evaluation +) +select * from formatted \ No newline at end of file diff --git a/models/tpdm_warehouse/fct_candidate_assessment.sql b/models/tpdm_warehouse/fct_candidate_assessment.sql new file mode 100644 index 00000000..e205d4a4 --- /dev/null +++ b/models/tpdm_warehouse/fct_candidate_assessment.sql @@ -0,0 +1,49 @@ +{{ + config( + post_hook=[ + "alter table {{ this }} add primary key (k_candidate, k_performance_evaluation, k_person)", + "alter table {{ this }} add constraint fk_{{ this.name }}_candidate foreign key (k_candidate) references {{ ref('dim_candidate') }}", + "alter table {{ this }} add constraint fk_{{ this.name }}_person foreign key (k_person) references {{ ref('dim_candidate') }}", + "alter table {{ this }} add constraint fk_{{ this.name }}_performance_evaluation foreign key (k_performance_evaluation) references {{ ref('dim_performance_evaluation') }}", + ] + ) +}} + +with stg_performance_evaluation_ratings as ( + select * from {{ ref('stg_tpdm__performance_evaluation_ratings') }} +), + +dim_candidate as ( + select * from {{ ref('dim_candidate') }} +), + +dim_performance_evaluation as ( + select * from {{ ref('dim_performance_evaluation') }} +), + +formatted as ( + select + {{ dbt_utils.generate_surrogate_key( + ['dim_candidate.k_candidate', + 'dim_performance_evaluation.k_performance_evaluation', + 'dim_candidate.k_person'] + ) }} as k_candidate_assessment, + dim_candidate.k_candidate, + dim_candidate.k_person, + dim_performance_evaluation.k_performance_evaluation, + stg_performance_evaluation_ratings.results as rating_results, + stg_performance_evaluation_ratings.performance_evaluation_rating_level as rating_level, + stg_performance_evaluation_ratings.reviewers, + stg_performance_evaluation_ratings.coteaching_style_observed, + stg_performance_evaluation_ratings.comments, + stg_performance_evaluation_ratings.is_announced, + stg_performance_evaluation_ratings.schedule_date, + stg_performance_evaluation_ratings.actual_date, + stg_performance_evaluation_ratings.actual_duration, + from dim_candidate + join stg_performance_evaluation_ratings + on dim_candidate.k_person = stg_performance_evaluation_ratings.k_person + join dim_performance_evaluation + on stg_performance_evaluation_ratings.k_performance_evaluation = dim_performance_evaluation.k_performance_evaluation +) +select * from formatted \ No newline at end of file From 0413d09d95b819c16280130666df1a7d83582bfd Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Thu, 31 Jul 2025 15:04:36 -0500 Subject: [PATCH 2/4] Add primary key to dim tables --- models/tpdm_warehouse/dim_candidate.sql | 8 ++++++++ models/tpdm_warehouse/dim_performance_evaluation.sql | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/models/tpdm_warehouse/dim_candidate.sql b/models/tpdm_warehouse/dim_candidate.sql index 26b06270..fcc54314 100644 --- a/models/tpdm_warehouse/dim_candidate.sql +++ b/models/tpdm_warehouse/dim_candidate.sql @@ -1,3 +1,11 @@ +{{ + config( + post_hook=[ + "alter table {{ this }} add primary key (k_candidate)" + ] + ) +}} + with stg_candidates as ( select * from {{ ref('stg_tpdm__candidates') }} ), diff --git a/models/tpdm_warehouse/dim_performance_evaluation.sql b/models/tpdm_warehouse/dim_performance_evaluation.sql index 7a67da25..64a9e1c5 100644 --- a/models/tpdm_warehouse/dim_performance_evaluation.sql +++ b/models/tpdm_warehouse/dim_performance_evaluation.sql @@ -1,3 +1,11 @@ +{{ + config( + post_hook=[ + "alter table {{ this }} add primary key (k_performance_evaluation)" + ] + ) +}} + with stg_performance_evaluations as ( select * from {{ ref('stg_tpdm__performance_evaluations') }} ), From e17ec3f29895c0bf9f5f3450817b9c526a526cee Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Tue, 5 Aug 2025 09:43:00 -0500 Subject: [PATCH 3/4] Create yml for warehouse tables --- models/tpdm_warehouse/dim_candidate.yml | 69 +++++++++++++++++++ .../dim_performance_evaluation.yml | 35 ++++++++++ .../fct_candidate_assessment.yml | 43 ++++++++++++ 3 files changed, 147 insertions(+) create mode 100644 models/tpdm_warehouse/dim_candidate.yml create mode 100644 models/tpdm_warehouse/dim_performance_evaluation.yml create mode 100644 models/tpdm_warehouse/fct_candidate_assessment.yml diff --git a/models/tpdm_warehouse/dim_candidate.yml b/models/tpdm_warehouse/dim_candidate.yml new file mode 100644 index 00000000..23e2edd6 --- /dev/null +++ b/models/tpdm_warehouse/dim_candidate.yml @@ -0,0 +1,69 @@ +version: 2 + +models: + - name: dim_candidate + description: > + ##### Overview: + Defines an educator candidate. + + ##### Primary Key: + `k_candidate` - There is one record per candidate + config: + tags: ['tpdm'] + columns: + - name: k_candidate + description: generated primary key from `tenant_code`, `api_year`, and `candidate_id` + tests: + - unique + - name: k_person + description: "Generated surrogate key from `tenant_code`, `api_year`, `person_id`, and `source_system`" + - name: tenant_code + description: "Code defining the Tenant (may be an LEA, SEA, etc.) of the Ed-Fi ODS from which this record was pulled" + - name: api_year + - name: candidate_id + description: A unique alphanumeric code assigned to a candidate + - name: person_id + description: A unique alphanumeric code assigned to a person + - name: first_name + - name: last_name + - name: middle_name + - name: maiden_name + - name: generation_code_suffix + - name: personal_title_prefix + - name: preferred_first_name + - name: preferred_last_name + - name: birth_city + - name: birth_date + - name: birth_international_province + - name: date_entered_us + - name: displacement_status + - name: is_economic_disadvantaged + - name: is_first_generation_student + - name: has_hispanic_latino_ethnicity + - name: is_multiple_birth + - name: gender + - name: sex + - name: birth_sex + - name: birth_state + - name: birth_country + description: xxxx + - name: english_language_exam + description: "Indicates that a person passed, failed, or did not take an English Language assessment (e.g., TOEFFL)." + - name: lep_code + description: "An indication that the student has been identified as limited English proficient by the Language Proficiency Assessment Committee (LPAC), or English proficient." + - name: v_addresses + description: The set of elements that describes an address, including the street address, city, state, and ZIP code. + - name: v_disabilities + description: "The disability condition(s) that best describes an individual's impairment." + - name: v_emails + description: The numbers, letters, and symbols used to identify an electronic mail (e-mail) user within the network to which the individual or organization belongs. + - name: v_languages + description: "The language(s) the individual uses to communicate" + - name: v_other_names + description: "Other names (e.g., alias, nickname, previous legal name) associated with a person." + - name: v_personal_identification_documents + description: Describes the documentation of citizenship. + - name: v_races + description: "The general racial category which most clearly reflects the individual's recognition of his or her community or with which the individual most identifies. The data model allows for multiple entries so that each individual can specify all appropriate races." + - name: v_telephones + description: "The 10-digit telephone number, including the area code, for the person." \ No newline at end of file diff --git a/models/tpdm_warehouse/dim_performance_evaluation.yml b/models/tpdm_warehouse/dim_performance_evaluation.yml new file mode 100644 index 00000000..c70048d3 --- /dev/null +++ b/models/tpdm_warehouse/dim_performance_evaluation.yml @@ -0,0 +1,35 @@ +version: 2 + +models: + - name: dim_performance_evaluation + description: > + ##### Overview: + Defines a performance evaluation of an educator. + + ##### Primary Key: + `k_performance_evaluation` - There is one record per performance evaluation + config: + tags: ['tpdm'] + columns: + - name: k_performance_evaluation + description: Generated surrogate key from [tenant_code, api_year, ed_org_id, evaluation_period, performance_evaluation_title, performance_evaluation_type, school_year, academic_term] + tests: + - unique + - name: ed_org_id + description: The identifier assigned to an education organization. + - name: performance_evaluation_title + description: An assigned unique identifier for the performance evaluation. + - name: performance_evaluation_type + description: "The type (e.g., walkthrough, summative) of performance evaluation conducted." + - name: school_year + description: The identifier for the school year. + - name: academic_term + description: The term for the session during the school year. + - name: performance_evaluation_description + description: The long description of the Performance Evaluation. + - name: academic_subject + description: The description of the content or subject area of a performance evaluation. + - name: v_grade_levels + description: The grade levels involved with the performance evaluation. + - name: v_rating_levels + description: "The descriptive level(s) of ratings (cut scores) for the evaluation." diff --git a/models/tpdm_warehouse/fct_candidate_assessment.yml b/models/tpdm_warehouse/fct_candidate_assessment.yml new file mode 100644 index 00000000..285820d2 --- /dev/null +++ b/models/tpdm_warehouse/fct_candidate_assessment.yml @@ -0,0 +1,43 @@ +version: 2 + +models: + - name: fct_candidate assessment + description: > + ##### Overview: + Defines a performance-based assessment of an educator candidate + + ##### Primary Key: + `k_candidate_assessment` - There is one record per performance evaluation for a given candidate. This is a generated surrogate key + from [k_candidate, k_performance_evaluation, k_person] + config: + tags: ['tpdm'] + columns: + - name: k_candidate_assessment + description: Generated surrogate key from `k_candidate`, `k_performance_evaluation`, and `k_person` + tests: + - unique + - name: k_candidate + description: Foreign key to dim_candidate, generated from `tenant_code`, `api_year`, and `candidate_id` + - name: k_person + description: "Generated surrogate key from `tenant_code`, `api_year`, `person_id`, and `source_system`" + - name: k_performance_evaluation + description: Foreign key to dim_performance_evaluation, generated from [tenant_code, api_year, ed_org_id, evaluation_period, performance_evaluation_title, performance_evaluation_type, school_year, academic_term] + - name: rating_results + description: The numerical summary rating or score for the performance evaluation. + - name: rating_level + description: The rating level achieved based upon the rating or score. + - name: reviewers + description: The person(s) that conducted the performance evaluation. + - name: coteaching_style_observed + description: A type of co-teaching observed as part of the performance evaluation. + - name: comments + description: Any comments about the performance evaluation to be captured. + - name: is_announced + description: An indicator of whether the performance evaluation was announced or not. + - name: schedule_date + description: The month, day, and year on which the performance evaluation was to be conducted. + - name: actual_date + description: The month, day, and year on which the performance evaluation was conducted. + - name: actual_duration + description: The actual or estimated number of clock minutes during which the performance evaluation was conducted. + From f766569675e5c47bff917b4de3ef180e43d6b90f Mon Sep 17 00:00:00 2001 From: oscarrezab Date: Fri, 8 Aug 2025 09:33:27 -0500 Subject: [PATCH 4/4] Remove key from fact table and fix join in dim --- models/tpdm_warehouse/dim_performance_evaluation.sql | 8 ++++---- models/tpdm_warehouse/fct_candidate_assessment.sql | 11 +++-------- models/tpdm_warehouse/fct_candidate_assessment.yml | 7 +------ 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/models/tpdm_warehouse/dim_performance_evaluation.sql b/models/tpdm_warehouse/dim_performance_evaluation.sql index 64a9e1c5..6a359f00 100644 --- a/models/tpdm_warehouse/dim_performance_evaluation.sql +++ b/models/tpdm_warehouse/dim_performance_evaluation.sql @@ -16,7 +16,7 @@ stg_performance_evaluation_ratings as ( formatted as ( select - stg_performance_evaluation_ratings.k_performance_evaluation, + stg_performance_evaluations.k_performance_evaluation, stg_performance_evaluations.ed_org_id, stg_performance_evaluations.performance_evaluation_title, stg_performance_evaluations.performance_evaluation_type, @@ -26,8 +26,8 @@ formatted as ( stg_performance_evaluations.academic_subject, stg_performance_evaluations.v_grade_levels, stg_performance_evaluations.v_rating_levels - from stg_performance_evaluation_ratings - join stg_performance_evaluations - on stg_performance_evaluation_ratings.k_performance_evaluation = stg_performance_evaluations.k_performance_evaluation + from stg_performance_evaluations + join stg_performance_evaluation_ratings + on stg_performance_evaluations.k_performance_evaluation = stg_performance_evaluation_ratings.k_performance_evaluation ) select * from formatted \ No newline at end of file diff --git a/models/tpdm_warehouse/fct_candidate_assessment.sql b/models/tpdm_warehouse/fct_candidate_assessment.sql index e205d4a4..3eb1ff06 100644 --- a/models/tpdm_warehouse/fct_candidate_assessment.sql +++ b/models/tpdm_warehouse/fct_candidate_assessment.sql @@ -23,11 +23,6 @@ dim_performance_evaluation as ( formatted as ( select - {{ dbt_utils.generate_surrogate_key( - ['dim_candidate.k_candidate', - 'dim_performance_evaluation.k_performance_evaluation', - 'dim_candidate.k_person'] - ) }} as k_candidate_assessment, dim_candidate.k_candidate, dim_candidate.k_person, dim_performance_evaluation.k_performance_evaluation, @@ -40,9 +35,9 @@ formatted as ( stg_performance_evaluation_ratings.schedule_date, stg_performance_evaluation_ratings.actual_date, stg_performance_evaluation_ratings.actual_duration, - from dim_candidate - join stg_performance_evaluation_ratings - on dim_candidate.k_person = stg_performance_evaluation_ratings.k_person + from stg_performance_evaluation_ratings + join dim_candidate + on stg_performance_evaluation_ratings.k_person = dim_candidate.k_person join dim_performance_evaluation on stg_performance_evaluation_ratings.k_performance_evaluation = dim_performance_evaluation.k_performance_evaluation ) diff --git a/models/tpdm_warehouse/fct_candidate_assessment.yml b/models/tpdm_warehouse/fct_candidate_assessment.yml index 285820d2..ad85165d 100644 --- a/models/tpdm_warehouse/fct_candidate_assessment.yml +++ b/models/tpdm_warehouse/fct_candidate_assessment.yml @@ -7,15 +7,10 @@ models: Defines a performance-based assessment of an educator candidate ##### Primary Key: - `k_candidate_assessment` - There is one record per performance evaluation for a given candidate. This is a generated surrogate key - from [k_candidate, k_performance_evaluation, k_person] + `k_candidate`, `k_performance_evaluation`, and `k_person` - There is one record per performance evaluation for a given candidate, who is a type of person config: tags: ['tpdm'] columns: - - name: k_candidate_assessment - description: Generated surrogate key from `k_candidate`, `k_performance_evaluation`, and `k_person` - tests: - - unique - name: k_candidate description: Foreign key to dim_candidate, generated from `tenant_code`, `api_year`, and `candidate_id` - name: k_person