test/custom-naming

.buildkite/scripts/run_models.sh

@@ -22,4 +22,4 @@ dbt test --target "$db"
 dbt run --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db" --full-refresh
 dbt test --vars '{amazon_ads__portfolio_history_enabled: false}' --target "$db"
 
-dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
+dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"

integration_tests/dbt_project.yml

@@ -19,6 +19,14 @@ vars:
     amazon_ads_keyword_history_identifier: "keyword_history_data"
     amazon_ads_targeting_keyword_report_identifier: "targeting_keyword_report_data"
     amazon_ads_search_term_ad_keyword_report_identifier: "search_term_ad_keyword_report_data"
+  amazon_ads_using_custom_names: true
+
+  amazon_ads_custom_column_names:
+    ad_group_history:
+      creation_date: creationDate
+      default_bid: defaultBid
+    ad_group_level_report:
+      campaign_bidding_strategy: campaignBiddingStrategy
 
   amazon_ads__campaign_passthrough_metrics:
       - name: sales_7_d

integration_tests/seeds/ad_group_history_data.csv

@@ -1,3 +1,3 @@
-id,last_updated_date,_fivetran_synced,campaign_id,creation_date,default_bid,name,serving_status,state
+id,last_updated_date,_fivetran_synced,campaign_id,creationDate,defaultBid,name,serving_status,state
 421,2022-07-11 22:38:16.551000,2022-12-13 17:10:16.297000,2187,2022-07-11 22:38:16.551000,1.75,Red 7,CAMPAIGN_PAUSED,enabled
 501,2022-11-09 14:37:05.332000,2022-12-13 17:11:08.594000,5555,2022-11-09 14:37:05.332000,1.45,All,AD_GROUP_STATUS_ENABLED,enabled

integration_tests/seeds/ad_group_level_report_data.csv

@@ -1,4 +1,4 @@
-ad_group_id,date,_fivetran_synced,campaign_bidding_strategy,clicks,cost,impressions
+ad_group_id,date,_fivetran_synced,campaignBiddingStrategy,clicks,cost,impressions
 501,2022-12-03,2022-12-13 11:35:56.905000,legacy,0,0.0,88
 501,2022-12-04,2022-12-13 11:35:56.908000,legacy,0,0.0,111
 501,2022-12-01,2022-12-13 11:35:56.899000,legacy,0,0.0,252

macros/staging/get_ad_group_history_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_ad_group_history_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "campaign_id", "datatype": dbt.type_int()},
     {"name": "creation_date", "datatype": dbt.type_timestamp()},
     {"name": "default_bid", "datatype": dbt.type_float()},
@@ -11,6 +11,8 @@
     {"name": "state", "datatype": dbt.type_string()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('ad_group_history', _fivetran_columns) %}
+
 {{ return(columns) }}
 
 {% endmacro %}

macros/staging/get_ad_group_level_report_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_ad_group_level_report_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "ad_group_id", "datatype": dbt.type_int()},
     {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()},
     {"name": "clicks", "datatype": dbt.type_int()},
@@ -11,6 +11,8 @@
     {"name": "sales_30_d", "datatype": dbt.type_float()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('ad_group_level_report', _fivetran_columns) %}
+
 {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
 {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__ad_group_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
 

macros/staging/get_advertised_product_report_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_advertised_product_report_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "ad_group_id", "datatype": dbt.type_int()},
     {"name": "ad_id", "datatype": dbt.type_int()},
     {"name": "advertised_asin", "datatype": dbt.type_string()},
@@ -17,6 +17,8 @@
     {"name": "sales_30_d", "datatype": dbt.type_float()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('advertised_product_report', _fivetran_columns) %}
+
 {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
 {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__advertised_product_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
 

macros/staging/get_campaign_history_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_campaign_history_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "bidding_strategy", "datatype": dbt.type_string()},
     {"name": "creation_date", "datatype": dbt.type_timestamp()},
     {"name": "end_date", "datatype": "date"},
@@ -18,6 +18,8 @@
     {"name": "effective_budget", "datatype": dbt.type_float()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('campaign_history', _fivetran_columns) %}
+
 {{ return(columns) }}
 
 {% endmacro %}

macros/staging/get_campaign_level_report_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_campaign_level_report_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "campaign_applicable_budget_rule_id", "datatype": dbt.type_string()},
     {"name": "campaign_applicable_budget_rule_name", "datatype": dbt.type_string()},
     {"name": "campaign_bidding_strategy", "datatype": dbt.type_string()},
@@ -17,6 +17,8 @@
     {"name": "sales_30_d", "datatype": dbt.type_float()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('campaign_level_report', _fivetran_columns) %}
+
 {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
 {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__campaign_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
 

macros/staging/get_keyword_history_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_keyword_history_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "ad_group_id", "datatype": dbt.type_int()},
     {"name": "bid", "datatype": dbt.type_float()},
     {"name": "campaign_id", "datatype": dbt.type_int()},
@@ -15,6 +15,8 @@
     {"name": "native_language_locale", "datatype": dbt.type_string()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('keyword_history', _fivetran_columns) %}
+
 {{ return(columns) }}
 
 {% endmacro %}

macros/staging/get_portfolio_history_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_portfolio_history_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "budget_amount", "datatype": dbt.type_float()},
     {"name": "budget_currency_code", "datatype": dbt.type_string()},
     {"name": "budget_end_date", "datatype": "date"},
@@ -16,6 +16,8 @@
     {"name": "state", "datatype": dbt.type_string()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('portfolio_history', _fivetran_columns) %}
+
 {{ return(columns) }}
 
 {% endmacro %}

macros/staging/get_product_ad_history_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_product_ad_history_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "ad_group_id", "datatype": dbt.type_int()},
     {"name": "asin", "datatype": dbt.type_string()},
     {"name": "campaign_id", "datatype": dbt.type_int()},
@@ -12,6 +12,8 @@
     {"name": "state", "datatype": dbt.type_string()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('product_ad_history', _fivetran_columns) %}
+
 {{ return(columns) }}
 
 {% endmacro %}

macros/staging/get_profile_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_profile_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "id", "datatype": dbt.type_int()},
     {"name": "account_id", "datatype": dbt.type_string()},
     {"name": "account_marketplace_string_id", "datatype": dbt.type_string()},
@@ -15,6 +15,8 @@
     {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('profile', _fivetran_columns) %}
+
 {{ return(columns) }}
 
 {% endmacro %}

macros/staging/get_search_term_ad_keyword_report_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_search_term_ad_keyword_report_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "ad_group_id", "datatype": dbt.type_int()},
     {"name": "ad_keyword_status", "datatype": dbt.type_string()},
     {"name": "campaign_budget_amount", "datatype": dbt.type_float()},
@@ -19,6 +19,8 @@
     {"name": "sales_30_d", "datatype": dbt.type_float()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('search_term_ad_keyword_report', _fivetran_columns) %}
+
 {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
 {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__search_term_ad_keyword_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
 

macros/staging/get_targeting_keyword_report_columns.sql

@@ -1,6 +1,6 @@
 {% macro get_targeting_keyword_report_columns() %}
 
-{% set columns = [
+{% set _fivetran_columns = [
     {"name": "ad_group_id", "datatype": dbt.type_int()},
     {"name": "ad_keyword_status", "datatype": dbt.type_string()},
     {"name": "campaign_budget_amount", "datatype": dbt.type_float()},
@@ -20,6 +20,8 @@
     {"name": "sales_30_d", "datatype": dbt.type_float()}
 ] %}
 
+{% set columns = amazon_ads.resolve_column_names('targeting_keyword_report', _fivetran_columns) %}
+
 {# Add backwards compatibility if conversion metrics were added via passthrough columns prior to them being brought in by default #}
 {{ amazon_ads_add_pass_through_columns(base_columns=columns, pass_through_fields=var('amazon_ads__targeting_keyword_passthrough_metrics'), except_fields=['purchases_30_d', 'sales_30_d']) }}
 

macros/staging/resolve_column_names.sql

@@ -0,0 +1,22 @@
+{% macro resolve_column_names(_fivetran_table_name, _fivetran_columns, package_name='amazon_ads') %}
+    {{ return(adapter.dispatch('resolve_column_names', 'amazon_ads')(_fivetran_table_name, _fivetran_columns, package_name)) }}
+{% endmacro %}
+
+{% macro default__resolve_column_names(_fivetran_table_name, _fivetran_columns, package_name) %}
+
+{% if var(package_name ~ '_using_custom_names', false) %}
+    {% set resolved_column_names = [] %}
+    {% set custom_column_names = var(package_name ~ '_custom_column_names', {}) %}
+
+    {% for column in _fivetran_columns %}
+        {% set column_name = column.name %}
+        {# Use the custom name if it exists, otherwise use the original column name #}
+        {% set resolved_name = custom_column_names.get(_fivetran_table_name, {}).get(column_name, column_name) %}
+        {% set updated_column = dict(column, name=resolved_name, alias=column_name) %}
+        {% do resolved_column_names.append(updated_column) %}
+    {% endfor %}
+{% endif %}
+
+{{ return(resolved_column_names) }}
+
+{% endmacro %}