diff --git a/.circleci/config.yml b/.circleci/config.yml index 4635df2f..cc353373 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: integration-redshift: docker: - - image: circleci/python:3.6.3-stretch + - image: circleci/python:3.6.13-stretch steps: - checkout - run: @@ -19,7 +19,7 @@ jobs: integration-snowflake: docker: - - image: circleci/python:3.6.3-stretch + - image: circleci/python:3.6.13-stretch steps: - checkout - run: @@ -32,7 +32,7 @@ jobs: environment: BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json" docker: - - image: circleci/python:3.6.3-stretch + - image: circleci/python:3.6.13-stretch steps: - checkout - run: diff --git a/README.md b/README.md index 5e732b39..fbe4357b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # External sources in dbt -dbt v0.15.0 added support for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties. +dbt v0.15.0 [added support](https://github.com/dbt-labs/dbt/pull/1784) for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties. This package provides: * Macros to create/replace external tables and refresh their partitions, using the metadata provided in your `.yml` file source definitions @@ -19,7 +19,7 @@ This package provides: ## Installation -Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/fishtown-analytics/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`. +Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/dbt-labs/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`. ## Syntax @@ -115,9 +115,9 @@ execute the appropriate `create`, `refresh`, and/or `drop` commands: `stage_external_sources` runs as an operation * [`tested specs`](integration_tests/models/plugins): source spec variations that are confirmed to work on each database, via integration tests -If you encounter issues using this package or have questions, please check the [open issues](https://github.com/fishtown-analytics/dbt-external-tables/issues), as there's a chance it's a known limitation or work in progress. If not, you can: +If you encounter issues using this package or have questions, please check the [open issues](https://github.com/dbt-labs/dbt-external-tables/issues), as there's a chance it's a known limitation or work in progress. If not, you can: - open a new issue to report a bug or suggest an enhancement - post a technical question to [StackOverflow](https://stackoverflow.com/questions/tagged/dbt) - post a conceptual question to the relevant database channel (#db-redshift, #dbt-snowflake, etc) in the [dbt Slack community](https://community.getdbt.com/) -Additional contributions to this package are very welcome! Please create issues or open PRs against `master`. Check out [this post](https://discourse.getdbt.com/t/contributing-to-an-external-dbt-package/657) on the best workflow for contributing to a package. +Additional contributions to this package are very welcome! Please create issues or open PRs against `master`. Check out [this post](https://discourse.getdbt.com/t/contributing-to-an-external-dbt-package/657) on the best workflow for contributing to a package. \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index a147dfe3..44e0151a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,9 +1,9 @@ name: 'dbt_external_tables' -version: '0.3.0' +version: '0.7.0' config-version: 2 -require-dbt-version: ">=0.18.0" +require-dbt-version: ">=0.20.0" source-paths: ["models"] analysis-paths: ["analysis"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 2524779d..2bb3ce2d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -6,10 +6,7 @@ profile: 'integration_tests' config-version: 2 -source-paths: - - "sources/common" - - "sources/plugins/{{ target.type }}" - +source-paths: ["models"] analysis-paths: ["analysis"] test-paths: ["tests"] data-paths: ["data"] @@ -20,8 +17,25 @@ clean-targets: - "target" - "dbt_modules" -vars: - dbt_external_tables_dispatch_list: ['dbt_external_tables_integration_tests'] +dispatch: + - macro_namespace: dbt_external_tables + search_order: ['dbt_external_tables_integration_tests', 'dbt_external_tables'] seeds: +quote_columns: false + +sources: + dbt_external_tables_integration_tests: + plugins: + redshift_external: + +enabled: "{{ target.type == 'redshift' }}" + snowflake_external: + +enabled: "{{ target.type == 'snowflake' }}" + bigquery_external: + +enabled: "{{ target.type == 'bigquery' }}" + spark_external: + +enabled: "{{ target.type == 'spark' }}" + synapse_external: + +enabled: "{{ target.type == 'synapse' }}" + azuresql_external: + +enabled: "{{ target.type == 'sqlserver' }}" diff --git a/integration_tests/macros/common/prep_external.sql b/integration_tests/macros/common/prep_external.sql index 0430e4a6..de754c7e 100644 --- a/integration_tests/macros/common/prep_external.sql +++ b/integration_tests/macros/common/prep_external.sql @@ -1,5 +1,5 @@ {% macro prep_external() %} - {{ return(adapter.dispatch('prep_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }} + {{ return(adapter.dispatch('prep_external', 'dbt_external_tables')()) }} {% endmacro %} {% macro default__prep_external() %} diff --git a/integration_tests/sources/common/control.yml b/integration_tests/models/common/control.yml similarity index 100% rename from integration_tests/sources/common/control.yml rename to integration_tests/models/common/control.yml diff --git a/integration_tests/sources/plugins/sqlserver/azuresql_external.yml b/integration_tests/models/plugins/azuresql_external.yml similarity index 100% rename from integration_tests/sources/plugins/sqlserver/azuresql_external.yml rename to integration_tests/models/plugins/azuresql_external.yml diff --git a/integration_tests/sources/plugins/bigquery/bigquery_external.yml b/integration_tests/models/plugins/bigquery_external.yml similarity index 100% rename from integration_tests/sources/plugins/bigquery/bigquery_external.yml rename to integration_tests/models/plugins/bigquery_external.yml diff --git a/integration_tests/sources/plugins/redshift/redshift_external.yml b/integration_tests/models/plugins/redshift_external.yml similarity index 100% rename from integration_tests/sources/plugins/redshift/redshift_external.yml rename to integration_tests/models/plugins/redshift_external.yml diff --git a/integration_tests/sources/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake_external.yml similarity index 100% rename from integration_tests/sources/plugins/snowflake/snowflake_external.yml rename to integration_tests/models/plugins/snowflake_external.yml diff --git a/integration_tests/sources/plugins/spark/spark_external.yml b/integration_tests/models/plugins/spark_external.yml similarity index 100% rename from integration_tests/sources/plugins/spark/spark_external.yml rename to integration_tests/models/plugins/spark_external.yml diff --git a/integration_tests/sources/plugins/synapse/synapse_external.yml b/integration_tests/models/plugins/synapse_external.yml similarity index 100% rename from integration_tests/sources/plugins/synapse/synapse_external.yml rename to integration_tests/models/plugins/synapse_external.yml diff --git a/macros/common/create_external_table.sql b/macros/common/create_external_table.sql index f7256c21..d4f6b1cd 100644 --- a/macros/common/create_external_table.sql +++ b/macros/common/create_external_table.sql @@ -1,7 +1,5 @@ {% macro create_external_table(source_node) %} - {{ adapter.dispatch('create_external_table', - packages = dbt_external_tables._get_dbt_external_tables_namespaces()) - (source_node) }} + {{ adapter.dispatch('create_external_table', 'dbt_external_tables')(source_node) }} {% endmacro %} {% macro default__create_external_table(source_node) %} diff --git a/macros/common/get_external_build_plan.sql b/macros/common/get_external_build_plan.sql index 240385fa..d94d7e01 100644 --- a/macros/common/get_external_build_plan.sql +++ b/macros/common/get_external_build_plan.sql @@ -1,7 +1,5 @@ {% macro get_external_build_plan(source_node) %} - {{ return(adapter.dispatch('get_external_build_plan', - packages = dbt_external_tables._get_dbt_external_tables_namespaces()) - (source_node)) }} + {{ return(adapter.dispatch('get_external_build_plan', 'dbt_external_tables')(source_node)) }} {% endmacro %} {% macro default__get_external_build_plan(source_node) %} diff --git a/macros/common/helpers/dropif.sql b/macros/common/helpers/dropif.sql index 1fcca20b..dfb91065 100644 --- a/macros/common/helpers/dropif.sql +++ b/macros/common/helpers/dropif.sql @@ -1,7 +1,5 @@ {% macro dropif(node) %} - {{ adapter.dispatch('dropif', - packages = dbt_external_tables._get_dbt_external_tables_namespaces()) - (node) }} + {{ adapter.dispatch('dropif', 'dbt_external_tables')(node) }} {% endmacro %} {% macro default__dropif() %} diff --git a/macros/common/helpers/get_external_tables_namespace.sql b/macros/common/helpers/get_external_tables_namespace.sql deleted file mode 100644 index 4dd60ca1..00000000 --- a/macros/common/helpers/get_external_tables_namespace.sql +++ /dev/null @@ -1,4 +0,0 @@ -{% macro _get_dbt_external_tables_namespaces() %} - {% set override_namespaces = var('dbt_external_tables_dispatch_list', []) %} - {% do return(override_namespaces + ['dbt_external_tables']) %} -{% endmacro %} diff --git a/macros/common/helpers/transaction.sql b/macros/common/helpers/transaction.sql index 569108ec..89124859 100644 --- a/macros/common/helpers/transaction.sql +++ b/macros/common/helpers/transaction.sql @@ -1,5 +1,5 @@ {% macro exit_transaction() %} - {{ return(adapter.dispatch('exit_transaction', dbt_external_tables._get_dbt_external_tables_namespaces())()) }} + {{ return(adapter.dispatch('exit_transaction', 'dbt_external_tables')()) }} {% endmacro %} {% macro default__exit_transaction() %} diff --git a/macros/common/refresh_external_table.sql b/macros/common/refresh_external_table.sql index 1b9748be..6b4ca53e 100644 --- a/macros/common/refresh_external_table.sql +++ b/macros/common/refresh_external_table.sql @@ -1,7 +1,5 @@ {% macro refresh_external_table(source_node) %} - {{ return(adapter.dispatch('refresh_external_table', - packages = dbt_external_tables._get_dbt_external_tables_namespaces()) - (source_node)) }} + {{ return(adapter.dispatch('refresh_external_table', 'dbt_external_tables')(source_node)) }} {% endmacro %} {% macro default__refresh_external_table(source_node) %} diff --git a/macros/common/stage_external_sources.sql b/macros/common/stage_external_sources.sql index de17b21c..21d885cd 100644 --- a/macros/common/stage_external_sources.sql +++ b/macros/common/stage_external_sources.sql @@ -49,7 +49,7 @@ {% for q in run_queue %} - {% set q_msg = q|trim %} + {% set q_msg = q|replace('\n','')|replace('begin;','')|trim %} {% set q_log = q_msg[:50] ~ '... ' if q_msg|length > 50 else q_msg %} {% do dbt_utils.log_info(loop_label ~ ' (' ~ loop.index ~ ') ' ~ q_log) %} diff --git a/macros/plugins/bigquery/create_external_table.sql b/macros/plugins/bigquery/create_external_table.sql index 1adfe9ef..8676ea40 100644 --- a/macros/plugins/bigquery/create_external_table.sql +++ b/macros/plugins/bigquery/create_external_table.sql @@ -30,7 +30,11 @@ uris = [{%- for uri in uris -%} '{{uri}}' {{- "," if not loop.last}} {%- endfor -%}] {%- if options is mapping -%} {%- for key, value in options.items() if key != 'uris' %} + {%- if value is string -%} + , {{key}} = '{{value}}' + {%- else -%} , {{key}} = {{value}} + {%- endif -%} {%- endfor -%} {%- endif -%} ) diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql index da7da16f..3956f3b4 100644 --- a/macros/plugins/snowflake/create_external_table.sql +++ b/macros/plugins/snowflake/create_external_table.sql @@ -27,7 +27,8 @@ {%- endif -%} {% if partitions %} partition by ({{partitions|map(attribute='name')|join(', ')}}) {% endif %} location = {{external.location}} {# stage #} - {% if external.auto_refresh -%} auto_refresh = {{external.auto_refresh}} {%- endif %} + {% if external.auto_refresh is not none -%} auto_refresh = {{external.auto_refresh}} {%- endif %} {% if external.pattern -%} pattern = '{{external.pattern}}' {%- endif %} + {% if external.integration -%} integration = '{{external.integration}}' {%- endif %} file_format = {{external.file_format}} {% endmacro %} diff --git a/macros/plugins/snowflake/get_external_build_plan.sql b/macros/plugins/snowflake/get_external_build_plan.sql index ec930b0d..0f73a0b2 100644 --- a/macros/plugins/snowflake/get_external_build_plan.sql +++ b/macros/plugins/snowflake/get_external_build_plan.sql @@ -15,7 +15,7 @@ {% if create_or_replace %} {% set build_plan = build_plan + [ dbt_external_tables.snowflake_create_empty_table(source_node), - dbt_external_tables.snowflake_get_copy_sql(source_node), + dbt_external_tables.snowflake_get_copy_sql(source_node, explicit_transaction=true), dbt_external_tables.snowflake_create_snowpipe(source_node) ] %} {% else %} diff --git a/macros/plugins/snowflake/refresh_external_table.sql b/macros/plugins/snowflake/refresh_external_table.sql index 79eab423..8f28ae57 100644 --- a/macros/plugins/snowflake/refresh_external_table.sql +++ b/macros/plugins/snowflake/refresh_external_table.sql @@ -11,7 +11,9 @@ {% if manual_refresh %} {% set ddl %} - alter external table {{source(source_node.source_name, source_node.name)}} refresh + begin; + alter external table {{source(source_node.source_name, source_node.name)}} refresh; + commit; {% endset %} {% do return([ddl]) %} diff --git a/macros/plugins/snowflake/snowpipe/get_copy_sql.sql b/macros/plugins/snowflake/snowpipe/get_copy_sql.sql index b313fbc6..b4cdc42e 100644 --- a/macros/plugins/snowflake/snowpipe/get_copy_sql.sql +++ b/macros/plugins/snowflake/snowpipe/get_copy_sql.sql @@ -1,4 +1,4 @@ -{% macro snowflake_get_copy_sql(source_node) %} +{% macro snowflake_get_copy_sql(source_node, explicit_transaction=false) %} {# This assumes you have already created an external stage #} {%- set columns = source_node.columns.values() -%} @@ -6,6 +6,8 @@ {%- set is_csv = dbt_external_tables.is_csv(external.file_format) %} {%- set copy_options = external.snowpipe.get('copy_options', none) -%} + {%- if explicit_transaction -%} begin; {%- endif %} + copy into {{source(source_node.source_name, source_node.name)}} from ( select @@ -27,6 +29,8 @@ from {{external.location}} {# stage #} ) file_format = {{external.file_format}} - {% if copy_options %} {{copy_options}} {% endif %} + {% if copy_options %} {{copy_options}} {% endif %}; + + {% if explicit_transaction -%} commit; {%- endif -%} {% endmacro %} diff --git a/macros/plugins/spark/create_external_table.sql b/macros/plugins/spark/create_external_table.sql index b36741d7..a36a2153 100644 --- a/macros/plugins/spark/create_external_table.sql +++ b/macros/plugins/spark/create_external_table.sql @@ -13,7 +13,7 @@ {{- ',' if not loop.last -}} {% endfor %} ) {% endif -%} - {% if external.using -%} using {{external.using}} {%- endif %} + {% if external.using %} using {{external.using}} {%- endif %} {% if options -%} options ( {%- for key, value in options.items() -%} '{{ key }}' = '{{value}}' {{- ', \n' if not loop.last -}} diff --git a/packages.yml b/packages.yml index 2246ac8b..5302a1f2 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,3 @@ packages: - - package: fishtown-analytics/dbt_utils - version: [">=0.6.0", "<0.7.0"] + - package: dbt-labs/dbt_utils + version: [">=0.7.0", "<0.8.0"] diff --git a/run_test.sh b/run_test.sh index a4faf193..aa03d9d3 100755 --- a/run_test.sh +++ b/run_test.sh @@ -9,14 +9,14 @@ if [[ ! -f $VENV ]]; then if [ $1 == 'databricks' ] then echo "Installing dbt-spark" - pip install dbt-spark[ODBC] --upgrade + pip install dbt-spark[ODBC] --upgrade --pre elif [ $1 == 'azuresql' ] then echo "Installing dbt-sqlserver" - pip install dbt-sqlserver --upgrade + pip install dbt-sqlserver --upgrade --pre else echo "Installing dbt-$1" - pip install dbt-$1 --upgrade + pip install dbt-$1 --upgrade --pre fi fi