update from upstream

.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
 
   integration-redshift:
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
@@ -19,7 +19,7 @@ jobs:
 
   integration-snowflake:
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
@@ -32,7 +32,7 @@ jobs:
     environment:
       BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:

README.md

@@ -1,6 +1,6 @@
 # External sources in dbt
 
-dbt v0.15.0 added support for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
+dbt v0.15.0 [added support](https://github.com/dbt-labs/dbt/pull/1784) for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
 
 This package provides:
 * Macros to create/replace external tables and refresh their partitions, using the metadata provided in your `.yml` file source definitions
@@ -19,7 +19,7 @@ This package provides:
 
 ## Installation
 
-Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/fishtown-analytics/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`.
+Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/dbt-labs/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`.
 
 ## Syntax
 
@@ -115,9 +115,9 @@ execute the appropriate `create`, `refresh`, and/or `drop` commands:
 `stage_external_sources` runs as an operation
 * [`tested specs`](integration_tests/models/plugins): source spec variations that are confirmed to work on each database, via integration tests
 
-If you encounter issues using this package or have questions, please check the [open issues](https://github.com/fishtown-analytics/dbt-external-tables/issues), as there's a chance it's a known limitation or work in progress. If not, you can:
+If you encounter issues using this package or have questions, please check the [open issues](https://github.com/dbt-labs/dbt-external-tables/issues), as there's a chance it's a known limitation or work in progress. If not, you can:
 - open a new issue to report a bug or suggest an enhancement
 - post a technical question to [StackOverflow](https://stackoverflow.com/questions/tagged/dbt)
 - post a conceptual question to the relevant database channel (#db-redshift, #dbt-snowflake, etc) in the [dbt Slack community](https://community.getdbt.com/)
 
-Additional contributions to this package are very welcome! Please create issues or open PRs against `master`. Check out [this post](https://discourse.getdbt.com/t/contributing-to-an-external-dbt-package/657) on the best workflow for contributing to a package.
+Additional contributions to this package are very welcome! Please create issues or open PRs against `master`. Check out [this post](https://discourse.getdbt.com/t/contributing-to-an-external-dbt-package/657) on the best workflow for contributing to a package.

dbt_project.yml

@@ -1,9 +1,9 @@
 name: 'dbt_external_tables'
-version: '0.3.0'
+version: '0.7.0'
 
 config-version: 2
 
-require-dbt-version: ">=0.18.0"
+require-dbt-version: ">=0.20.0"
 
 source-paths: ["models"]
 analysis-paths: ["analysis"]

integration_tests/dbt_project.yml

@@ -6,10 +6,7 @@ profile: 'integration_tests'
 
 config-version: 2
 
-source-paths:
-  - "sources/common"
-  - "sources/plugins/{{ target.type }}"
-
+source-paths: ["models"]
 analysis-paths: ["analysis"]
 test-paths: ["tests"]
 data-paths: ["data"]
@@ -20,8 +17,25 @@ clean-targets:
     - "target"
     - "dbt_modules"
 
-vars:
-  dbt_external_tables_dispatch_list: ['dbt_external_tables_integration_tests']
+dispatch:
+  - macro_namespace: dbt_external_tables
+    search_order: ['dbt_external_tables_integration_tests', 'dbt_external_tables']
 
 seeds:
   +quote_columns: false
+
+sources:
+  dbt_external_tables_integration_tests:
+    plugins:
+      redshift_external:
+        +enabled: "{{ target.type == 'redshift' }}"
+      snowflake_external:
+        +enabled: "{{ target.type == 'snowflake' }}"
+      bigquery_external:
+        +enabled: "{{ target.type == 'bigquery' }}"
+      spark_external:
+        +enabled: "{{ target.type == 'spark' }}"
+      synapse_external:
+        +enabled: "{{ target.type == 'synapse' }}"
+      azuresql_external:
+        +enabled: "{{ target.type == 'sqlserver' }}"

integration_tests/macros/common/prep_external.sql

@@ -1,5 +1,5 @@
 {% macro prep_external() %}
-    {{ return(adapter.dispatch('prep_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }}
+    {{ return(adapter.dispatch('prep_external', 'dbt_external_tables')()) }}
 {% endmacro %}
 
 {% macro default__prep_external() %}

macros/common/create_external_table.sql

@@ -1,7 +1,5 @@
 {% macro create_external_table(source_node) %}
-    {{ adapter.dispatch('create_external_table', 
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces()) 
-        (source_node) }}
+    {{ adapter.dispatch('create_external_table', 'dbt_external_tables')(source_node) }}
 {% endmacro %}
 
 {% macro default__create_external_table(source_node) %}

macros/common/get_external_build_plan.sql

@@ -1,7 +1,5 @@
 {% macro get_external_build_plan(source_node) %}
-    {{ return(adapter.dispatch('get_external_build_plan',
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces())
-        (source_node)) }}
+    {{ return(adapter.dispatch('get_external_build_plan', 'dbt_external_tables')(source_node)) }}
 {% endmacro %}
 
 {% macro default__get_external_build_plan(source_node) %}

macros/common/helpers/dropif.sql

@@ -1,7 +1,5 @@
 {% macro dropif(node) %}
-    {{ adapter.dispatch('dropif', 
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces()) 
-        (node) }}
+    {{ adapter.dispatch('dropif', 'dbt_external_tables')(node) }}
 {% endmacro %}
 
 {% macro default__dropif() %}

macros/common/helpers/transaction.sql

@@ -1,5 +1,5 @@
 {% macro exit_transaction() %}
-    {{ return(adapter.dispatch('exit_transaction', dbt_external_tables._get_dbt_external_tables_namespaces())()) }}
+    {{ return(adapter.dispatch('exit_transaction', 'dbt_external_tables')()) }}
 {% endmacro %}
 
 {% macro default__exit_transaction() %}

macros/common/refresh_external_table.sql

@@ -1,7 +1,5 @@
 {% macro refresh_external_table(source_node) %}
-    {{ return(adapter.dispatch('refresh_external_table', 
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces()) 
-        (source_node)) }}
+    {{ return(adapter.dispatch('refresh_external_table', 'dbt_external_tables')(source_node)) }}
 {% endmacro %}
 
 {% macro default__refresh_external_table(source_node) %}

macros/common/stage_external_sources.sql

@@ -49,7 +49,7 @@
 
         {% for q in run_queue %}
 
-            {% set q_msg = q|trim %}
+            {% set q_msg = q|replace('\n','')|replace('begin;','')|trim %}
             {% set q_log = q_msg[:50] ~ '...  ' if q_msg|length > 50 else q_msg %}
 
             {% do dbt_utils.log_info(loop_label ~ ' (' ~ loop.index ~ ') ' ~ q_log) %}

macros/plugins/bigquery/create_external_table.sql

@@ -30,7 +30,11 @@
             uris = [{%- for uri in uris -%} '{{uri}}' {{- "," if not loop.last}} {%- endfor -%}]
             {%- if options is mapping -%}
             {%- for key, value in options.items() if key != 'uris' %}
+                {%- if value is string -%}
+                , {{key}} = '{{value}}'
+                {%- else -%}
                 , {{key}} = {{value}}
+                {%- endif -%}
             {%- endfor -%}
             {%- endif -%}
         )

macros/plugins/snowflake/create_external_table.sql

@@ -27,7 +27,8 @@
     {%- endif -%}
     {% if partitions %} partition by ({{partitions|map(attribute='name')|join(', ')}}) {% endif %}
     location = {{external.location}} {# stage #}
-    {% if external.auto_refresh -%} auto_refresh = {{external.auto_refresh}} {%- endif %}
+    {% if external.auto_refresh is not none -%} auto_refresh = {{external.auto_refresh}} {%- endif %}
     {% if external.pattern -%} pattern = '{{external.pattern}}' {%- endif %}
+    {% if external.integration -%} integration = '{{external.integration}}' {%- endif %}
     file_format = {{external.file_format}}
 {% endmacro %}

macros/plugins/snowflake/get_external_build_plan.sql

@@ -15,7 +15,7 @@
         {% if create_or_replace %}
             {% set build_plan = build_plan + [
                 dbt_external_tables.snowflake_create_empty_table(source_node),
-                dbt_external_tables.snowflake_get_copy_sql(source_node),
+                dbt_external_tables.snowflake_get_copy_sql(source_node, explicit_transaction=true),
                 dbt_external_tables.snowflake_create_snowpipe(source_node)
             ] %}
         {% else %}

macros/plugins/snowflake/refresh_external_table.sql

@@ -11,7 +11,9 @@
     {% if manual_refresh %}
 
         {% set ddl %}
-        alter external table {{source(source_node.source_name, source_node.name)}} refresh
+        begin;
+        alter external table {{source(source_node.source_name, source_node.name)}} refresh;
+        commit;
         {% endset %}
 
         {% do return([ddl]) %}

macros/plugins/snowflake/snowpipe/get_copy_sql.sql

@@ -1,11 +1,13 @@
-{% macro snowflake_get_copy_sql(source_node) %}
+{% macro snowflake_get_copy_sql(source_node, explicit_transaction=false) %}
 {# This assumes you have already created an external stage #}
 
     {%- set columns = source_node.columns.values() -%}
     {%- set external = source_node.external -%}
     {%- set is_csv = dbt_external_tables.is_csv(external.file_format) %}
     {%- set copy_options = external.snowpipe.get('copy_options', none) -%}
 
+    {%- if explicit_transaction -%} begin; {%- endif %}
+
     copy into {{source(source_node.source_name, source_node.name)}}
     from ( 
         select
@@ -27,6 +29,8 @@
         from {{external.location}} {# stage #}
     )
     file_format = {{external.file_format}}
-    {% if copy_options %} {{copy_options}} {% endif %}
+    {% if copy_options %} {{copy_options}} {% endif %};
+
+    {% if explicit_transaction -%} commit; {%- endif -%}
 
 {% endmacro %}

macros/plugins/spark/create_external_table.sql

@@ -13,7 +13,7 @@
             {{- ',' if not loop.last -}}
         {% endfor %}
     ) {% endif -%}
-    {% if external.using -%} using {{external.using}} {%- endif %}
+    {% if external.using %} using {{external.using}} {%- endif %}
     {% if options -%} options (
         {%- for key, value in options.items() -%}
             '{{ key }}' = '{{value}}' {{- ', \n' if not loop.last -}}

packages.yml

@@ -1,3 +1,3 @@
 packages:
-  - package: fishtown-analytics/dbt_utils
-    version: [">=0.6.0", "<0.7.0"]
+  - package: dbt-labs/dbt_utils
+    version: [">=0.7.0", "<0.8.0"]

run_test.sh

@@ -9,14 +9,14 @@ if [[ ! -f $VENV ]]; then
     if [ $1 == 'databricks' ]
     then
         echo "Installing dbt-spark"
-        pip install dbt-spark[ODBC] --upgrade
+        pip install dbt-spark[ODBC] --upgrade --pre
     elif [ $1 == 'azuresql' ]
     then
         echo "Installing dbt-sqlserver"
-        pip install dbt-sqlserver --upgrade
+        pip install dbt-sqlserver --upgrade --pre
     else
         echo "Installing dbt-$1"
-        pip install dbt-$1 --upgrade
+        pip install dbt-$1 --upgrade --pre
     fi
 fi