From a37da9cab1ab0a4144e5646a9ba47bcb1181aa4b Mon Sep 17 00:00:00 2001
From: Jason Zondor <jason@zondor.com>
Date: Tue, 6 Oct 2020 16:05:14 -0500
Subject: [PATCH 01/22] Added in the ability to create columns on external
 tables by defining them in the schema.yml.  external -> calc_columns

---
 macros/external/create_external_table.sql | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/macros/external/create_external_table.sql b/macros/external/create_external_table.sql
index aa08e4cb..81f01c30 100644
--- a/macros/external/create_external_table.sql
+++ b/macros/external/create_external_table.sql
@@ -64,17 +64,21 @@
     {%- set columns = source_node.columns.values() -%}
     {%- set external = source_node.external -%}
     {%- set partitions = external.partitions -%}
+    {%- set calc_columns = external.calc_columns -%}
 
     {%- set is_csv = dbt_external_tables.is_csv(external.file_format) -%}
 
 {# https://docs.snowflake.net/manuals/sql-reference/sql/create-external-table.html #}
 {# This assumes you have already created an external stage #}
     create or replace external table {{source(source_node.source_name, source_node.name)}}
-    {%- if columns or partitions -%}
+    {%- if columns or partitions or calc_columns -%}
     (
         {%- if partitions -%}{%- for partition in partitions %}
             {{partition.name}} {{partition.data_type}} as {{partition.expression}}{{- ',' if columns|length > 0 -}}
         {%- endfor -%}{%- endif -%}
+        {%- if calc_columns -%}{%- for calc_column in calc_columns -%}
+            {{calc_column.name}} {{calc_column.data_type}} as {{calc_column.expression}}{{- ',' if calc_columns|length > 0 -}}
+        {%- endfor -%}{%- endif -%}
         {%- for column in columns %}
             {%- set col_expression -%}
                 {%- if is_csv -%}nullif(value:c{{loop.index}},''){# special case: get columns by ordinal position #}
@@ -84,6 +88,7 @@
             {{column.name}} {{column.data_type}} as ({{col_expression}}::{{column.data_type}})
             {{- ',' if not loop.last -}}
         {% endfor %}
+        
     )
     {%- endif -%}
     {% if partitions %} partition by ({{partitions|map(attribute='name')|join(', ')}}) {% endif %}

From 7c6a9355fb27dc25c8130b5eab9b71a43df7341a Mon Sep 17 00:00:00 2001
From: Jason Zondor <jason.zondor@loves.com>
Date: Tue, 6 Oct 2020 16:24:00 -0500
Subject: [PATCH 02/22] updated readme and sample source

---
 README.md                    | 8 +++++++-
 sample_sources/snowflake.yml | 4 ++++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d4eb9bae..da9afa69 100644
--- a/README.md
+++ b/README.md
@@ -69,7 +69,13 @@ sources:
             - name: collector_date
               data_type: date
               expression: to_date(substr(metadata$filename, 8, 10), 'YYYY/MM/DD')
-              
+          
+          # ------ SNOWFLAKE - Calculated Columns ------
+          calc_columns:
+            - name: landed_time
+              data_type: time
+              expression: "IFF(try_to_time(substr(metadata$filename, 75, 6), 'HH24MISS') IS NOT NULL, to_time(substr(metadata$filename, 75, 6), 'HH24MISS'), to_time('080000', 'HH24MISS'))"
+          
           # ------ REDSHIFT -------
           partitions:
             - name: appId
diff --git a/sample_sources/snowflake.yml b/sample_sources/snowflake.yml
index d423283b..d57157d1 100644
--- a/sample_sources/snowflake.yml
+++ b/sample_sources/snowflake.yml
@@ -17,6 +17,10 @@ sources:
             - name: collector_hour
               data_type: timestamp
               expression: to_timestamp(substr(metadata$filename, 8, 13), 'YYYY/MM/DD/HH24')
+          calc_columns:
+            - name: landed_time
+              data_type: time
+              expression: "IFF(try_to_time(substr(metadata$filename, 75, 6), 'HH24MISS') IS NOT NULL, to_time(substr(metadata$filename, 75, 6), 'HH24MISS'), to_time('080000', 'HH24MISS'))"
         columns:
           - name: app_id
             data_type: varchar(255)

From 2280546fa7830c52735a74aab5942979eb237894 Mon Sep 17 00:00:00 2001
From: Jason Zondor <jason.zondor@loves.com>
Date: Mon, 9 Nov 2020 09:21:56 -0600
Subject: [PATCH 03/22] Fixing commit issue for snowflake on refreshes.

---
 macros/external/refresh_external_table.sql | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/macros/external/refresh_external_table.sql b/macros/external/refresh_external_table.sql
index 7d8c214d..f9bb5648 100644
--- a/macros/external/refresh_external_table.sql
+++ b/macros/external/refresh_external_table.sql
@@ -92,12 +92,10 @@
     
     {% if manual_refresh %}
 
-        {% set ddl %}
-        alter external table {{source(source_node.source_name, source_node.name)}} refresh
-        {% endset %}
-        
-        {% do return([ddl]) %}
-    
+        BEGIN;
+        alter external table {{source(source_node.source_name, source_node.name)}} refresh;
+        COMMIT;
+
     {% else %}
     
         {% do return([]) %}

From 14c4d8c2350d1ea909708aa40f51ca52ebb6661a Mon Sep 17 00:00:00 2001
From: Jason Zondor <jason.zondor@loves.com>
Date: Mon, 9 Nov 2020 10:08:35 -0600
Subject: [PATCH 04/22] oops, forgot to set ddl for external refresh.

---
 macros/external/refresh_external_table.sql | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/macros/external/refresh_external_table.sql b/macros/external/refresh_external_table.sql
index f9bb5648..1645ad7b 100644
--- a/macros/external/refresh_external_table.sql
+++ b/macros/external/refresh_external_table.sql
@@ -92,9 +92,13 @@
     
     {% if manual_refresh %}
 
+        {% set ddl %}
         BEGIN;
         alter external table {{source(source_node.source_name, source_node.name)}} refresh;
         COMMIT;
+        {% endset %}
+
+         {% do return([ddl]) %}
 
     {% else %}
     

From 6a65d4ecb14d6bed512b3acdc48d43f779a794fb Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Sat, 14 Nov 2020 17:46:30 -0500
Subject: [PATCH 05/22] Commit transactions on Snowflake

---
 macros/helpers/{redshift => }/transaction.sql | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)
 rename macros/helpers/{redshift => }/transaction.sql (71%)

diff --git a/macros/helpers/redshift/transaction.sql b/macros/helpers/transaction.sql
similarity index 71%
rename from macros/helpers/redshift/transaction.sql
rename to macros/helpers/transaction.sql
index b41550a9..dee91f8d 100644
--- a/macros/helpers/redshift/transaction.sql
+++ b/macros/helpers/transaction.sql
@@ -3,9 +3,13 @@
 {% endmacro %}
 
 {% macro default__exit_transaction() %}
+    {{ return('begin; commit;') }}
+{% endmacro %}
+
+{% macro bigquery__exit_transaction() %}
     {{ return('') }}
 {% endmacro %}
 
-{% macro redshift__exit_transaction() %}
-    {{ return('begin; commit;') }}
+{% macro spark__exit_transaction() %}
+    {{ return('') }}
 {% endmacro %}

From 0d01b30c83013b3b9d2decd1d387772aec41672f Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Mon, 16 Nov 2020 01:21:53 -0500
Subject: [PATCH 06/22] Try replicating via autocommit

---
 .circleci/config.yml                          |  2 ++
 integration_tests/Makefile                    |  2 ++
 integration_tests/macros/cleanup_external.sql | 19 +++++++++++++++++++
 integration_tests/macros/prep_external.sql    | 12 ++++++++++++
 4 files changed, 35 insertions(+)
 create mode 100644 integration_tests/macros/cleanup_external.sql

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6939e568..2a88b4a3 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -35,6 +35,7 @@ jobs:
             dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target redshift
             dbt --warn-error run-operation stage_external_sources --target redshift
             dbt --warn-error test --target redshift
+          	dbt run-operation cleanup_external --target redshift
       
       - run:
           name: "Run Tests - Snowflake"
@@ -48,6 +49,7 @@ jobs:
             dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target snowflake
             dbt --warn-error run-operation stage_external_sources --target snowflake
             dbt --warn-error test --target snowflake
+          	dbt run-operation cleanup_external --target snowflake
       
       - save_cache:
           key: deps1-{{ .Branch }}
diff --git a/integration_tests/Makefile b/integration_tests/Makefile
index 7cf524cd..54a6a870 100644
--- a/integration_tests/Makefile
+++ b/integration_tests/Makefile
@@ -5,6 +5,7 @@ test-redshift:
 	dbt run-operation stage_external_sources --var 'ext_full_refresh: true' --target redshift
 	dbt run-operation stage_external_sources --target redshift
 	dbt test --target redshift
+	dbt run-operation cleanup_external --target redshift
 	
 test-snowflake:
 	dbt deps --target snowflake
@@ -13,3 +14,4 @@ test-snowflake:
 	dbt run-operation stage_external_sources --var 'ext_full_refresh: true' --target snowflake
 	dbt run-operation stage_external_sources --target snowflake
 	dbt test --target snowflake
+	dbt run-operation cleanup_external --target snowflake
diff --git a/integration_tests/macros/cleanup_external.sql b/integration_tests/macros/cleanup_external.sql
new file mode 100644
index 00000000..c1d4d842
--- /dev/null
+++ b/integration_tests/macros/cleanup_external.sql
@@ -0,0 +1,19 @@
+{% macro cleanup_external() %}
+    {{ return(adapter.dispatch('cleanup_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }}
+{% endmacro %}
+
+{% macro default__cleanup_external() %}
+    {% do log('No cleanup necessary, skipping', info = true) %}
+    {# noop #}
+{% endmacro %}
+
+{% macro snowflake__cleanup_external() %}
+
+    {% set unset_autocommit %}
+        alter user {{ target.user }} unset autocommit;
+    {% endset %}
+
+    {% do log('Unsetting autocommit parameter for user ' ~ target.user, info = true) %}
+    {% do run_query(unset_autocommit) %}
+    
+{% endmacro %}
diff --git a/integration_tests/macros/prep_external.sql b/integration_tests/macros/prep_external.sql
index 9a9a3dfa..834e4acc 100644
--- a/integration_tests/macros/prep_external.sql
+++ b/integration_tests/macros/prep_external.sql
@@ -2,6 +2,11 @@
     {{ return(adapter.dispatch('prep_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }}
 {% endmacro %}
 
+{% macro default__prep_external() %}
+    {% do log('No prep necessary, skipping', info = true) %}
+    {# noop #}
+{% endmacro %}
+
 {% macro redshift__prep_external() %}
 
     {% set external_schema = target.schema ~ '_spectrum' %}
@@ -37,4 +42,11 @@
     {% do log('Creating external stage ' ~ external_stage, info = true) %}
     {% do run_query(create_external_stage) %}
     
+    {% set set_autocommit_false %}
+        alter user {{ target.user }} set autocommit = false;
+    {% endset %}
+    
+    {% do log('Turning off autocommit for user ' ~ target.user, info = true) %}
+    {% do run_query(set_autocommit_false) %}
+    
 {% endmacro %}

From de571e5d6ccb872eeb1ce9ab97a8555e2cef9622 Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Mon, 16 Nov 2020 01:23:33 -0500
Subject: [PATCH 07/22] Circle hates tabs

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2a88b4a3..31ec6ec5 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -35,7 +35,7 @@ jobs:
             dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target redshift
             dbt --warn-error run-operation stage_external_sources --target redshift
             dbt --warn-error test --target redshift
-          	dbt run-operation cleanup_external --target redshift
+            dbt run-operation cleanup_external --target redshift
       
       - run:
           name: "Run Tests - Snowflake"
@@ -49,7 +49,7 @@ jobs:
             dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target snowflake
             dbt --warn-error run-operation stage_external_sources --target snowflake
             dbt --warn-error test --target snowflake
-          	dbt run-operation cleanup_external --target snowflake
+            dbt run-operation cleanup_external --target snowflake
       
       - save_cache:
           key: deps1-{{ .Branch }}

From 557bf47c292515e57925b74376f5a2c3828d2954 Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Mon, 25 Jan 2021 11:51:10 +0100
Subject: [PATCH 08/22] Add back code lost in merge

---
 .../macros/{ => common}/cleanup_external.sql          | 11 -----------
 .../macros/plugins/snowflake/cleanup_external.sql     | 10 ++++++++++
 .../macros/plugins/snowflake/prep_external.sql        |  7 +++++++
 3 files changed, 17 insertions(+), 11 deletions(-)
 rename integration_tests/macros/{ => common}/cleanup_external.sql (50%)
 create mode 100644 integration_tests/macros/plugins/snowflake/cleanup_external.sql

diff --git a/integration_tests/macros/cleanup_external.sql b/integration_tests/macros/common/cleanup_external.sql
similarity index 50%
rename from integration_tests/macros/cleanup_external.sql
rename to integration_tests/macros/common/cleanup_external.sql
index c1d4d842..5ddcaf19 100644
--- a/integration_tests/macros/cleanup_external.sql
+++ b/integration_tests/macros/common/cleanup_external.sql
@@ -6,14 +6,3 @@
     {% do log('No cleanup necessary, skipping', info = true) %}
     {# noop #}
 {% endmacro %}
-
-{% macro snowflake__cleanup_external() %}
-
-    {% set unset_autocommit %}
-        alter user {{ target.user }} unset autocommit;
-    {% endset %}
-
-    {% do log('Unsetting autocommit parameter for user ' ~ target.user, info = true) %}
-    {% do run_query(unset_autocommit) %}
-    
-{% endmacro %}
diff --git a/integration_tests/macros/plugins/snowflake/cleanup_external.sql b/integration_tests/macros/plugins/snowflake/cleanup_external.sql
new file mode 100644
index 00000000..305b9f48
--- /dev/null
+++ b/integration_tests/macros/plugins/snowflake/cleanup_external.sql
@@ -0,0 +1,10 @@
+{% macro snowflake__cleanup_external() %}
+
+    {% set unset_autocommit %}
+        alter user {{ target.user }} unset autocommit;
+    {% endset %}
+
+    {% do log('Unsetting autocommit parameter for user ' ~ target.user, info = true) %}
+    {% do run_query(unset_autocommit) %}
+    
+{% endmacro %}
diff --git a/integration_tests/macros/plugins/snowflake/prep_external.sql b/integration_tests/macros/plugins/snowflake/prep_external.sql
index 5dd4dca9..6d33ca0b 100644
--- a/integration_tests/macros/plugins/snowflake/prep_external.sql
+++ b/integration_tests/macros/plugins/snowflake/prep_external.sql
@@ -13,4 +13,11 @@
     {% do log('Creating external stage ' ~ external_stage, info = true) %}
     {% do run_query(create_external_stage) %}
     
+    {% set set_autocommit_false %}
+        alter user {{ target.user }} set autocommit = false;
+    {% endset %}
+
+    {% do log('Turning off autocommit for user ' ~ target.user, info = true) %}
+    {% do run_query(set_autocommit_false) %}
+    
 {% endmacro %}

From 970452b04322bd66d02935e50809053456925fcd Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Mon, 25 Jan 2021 11:51:48 +0100
Subject: [PATCH 09/22] Readd cleanup step

---
 run_test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/run_test.sh b/run_test.sh
index db5b196b..7f040477 100755
--- a/run_test.sh
+++ b/run_test.sh
@@ -33,3 +33,4 @@ dbt run-operation prep_external --target $1
 dbt run-operation stage_external_sources --var 'ext_full_refresh: true' --target $1
 dbt run-operation stage_external_sources --target $1
 dbt test --target $1
+dbt run-operation cleanup_external --target $1

From 62e60eb08db468f3aaa269fcc961a3ce94b74f63 Mon Sep 17 00:00:00 2001
From: Bertrand Bossy <bertrand.bossy@vontobel.com>
Date: Wed, 21 Apr 2021 07:01:34 +0000
Subject: [PATCH 10/22] Snowflake Azure: auto refresh external tables

---
 macros/plugins/snowflake/create_external_table.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql
index da7da16f..2ff0d801 100644
--- a/macros/plugins/snowflake/create_external_table.sql
+++ b/macros/plugins/snowflake/create_external_table.sql
@@ -29,5 +29,6 @@
     location = {{external.location}} {# stage #}
     {% if external.auto_refresh -%} auto_refresh = {{external.auto_refresh}} {%- endif %}
     {% if external.pattern -%} pattern = '{{external.pattern}}' {%- endif %}
+    {% if external.integration -%} integration = {{external.integration}} {%- endif %}
     file_format = {{external.file_format}}
 {% endmacro %}

From ed05c70958acfa2ffdc9fbf0dc540bdf0ce8ea7c Mon Sep 17 00:00:00 2001
From: Bertrand Bossy <bertrand.bossy@vontobel.com>
Date: Wed, 21 Apr 2021 13:23:21 +0000
Subject: [PATCH 11/22] Quote notification integration parameter

---
 macros/plugins/snowflake/create_external_table.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql
index 2ff0d801..c29e5d25 100644
--- a/macros/plugins/snowflake/create_external_table.sql
+++ b/macros/plugins/snowflake/create_external_table.sql
@@ -29,6 +29,6 @@
     location = {{external.location}} {# stage #}
     {% if external.auto_refresh -%} auto_refresh = {{external.auto_refresh}} {%- endif %}
     {% if external.pattern -%} pattern = '{{external.pattern}}' {%- endif %}
-    {% if external.integration -%} integration = {{external.integration}} {%- endif %}
+    {% if external.integration -%} integration = '{{external.integration}}' {%- endif %}
     file_format = {{external.file_format}}
 {% endmacro %}

From 4f1c7d2d246aab3967c30e209407a001ea8e65a1 Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Mon, 17 May 2021 14:19:22 -0400
Subject: [PATCH 12/22] Try reverting test changes, upgrading python in ci

---
 .circleci/config.yml                          |  6 +++---
 integration_tests/dbt_project.yml             | 21 +++++++++++++++----
 .../{sources => models}/common/control.yml    |  0
 .../plugins}/azuresql_external.yml            |  0
 .../plugins}/bigquery_external.yml            |  0
 .../plugins}/redshift_external.yml            |  0
 .../plugins}/snowflake_external.yml           |  0
 .../plugins}/spark_external.yml               |  0
 .../plugins}/synapse_external.yml             |  0
 9 files changed, 20 insertions(+), 7 deletions(-)
 rename integration_tests/{sources => models}/common/control.yml (100%)
 rename integration_tests/{sources/plugins/sqlserver => models/plugins}/azuresql_external.yml (100%)
 rename integration_tests/{sources/plugins/bigquery => models/plugins}/bigquery_external.yml (100%)
 rename integration_tests/{sources/plugins/redshift => models/plugins}/redshift_external.yml (100%)
 rename integration_tests/{sources/plugins/snowflake => models/plugins}/snowflake_external.yml (100%)
 rename integration_tests/{sources/plugins/spark => models/plugins}/spark_external.yml (100%)
 rename integration_tests/{sources/plugins/synapse => models/plugins}/synapse_external.yml (100%)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 4635df2f..cc353373 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -8,7 +8,7 @@ jobs:
   
   integration-redshift:
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
@@ -19,7 +19,7 @@ jobs:
       
   integration-snowflake:
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
@@ -32,7 +32,7 @@ jobs:
     environment:
       BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
index 2524779d..a888c424 100644
--- a/integration_tests/dbt_project.yml
+++ b/integration_tests/dbt_project.yml
@@ -6,10 +6,7 @@ profile: 'integration_tests'
 
 config-version: 2
 
-source-paths:
-  - "sources/common"
-  - "sources/plugins/{{ target.type }}"
-
+source-paths: ["models"]
 analysis-paths: ["analysis"]
 test-paths: ["tests"]
 data-paths: ["data"]
@@ -25,3 +22,19 @@ vars:
 
 seeds:
   +quote_columns: false
+
+sources:
+  dbt_external_tables_integration_tests:
+    plugins:
+      redshift_external:
+        +enabled: "{{ target.type == 'redshift' }}"
+      snowflake_external:
+        +enabled: "{{ target.type == 'snowflake' }}"
+      bigquery_external:
+        +enabled: "{{ target.type == 'bigquery' }}"
+      spark_external:
+        +enabled: "{{ target.type == 'spark' }}"
+      synapse_external:
+        +enabled: "{{ target.type == 'synapse' }}"
+      azuresql_external:
+        +enabled: "{{ target.type == 'sqlserver' }}"
diff --git a/integration_tests/sources/common/control.yml b/integration_tests/models/common/control.yml
similarity index 100%
rename from integration_tests/sources/common/control.yml
rename to integration_tests/models/common/control.yml
diff --git a/integration_tests/sources/plugins/sqlserver/azuresql_external.yml b/integration_tests/models/plugins/azuresql_external.yml
similarity index 100%
rename from integration_tests/sources/plugins/sqlserver/azuresql_external.yml
rename to integration_tests/models/plugins/azuresql_external.yml
diff --git a/integration_tests/sources/plugins/bigquery/bigquery_external.yml b/integration_tests/models/plugins/bigquery_external.yml
similarity index 100%
rename from integration_tests/sources/plugins/bigquery/bigquery_external.yml
rename to integration_tests/models/plugins/bigquery_external.yml
diff --git a/integration_tests/sources/plugins/redshift/redshift_external.yml b/integration_tests/models/plugins/redshift_external.yml
similarity index 100%
rename from integration_tests/sources/plugins/redshift/redshift_external.yml
rename to integration_tests/models/plugins/redshift_external.yml
diff --git a/integration_tests/sources/plugins/snowflake/snowflake_external.yml b/integration_tests/models/plugins/snowflake_external.yml
similarity index 100%
rename from integration_tests/sources/plugins/snowflake/snowflake_external.yml
rename to integration_tests/models/plugins/snowflake_external.yml
diff --git a/integration_tests/sources/plugins/spark/spark_external.yml b/integration_tests/models/plugins/spark_external.yml
similarity index 100%
rename from integration_tests/sources/plugins/spark/spark_external.yml
rename to integration_tests/models/plugins/spark_external.yml
diff --git a/integration_tests/sources/plugins/synapse/synapse_external.yml b/integration_tests/models/plugins/synapse_external.yml
similarity index 100%
rename from integration_tests/sources/plugins/synapse/synapse_external.yml
rename to integration_tests/models/plugins/synapse_external.yml

From deb9fe95cbabb0e9a27b5c7f58faff866e254503 Mon Sep 17 00:00:00 2001
From: Tim Vergenz <vergenzt@gmail.com>
Date: Thu, 20 May 2021 13:15:21 -0400
Subject: [PATCH 13/22] Update README.md: link to dbt PR for `external`

I was curious what the semantics of that `source` config attribute were and couldn't find it in dbt's docs. :)
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 5e732b39..c5d976d3 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # External sources in dbt
 
-dbt v0.15.0 added support for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
+dbt v0.15.0 [added support](https://github.com/fishtown-analytics/dbt/pull/1784) for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
 
 This package provides:
 * Macros to create/replace external tables and refresh their partitions, using the metadata provided in your `.yml` file source definitions

From 85c12c32b02ab3986a7fc311d3268153cccec915 Mon Sep 17 00:00:00 2001
From: clegendre <cedric@datahawk.co>
Date: Sun, 23 May 2021 19:55:10 +0200
Subject: [PATCH 14/22] Fix if whitespace triming

---
 macros/plugins/spark/create_external_table.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/plugins/spark/create_external_table.sql b/macros/plugins/spark/create_external_table.sql
index b36741d7..a36a2153 100644
--- a/macros/plugins/spark/create_external_table.sql
+++ b/macros/plugins/spark/create_external_table.sql
@@ -13,7 +13,7 @@
             {{- ',' if not loop.last -}}
         {% endfor %}
     ) {% endif -%}
-    {% if external.using -%} using {{external.using}} {%- endif %}
+    {% if external.using %} using {{external.using}} {%- endif %}
     {% if options -%} options (
         {%- for key, value in options.items() -%}
             '{{ key }}' = '{{value}}' {{- ', \n' if not loop.last -}}

From 4869caf785384c8f699678824efb21bb9b19792d Mon Sep 17 00:00:00 2001
From: davesgonechina <davesgonechina@users.noreply.github.com>
Date: Wed, 26 May 2021 20:12:06 -0400
Subject: [PATCH 15/22] wrap options.items() values in single quotes

Not tested this comprehensively but not wrapping `hive_partition_uri_prefix` in single quotes results in a syntax error and single quotes work fine for `format`
---
 macros/plugins/bigquery/create_external_table.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/plugins/bigquery/create_external_table.sql b/macros/plugins/bigquery/create_external_table.sql
index 1adfe9ef..1c28dc06 100644
--- a/macros/plugins/bigquery/create_external_table.sql
+++ b/macros/plugins/bigquery/create_external_table.sql
@@ -30,7 +30,7 @@
             uris = [{%- for uri in uris -%} '{{uri}}' {{- "," if not loop.last}} {%- endfor -%}]
             {%- if options is mapping -%}
             {%- for key, value in options.items() if key != 'uris' %}
-                , {{key}} = {{value}}
+                , {{key}} = '{{value}}'
             {%- endfor -%}
             {%- endif -%}
         )

From e3dea14f79e809b7428a91d9626b4a14dd31078b Mon Sep 17 00:00:00 2001
From: davesgonechina <davesgonechina@users.noreply.github.com>
Date: Thu, 27 May 2021 09:22:17 -0400
Subject: [PATCH 16/22] single quote value if string

---
 macros/plugins/bigquery/create_external_table.sql | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/macros/plugins/bigquery/create_external_table.sql b/macros/plugins/bigquery/create_external_table.sql
index 1c28dc06..8676ea40 100644
--- a/macros/plugins/bigquery/create_external_table.sql
+++ b/macros/plugins/bigquery/create_external_table.sql
@@ -30,7 +30,11 @@
             uris = [{%- for uri in uris -%} '{{uri}}' {{- "," if not loop.last}} {%- endfor -%}]
             {%- if options is mapping -%}
             {%- for key, value in options.items() if key != 'uris' %}
+                {%- if value is string -%}
                 , {{key}} = '{{value}}'
+                {%- else -%}
+                , {{key}} = {{value}}
+                {%- endif -%}
             {%- endfor -%}
             {%- endif -%}
         )

From 289db8ddc208da2eb61f2151916b9ae96f744b01 Mon Sep 17 00:00:00 2001
From: Jason Zondor <jason@zondor.com>
Date: Mon, 7 Jun 2021 13:50:07 -0500
Subject: [PATCH 17/22] Adding begin and commit statements to refresh macro.

---
 macros/plugins/snowflake/refresh_external_table.sql | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/macros/plugins/snowflake/refresh_external_table.sql b/macros/plugins/snowflake/refresh_external_table.sql
index 79eab423..4f3da15b 100644
--- a/macros/plugins/snowflake/refresh_external_table.sql
+++ b/macros/plugins/snowflake/refresh_external_table.sql
@@ -11,7 +11,9 @@
     {% if manual_refresh %}
 
         {% set ddl %}
-        alter external table {{source(source_node.source_name, source_node.name)}} refresh
+        BEGIN;
+        alter external table {{source(source_node.source_name, source_node.name)}} refresh;
+        COMMIT;
         {% endset %}
         
         {% do return([ddl]) %}

From a4e48e60b8fe21957eb21a39364d340ed324f523 Mon Sep 17 00:00:00 2001
From: Jason Zondor <jason@zondor.com>
Date: Mon, 7 Jun 2021 14:19:04 -0500
Subject: [PATCH 18/22] Adding readme and snowflake sample files back in...

---
 README.md                    | 123 +++++++++++++++++++++++++++++++++++
 sample_sources/snowflake.yml |  62 ++++++++++++++++++
 2 files changed, 185 insertions(+)
 create mode 100644 README.md
 create mode 100644 sample_sources/snowflake.yml

diff --git a/README.md b/README.md
new file mode 100644
index 00000000..52460208
--- /dev/null
+++ b/README.md
@@ -0,0 +1,123 @@
+# External sources in dbt
+
+dbt v0.15.0 [added support](https://github.com/fishtown-analytics/dbt/pull/1784) for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
+
+This package provides:
+* Macros to create/replace external tables and refresh their partitions, using the metadata provided in your `.yml` file source definitions
+* Snowflake-specific macros to create, backfill, and refresh snowpipes, using the same metadata
+
+## Supported databases
+
+* Redshift (Spectrum)
+* Snowflake
+* BigQuery
+* Spark
+* Synapse
+* Azure SQL
+
+![sample docs](etc/sample_docs.png)
+
+## Installation
+
+Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/fishtown-analytics/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`.
+
+## Syntax
+
+The `stage_external_sources` macro is the primary point of entry when using this package. It has two operational modes: standard and "full refresh."
+
+```bash
+# iterate through all source nodes, create if missing, refresh metadata
+$ dbt run-operation stage_external_sources
+
+# iterate through all source nodes, create or replace (+ refresh if necessary)
+$ dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
+```
+
+The `stage_external_sources` macro accepts a limited node selection syntax similar to
+[snapshotting source freshness](https://docs.getdbt.com/docs/running-a-dbt-project/command-line-interface/source/#specifying-sources-to-snapshot):
+
+```bash
+# stage all Snowplow and Logs external sources:
+$ dbt run-operation stage_external_sources --args "select: snowplow logs"
+
+# stage a particular external source table:
+$ dbt run-operation stage_external_sources --args "select: snowplow.event"
+```
+
+## Setup
+
+The macros assume that you:
+1. Have already created your database's required scaffolding for external resources:
+  - an external stage (Snowflake)
+  - an external schema + S3 bucket (Redshift Spectrum)
+  - an external data source and file format (Synapse)
+  - an external data source and databse-scoped credential (Azure SQL)
+  - a Google Cloud Storage bucket (BigQuery)
+  - an accessible set of files (Spark)
+2. Have the appropriate permissions on to create tables using that scaffolding
+3. Have already created the database/project and/or schema/dataset in which dbt will create external tables (or snowpiped tables)
+
+## Spec
+
+```yml
+version: 2
+
+sources:
+  - name: snowplow
+    tables:
+      - name: event
+        description: >
+            This source table is actually a set of files in external storage.
+            The dbt-external-tables package provides handy macros for getting
+            those files queryable, just in time for modeling.
+                            
+        external:
+          location:         # required: S3 file path, GCS file path, Snowflake stage, Synapse data source
+          
+          ...               # database-specific properties of external table
+          
+          partitions:       # optional
+            - name: collector_date
+              data_type: date
+              ...           # database-specific properties
+
+        # Specify ALL column names + datatypes.
+        # Column order must match for CSVs, column names must match for other formats.
+        # Some databases support schema inference.
+
+        columns:
+          - name: app_id
+            data_type: varchar(255)
+            description: "Application ID"
+          - name: platform
+            data_type: varchar(255)
+            description: "Platform"
+          ...
+```
+
+The `stage_external_sources` macro will use this YAML config to compile and 
+execute the appropriate `create`, `refresh`, and/or `drop` commands:
+
+```
+19:01:48 + 1 of 1 START external source spectrum.my_partitioned_tbl
+19:01:48 + 1 of 1 (1) drop table if exists "db"."spectrum"."my_partitioned_tbl"
+19:01:48 + 1 of 1 (1) DROP TABLE
+19:01:48 + 1 of 1 (2) create external table "db"."spectrum"."my_partitioned_tbl"...
+19:01:48 + 1 of 1 (2) CREATE EXTERNAL TABLE
+19:01:48 + 1 of 1 (3) alter table "db"."spectrum"."my_partitioned_tbl"...
+19:01:49 + 1 of 1 (3) ALTER EXTERNAL TABLE
+```
+
+## Resources
+
+* [`sample_sources`](sample_sources): detailed example source specs, with annotations, for each database's implementation
+* [`sample_analysis`](sample_analysis): a "dry run" version of the compiled DDL/DML that
+`stage_external_sources` runs as an operation
+* [`tested specs`](integration_tests/models/plugins): source spec variations that are confirmed to work on each database, via integration tests
+
+If you encounter issues using this package or have questions, please check the [open issues](https://github.com/fishtown-analytics/dbt-external-tables/issues), as there's a chance it's a known limitation or work in progress. If not, you can:
+- open a new issue to report a bug or suggest an enhancement
+- post a technical question to [StackOverflow](https://stackoverflow.com/questions/tagged/dbt)
+- post a conceptual question to the relevant database channel (#db-redshift, #dbt-snowflake, etc) in the [dbt Slack community](https://community.getdbt.com/)
+
+Additional contributions to this package are very welcome! Please create issues or open PRs against `master`. Check out [this post](https://discourse.getdbt.com/t/contributing-to-an-external-dbt-package/657) on the best workflow for contributing to a package.
\ No newline at end of file
diff --git a/sample_sources/snowflake.yml b/sample_sources/snowflake.yml
new file mode 100644
index 00000000..9df1eae0
--- /dev/null
+++ b/sample_sources/snowflake.yml
@@ -0,0 +1,62 @@
+version: 2
+
+sources:
+  - name: snowplow
+    database: analytics
+    schema: snowplow_external
+    loader: S3
+    loaded_at_field: collector_hour
+    
+    tables:
+      - name: event_ext_tbl
+        description: "External table of Snowplow events stored as JSON files"
+        external:
+          location: "@raw.snowplow.snowplow"  # reference an existing external stage
+          file_format: "( type = json )"      # fully specified here, or reference an existing file format
+          auto_refresh: true                  # requires configuring an event notification from Amazon S3 or Azure
+          partitions:
+            - name: collector_hour
+              data_type: timestamp
+              expression: to_timestamp(substr(metadata$filename, 8, 13), 'YYYY/MM/DD/HH24')
+              
+        # all Snowflake external tables natively include a `metadata$filename` pseudocolumn
+        # and a `value` column (JSON blob-ified version of file contents), so there is no need to specify
+        # them here. you may optionally specify columns to unnest or parse from the file:
+        columns:
+          - name: app_id
+            data_type: varchar(255)
+            description: "Application ID"
+          - name: domain_sessionidx
+            data_type: int
+            description: "A visit / session index"
+          - name: etl_tstamp
+            data_type: timestamp
+            description: "Timestamp event began ETL"
+          - name: contexts
+            data_type: variant
+            description: "Contexts attached to event by Tracker"
+        
+              
+      - name: event_snowpipe
+        description: "Table of Snowplow events, stored as JSON files, loaded in near-real time via Snowpipe"
+        loader: S3 + snowpipe    # this is just for your reference
+        external:
+          location: "@raw.snowplow.snowplow"
+          file_format: "{{ target.schema }}.my_json_file_format"
+          
+          # Instead of an external tables, create an empty table, backfill it, and pipe new data
+          snowpipe:
+            auto_ingest:    true  # requires either `aws_sns_topic` or `integration`
+            aws_sns_topic:  # Amazon S3
+            integration:    # Google Cloud or Azure
+            copy_options:   "on_error = continue, enforce_length = false" # e.g.
+              
+        # dbt will include three metadata columns in addition to any `columns`
+        # specified for a snowpiped table:
+        #   `metadata_filename`: the file from which this row was loaded
+        #   `metadata_file_row_number`: the numbered row this was in that file
+        #   `_dbt_copied_at`: the current_timestamp when this row was loaded (backfilled or piped)
+        #
+        # if you do not specify *any* columns for a snowpiped table, dbt will also
+        # include `value`, the JSON blob of all file contents.
+  
\ No newline at end of file

From c09fc3f36f5a4b168ce51b4c07cf013da310b37f Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Fri, 25 Jun 2021 11:57:14 -0400
Subject: [PATCH 19/22] Wrap Snowflake DML in explicit transactions

---
 macros/plugins/snowflake/refresh_external_table.sql | 4 ++--
 macros/plugins/snowflake/snowpipe/get_copy_sql.sql  | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/macros/plugins/snowflake/refresh_external_table.sql b/macros/plugins/snowflake/refresh_external_table.sql
index 4f3da15b..8f28ae57 100644
--- a/macros/plugins/snowflake/refresh_external_table.sql
+++ b/macros/plugins/snowflake/refresh_external_table.sql
@@ -11,9 +11,9 @@
     {% if manual_refresh %}
 
         {% set ddl %}
-        BEGIN;
+        begin;
         alter external table {{source(source_node.source_name, source_node.name)}} refresh;
-        COMMIT;
+        commit;
         {% endset %}
         
         {% do return([ddl]) %}
diff --git a/macros/plugins/snowflake/snowpipe/get_copy_sql.sql b/macros/plugins/snowflake/snowpipe/get_copy_sql.sql
index b313fbc6..708e2451 100644
--- a/macros/plugins/snowflake/snowpipe/get_copy_sql.sql
+++ b/macros/plugins/snowflake/snowpipe/get_copy_sql.sql
@@ -6,6 +6,7 @@
     {%- set is_csv = dbt_external_tables.is_csv(external.file_format) %}
     {%- set copy_options = external.snowpipe.get('copy_options', none) -%}
     
+    begin;
     copy into {{source(source_node.source_name, source_node.name)}}
     from ( 
         select
@@ -27,6 +28,7 @@
         from {{external.location}} {# stage #}
     )
     file_format = {{external.file_format}}
-    {% if copy_options %} {{copy_options}} {% endif %}
+    {% if copy_options %} {{copy_options}} {% endif %};
+    commit;
 
 {% endmacro %}

From 6a05dc9a17a57be6c27465c1ae594c1535a87c6e Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Tue, 6 Jul 2021 22:24:03 -0400
Subject: [PATCH 20/22] Revert previous attempt at committing Snowflake DML via
 implicit transactions (#103)

* Revert "Commit transactions on Snowflake"

* Small touchups to transaction logic
---
 integration_tests/macros/common/cleanup_external.sql   |  8 --------
 .../macros/plugins/snowflake/cleanup_external.sql      | 10 ----------
 .../macros/plugins/snowflake/prep_external.sql         |  7 -------
 macros/common/stage_external_sources.sql               |  2 +-
 macros/plugins/snowflake/get_external_build_plan.sql   |  2 +-
 macros/plugins/snowflake/helpers/transaction.sql       |  3 ---
 macros/plugins/snowflake/snowpipe/get_copy_sql.sql     |  8 +++++---
 run_test.sh                                            |  1 -
 8 files changed, 7 insertions(+), 34 deletions(-)
 delete mode 100644 integration_tests/macros/common/cleanup_external.sql
 delete mode 100644 integration_tests/macros/plugins/snowflake/cleanup_external.sql
 delete mode 100644 macros/plugins/snowflake/helpers/transaction.sql

diff --git a/integration_tests/macros/common/cleanup_external.sql b/integration_tests/macros/common/cleanup_external.sql
deleted file mode 100644
index 5ddcaf19..00000000
--- a/integration_tests/macros/common/cleanup_external.sql
+++ /dev/null
@@ -1,8 +0,0 @@
-{% macro cleanup_external() %}
-    {{ return(adapter.dispatch('cleanup_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }}
-{% endmacro %}
-
-{% macro default__cleanup_external() %}
-    {% do log('No cleanup necessary, skipping', info = true) %}
-    {# noop #}
-{% endmacro %}
diff --git a/integration_tests/macros/plugins/snowflake/cleanup_external.sql b/integration_tests/macros/plugins/snowflake/cleanup_external.sql
deleted file mode 100644
index 305b9f48..00000000
--- a/integration_tests/macros/plugins/snowflake/cleanup_external.sql
+++ /dev/null
@@ -1,10 +0,0 @@
-{% macro snowflake__cleanup_external() %}
-
-    {% set unset_autocommit %}
-        alter user {{ target.user }} unset autocommit;
-    {% endset %}
-
-    {% do log('Unsetting autocommit parameter for user ' ~ target.user, info = true) %}
-    {% do run_query(unset_autocommit) %}
-    
-{% endmacro %}
diff --git a/integration_tests/macros/plugins/snowflake/prep_external.sql b/integration_tests/macros/plugins/snowflake/prep_external.sql
index 6d33ca0b..5dd4dca9 100644
--- a/integration_tests/macros/plugins/snowflake/prep_external.sql
+++ b/integration_tests/macros/plugins/snowflake/prep_external.sql
@@ -13,11 +13,4 @@
     {% do log('Creating external stage ' ~ external_stage, info = true) %}
     {% do run_query(create_external_stage) %}
     
-    {% set set_autocommit_false %}
-        alter user {{ target.user }} set autocommit = false;
-    {% endset %}
-
-    {% do log('Turning off autocommit for user ' ~ target.user, info = true) %}
-    {% do run_query(set_autocommit_false) %}
-    
 {% endmacro %}
diff --git a/macros/common/stage_external_sources.sql b/macros/common/stage_external_sources.sql
index de17b21c..21d885cd 100644
--- a/macros/common/stage_external_sources.sql
+++ b/macros/common/stage_external_sources.sql
@@ -49,7 +49,7 @@
         
         {% for q in run_queue %}
         
-            {% set q_msg = q|trim %}
+            {% set q_msg = q|replace('\n','')|replace('begin;','')|trim %}
             {% set q_log = q_msg[:50] ~ '...  ' if q_msg|length > 50 else q_msg %}
         
             {% do dbt_utils.log_info(loop_label ~ ' (' ~ loop.index ~ ') ' ~ q_log) %}
diff --git a/macros/plugins/snowflake/get_external_build_plan.sql b/macros/plugins/snowflake/get_external_build_plan.sql
index ec930b0d..0f73a0b2 100644
--- a/macros/plugins/snowflake/get_external_build_plan.sql
+++ b/macros/plugins/snowflake/get_external_build_plan.sql
@@ -15,7 +15,7 @@
         {% if create_or_replace %}
             {% set build_plan = build_plan + [
                 dbt_external_tables.snowflake_create_empty_table(source_node),
-                dbt_external_tables.snowflake_get_copy_sql(source_node),
+                dbt_external_tables.snowflake_get_copy_sql(source_node, explicit_transaction=true),
                 dbt_external_tables.snowflake_create_snowpipe(source_node)
             ] %}
         {% else %}
diff --git a/macros/plugins/snowflake/helpers/transaction.sql b/macros/plugins/snowflake/helpers/transaction.sql
deleted file mode 100644
index 15b3e003..00000000
--- a/macros/plugins/snowflake/helpers/transaction.sql
+++ /dev/null
@@ -1,3 +0,0 @@
-{% macro snowflake__exit_transaction() %}
-    {{ return('begin; commit;') }}
-{% endmacro %}
diff --git a/macros/plugins/snowflake/snowpipe/get_copy_sql.sql b/macros/plugins/snowflake/snowpipe/get_copy_sql.sql
index 708e2451..b4cdc42e 100644
--- a/macros/plugins/snowflake/snowpipe/get_copy_sql.sql
+++ b/macros/plugins/snowflake/snowpipe/get_copy_sql.sql
@@ -1,4 +1,4 @@
-{% macro snowflake_get_copy_sql(source_node) %}
+{% macro snowflake_get_copy_sql(source_node, explicit_transaction=false) %}
 {# This assumes you have already created an external stage #}
 
     {%- set columns = source_node.columns.values() -%}
@@ -6,7 +6,8 @@
     {%- set is_csv = dbt_external_tables.is_csv(external.file_format) %}
     {%- set copy_options = external.snowpipe.get('copy_options', none) -%}
     
-    begin;
+    {%- if explicit_transaction -%} begin; {%- endif %}
+    
     copy into {{source(source_node.source_name, source_node.name)}}
     from ( 
         select
@@ -29,6 +30,7 @@
     )
     file_format = {{external.file_format}}
     {% if copy_options %} {{copy_options}} {% endif %};
-    commit;
+    
+    {% if explicit_transaction -%} commit; {%- endif -%}
 
 {% endmacro %}
diff --git a/run_test.sh b/run_test.sh
index b9f61de6..a4faf193 100755
--- a/run_test.sh
+++ b/run_test.sh
@@ -37,4 +37,3 @@ dbt run-operation prep_external --target $1
 dbt run-operation stage_external_sources --var 'ext_full_refresh: true' --target $1
 dbt run-operation stage_external_sources --target $1
 dbt test --target $1
-dbt run-operation cleanup_external --target $1

From 850d5311d7d114ab749f07fe39f398fd6921a427 Mon Sep 17 00:00:00 2001
From: Jeremy Cohen <jeremy@fishtownanalytics.com>
Date: Tue, 6 Jul 2021 22:30:27 -0400
Subject: [PATCH 21/22] Upgrade dbt v0.20, dbt-utils v0.7 (#99)

---
 README.md                                               | 6 +++---
 dbt_project.yml                                         | 4 ++--
 integration_tests/dbt_project.yml                       | 5 +++--
 integration_tests/macros/common/prep_external.sql       | 2 +-
 macros/common/create_external_table.sql                 | 4 +---
 macros/common/get_external_build_plan.sql               | 4 +---
 macros/common/helpers/dropif.sql                        | 4 +---
 macros/common/helpers/get_external_tables_namespace.sql | 4 ----
 macros/common/helpers/transaction.sql                   | 2 +-
 macros/common/refresh_external_table.sql                | 4 +---
 packages.yml                                            | 4 ++--
 run_test.sh                                             | 6 +++---
 12 files changed, 19 insertions(+), 30 deletions(-)
 delete mode 100644 macros/common/helpers/get_external_tables_namespace.sql

diff --git a/README.md b/README.md
index 52460208..fbe4357b 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # External sources in dbt
 
-dbt v0.15.0 [added support](https://github.com/fishtown-analytics/dbt/pull/1784) for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
+dbt v0.15.0 [added support](https://github.com/dbt-labs/dbt/pull/1784) for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
 
 This package provides:
 * Macros to create/replace external tables and refresh their partitions, using the metadata provided in your `.yml` file source definitions
@@ -19,7 +19,7 @@ This package provides:
 
 ## Installation
 
-Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/fishtown-analytics/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`.
+Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/dbt-labs/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`.
 
 ## Syntax
 
@@ -115,7 +115,7 @@ execute the appropriate `create`, `refresh`, and/or `drop` commands:
 `stage_external_sources` runs as an operation
 * [`tested specs`](integration_tests/models/plugins): source spec variations that are confirmed to work on each database, via integration tests
 
-If you encounter issues using this package or have questions, please check the [open issues](https://github.com/fishtown-analytics/dbt-external-tables/issues), as there's a chance it's a known limitation or work in progress. If not, you can:
+If you encounter issues using this package or have questions, please check the [open issues](https://github.com/dbt-labs/dbt-external-tables/issues), as there's a chance it's a known limitation or work in progress. If not, you can:
 - open a new issue to report a bug or suggest an enhancement
 - post a technical question to [StackOverflow](https://stackoverflow.com/questions/tagged/dbt)
 - post a conceptual question to the relevant database channel (#db-redshift, #dbt-snowflake, etc) in the [dbt Slack community](https://community.getdbt.com/)
diff --git a/dbt_project.yml b/dbt_project.yml
index a147dfe3..44e0151a 100644
--- a/dbt_project.yml
+++ b/dbt_project.yml
@@ -1,9 +1,9 @@
 name: 'dbt_external_tables'
-version: '0.3.0'
+version: '0.7.0'
 
 config-version: 2
 
-require-dbt-version: ">=0.18.0"
+require-dbt-version: ">=0.20.0"
 
 source-paths: ["models"]
 analysis-paths: ["analysis"]
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
index a888c424..2bb3ce2d 100644
--- a/integration_tests/dbt_project.yml
+++ b/integration_tests/dbt_project.yml
@@ -17,8 +17,9 @@ clean-targets:
     - "target"
     - "dbt_modules"
 
-vars:
-  dbt_external_tables_dispatch_list: ['dbt_external_tables_integration_tests']
+dispatch:
+  - macro_namespace: dbt_external_tables
+    search_order: ['dbt_external_tables_integration_tests', 'dbt_external_tables']
 
 seeds:
   +quote_columns: false
diff --git a/integration_tests/macros/common/prep_external.sql b/integration_tests/macros/common/prep_external.sql
index 0430e4a6..de754c7e 100644
--- a/integration_tests/macros/common/prep_external.sql
+++ b/integration_tests/macros/common/prep_external.sql
@@ -1,5 +1,5 @@
 {% macro prep_external() %}
-    {{ return(adapter.dispatch('prep_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }}
+    {{ return(adapter.dispatch('prep_external', 'dbt_external_tables')()) }}
 {% endmacro %}
 
 {% macro default__prep_external() %}
diff --git a/macros/common/create_external_table.sql b/macros/common/create_external_table.sql
index f7256c21..d4f6b1cd 100644
--- a/macros/common/create_external_table.sql
+++ b/macros/common/create_external_table.sql
@@ -1,7 +1,5 @@
 {% macro create_external_table(source_node) %}
-    {{ adapter.dispatch('create_external_table', 
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces()) 
-        (source_node) }}
+    {{ adapter.dispatch('create_external_table', 'dbt_external_tables')(source_node) }}
 {% endmacro %}
 
 {% macro default__create_external_table(source_node) %}
diff --git a/macros/common/get_external_build_plan.sql b/macros/common/get_external_build_plan.sql
index 240385fa..d94d7e01 100644
--- a/macros/common/get_external_build_plan.sql
+++ b/macros/common/get_external_build_plan.sql
@@ -1,7 +1,5 @@
 {% macro get_external_build_plan(source_node) %}
-    {{ return(adapter.dispatch('get_external_build_plan',
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces())
-        (source_node)) }}
+    {{ return(adapter.dispatch('get_external_build_plan', 'dbt_external_tables')(source_node)) }}
 {% endmacro %}
 
 {% macro default__get_external_build_plan(source_node) %}
diff --git a/macros/common/helpers/dropif.sql b/macros/common/helpers/dropif.sql
index 1fcca20b..dfb91065 100644
--- a/macros/common/helpers/dropif.sql
+++ b/macros/common/helpers/dropif.sql
@@ -1,7 +1,5 @@
 {% macro dropif(node) %}
-    {{ adapter.dispatch('dropif', 
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces()) 
-        (node) }}
+    {{ adapter.dispatch('dropif', 'dbt_external_tables')(node) }}
 {% endmacro %}
 
 {% macro default__dropif() %}
diff --git a/macros/common/helpers/get_external_tables_namespace.sql b/macros/common/helpers/get_external_tables_namespace.sql
deleted file mode 100644
index 4dd60ca1..00000000
--- a/macros/common/helpers/get_external_tables_namespace.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-{% macro _get_dbt_external_tables_namespaces() %}
-  {% set override_namespaces = var('dbt_external_tables_dispatch_list', []) %}
-  {% do return(override_namespaces + ['dbt_external_tables']) %}
-{% endmacro %}
diff --git a/macros/common/helpers/transaction.sql b/macros/common/helpers/transaction.sql
index 569108ec..89124859 100644
--- a/macros/common/helpers/transaction.sql
+++ b/macros/common/helpers/transaction.sql
@@ -1,5 +1,5 @@
 {% macro exit_transaction() %}
-    {{ return(adapter.dispatch('exit_transaction', dbt_external_tables._get_dbt_external_tables_namespaces())()) }}
+    {{ return(adapter.dispatch('exit_transaction', 'dbt_external_tables')()) }}
 {% endmacro %}
 
 {% macro default__exit_transaction() %}
diff --git a/macros/common/refresh_external_table.sql b/macros/common/refresh_external_table.sql
index 1b9748be..6b4ca53e 100644
--- a/macros/common/refresh_external_table.sql
+++ b/macros/common/refresh_external_table.sql
@@ -1,7 +1,5 @@
 {% macro refresh_external_table(source_node) %}
-    {{ return(adapter.dispatch('refresh_external_table', 
-        packages = dbt_external_tables._get_dbt_external_tables_namespaces()) 
-        (source_node)) }}
+    {{ return(adapter.dispatch('refresh_external_table', 'dbt_external_tables')(source_node)) }}
 {% endmacro %}
 
 {% macro default__refresh_external_table(source_node) %}
diff --git a/packages.yml b/packages.yml
index 2246ac8b..5302a1f2 100644
--- a/packages.yml
+++ b/packages.yml
@@ -1,3 +1,3 @@
 packages:
-  - package: fishtown-analytics/dbt_utils
-    version: [">=0.6.0", "<0.7.0"]
+  - package: dbt-labs/dbt_utils
+    version: [">=0.7.0", "<0.8.0"]
diff --git a/run_test.sh b/run_test.sh
index a4faf193..aa03d9d3 100755
--- a/run_test.sh
+++ b/run_test.sh
@@ -9,14 +9,14 @@ if [[ ! -f $VENV ]]; then
     if [ $1 == 'databricks' ]
     then
         echo "Installing dbt-spark"
-        pip install dbt-spark[ODBC] --upgrade
+        pip install dbt-spark[ODBC] --upgrade --pre
     elif [ $1 == 'azuresql' ]
     then
         echo "Installing dbt-sqlserver"
-        pip install dbt-sqlserver --upgrade
+        pip install dbt-sqlserver --upgrade --pre
     else
         echo "Installing dbt-$1"
-        pip install dbt-$1 --upgrade
+        pip install dbt-$1 --upgrade --pre
     fi
 fi
 

From 29ff5f46d6a5e455eead48dff97aee3c817baa17 Mon Sep 17 00:00:00 2001
From: Scott Barber <74067474+barberscott@users.noreply.github.com>
Date: Fri, 13 Aug 2021 19:36:29 -0500
Subject: [PATCH 22/22] Explicitly set auto_refresh if populated

If auto_refresh is set to false, we need to explicitly declare it -- do not include only if not included in the yml
---
 macros/plugins/snowflake/create_external_table.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/macros/plugins/snowflake/create_external_table.sql b/macros/plugins/snowflake/create_external_table.sql
index c29e5d25..3956f3b4 100644
--- a/macros/plugins/snowflake/create_external_table.sql
+++ b/macros/plugins/snowflake/create_external_table.sql
@@ -27,7 +27,7 @@
     {%- endif -%}
     {% if partitions %} partition by ({{partitions|map(attribute='name')|join(', ')}}) {% endif %}
     location = {{external.location}} {# stage #}
-    {% if external.auto_refresh -%} auto_refresh = {{external.auto_refresh}} {%- endif %}
+    {% if external.auto_refresh is not none -%} auto_refresh = {{external.auto_refresh}} {%- endif %}
     {% if external.pattern -%} pattern = '{{external.pattern}}' {%- endif %}
     {% if external.integration -%} integration = '{{external.integration}}' {%- endif %}
     file_format = {{external.file_format}}