Skip to content

Commit 1551cbd

Browse files
prmoore77claude
andcommitted
Feat(gizmosql): Use temporary tables for ADBC bulk ingestion
Use the temporary=True flag with adbc_ingest for more efficient DataFrame loading. DuckDB temporary tables cannot have catalog/schema prefixes, so we reference them by name only. Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 1040ee1 commit 1551cbd

File tree

1 file changed

+9
-11
lines changed

1 file changed

+9
-11
lines changed

sqlmesh/core/engine_adapter/gizmosql.py

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -155,12 +155,7 @@ def _df_to_source_queries(
155155
"""
156156
import pyarrow as pa
157157

158-
# Generate a simple temp table name without schema prefix
159-
# adbc_ingest creates tables in the current schema and treats the full
160-
# string as a literal table name (doesn't parse schema.table)
161158
temp_table = self._get_temp_table(target_table)
162-
# Extract just the table name without schema/catalog
163-
temp_table_name = temp_table.name
164159

165160
# Select only the source columns in the right order
166161
source_columns_to_types = (
@@ -173,22 +168,25 @@ def _df_to_source_queries(
173168
# Convert DataFrame to PyArrow Table for bulk ingestion
174169
arrow_table = pa.Table.from_pandas(ordered_df)
175170

176-
# Use ADBC bulk ingestion - much faster than row-by-row INSERT
171+
# Use ADBC bulk ingestion with temporary table
172+
# Note: DuckDB temporary tables cannot have catalog/schema prefixes,
173+
# so we only pass the table name when temporary=True
177174
self.cursor.adbc_ingest(
178-
table_name=temp_table_name,
175+
table_name=temp_table.name,
179176
data=arrow_table,
180177
mode="create",
178+
temporary=True,
181179
)
182180

183-
# Create a simple table reference for queries (no schema prefix)
184-
temp_table_ref = exp.to_table(temp_table_name)
181+
# Reference temp table by name only (no schema prefix for temporary tables)
182+
temp_table_name = exp.to_table(temp_table.name)
185183

186184
return [
187185
SourceQuery(
188186
query_factory=lambda: self._select_columns(target_columns_to_types).from_(
189-
temp_table_ref
187+
temp_table_name
190188
),
191-
cleanup_func=lambda: self.drop_table(temp_table_ref),
189+
cleanup_func=lambda: self.drop_table(temp_table_name),
192190
)
193191
]
194192

0 commit comments

Comments
 (0)