From e0422766eb7239500d3f0ca240944745b881a977 Mon Sep 17 00:00:00 2001 From: Tobias Cadee Date: Thu, 2 Apr 2026 13:31:47 +0200 Subject: [PATCH] compress S3 uploads with GZIP before COPY to Redshift Co-Authored-By: Claude Sonnet 4.6 --- target_redshift/sinks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/target_redshift/sinks.py b/target_redshift/sinks.py index 72ad8e1..f4e761c 100644 --- a/target_redshift/sinks.py +++ b/target_redshift/sinks.py @@ -160,7 +160,7 @@ def s3_key(self) -> str: Returns: The target s3 key. """ - p = Path(self.config["s3_key_prefix"]) / Path(f"{self.stream_name}-{self.temp_table_name}.csv") + p = Path(self.config["s3_key_prefix"]) / Path(f"{self.stream_name}-{self.temp_table_name}.csv.gz") return str(p) def bulk_insert_records( # type: ignore[override] @@ -302,6 +302,7 @@ def copy_to_redshift(self, table: sqlalchemy.Table, cursor: Cursor) -> None: FROM '{self.s3_uri()}' {copy_credentials} {copy_options} + GZIP CSV """ cursor.execute(copy_sql)