From 522d98a9dc906c0e434391f0f0d7a52253c2bd3d Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 6 Feb 2026 10:19:44 +0800 Subject: [PATCH 01/13] AWS: Add chunked encoding configuration for S3 requests --- .../iceberg/aws/s3/S3FileIOProperties.java | 22 +++++++++++++++++++ .../aws/s3/TestS3FileIOProperties.java | 21 ++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java index ad5181fd2798..b2649b558ff4 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java @@ -295,6 +295,18 @@ public class S3FileIOProperties implements Serializable { public static final boolean REMOTE_SIGNING_ENABLED_DEFAULT = false; + /** + * Enables or disables chunked encoding for S3 requests. + * + *

This feature is enabled by default. + * + *

For more details see: + * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Configuration.html#chunkedEncodingEnabled() + */ + public static final String CHUNKED_ENCODING_ENABLED = "s3.chunked-encoding-enabled"; + + public static final boolean CHUNKED_ENCODING_ENABLED_DEFAULT = true; + /** Configure the batch size used when deleting multiple files from a given S3 bucket */ public static final String DELETE_BATCH_SIZE = "s3.delete.batch-size"; @@ -509,6 +521,7 @@ public class S3FileIOProperties implements Serializable { private String stagingDirectory; private ObjectCannedACL acl; private boolean isChecksumEnabled; + private boolean isChunkedEncodingEnabled; private final Set writeTags; private boolean isWriteTableTagEnabled; private boolean isWriteNamespaceTagEnabled; @@ -551,6 +564,7 @@ public S3FileIOProperties() { this.deleteBatchSize = DELETE_BATCH_SIZE_DEFAULT; this.stagingDirectory = System.getProperty("java.io.tmpdir"); this.isChecksumEnabled = CHECKSUM_ENABLED_DEFAULT; + this.isChunkedEncodingEnabled = CHUNKED_ENCODING_ENABLED_DEFAULT; this.writeTags = Sets.newHashSet(); this.isWriteTableTagEnabled = WRITE_TABLE_TAG_ENABLED_DEFAULT; this.isWriteNamespaceTagEnabled = WRITE_NAMESPACE_TAG_ENABLED_DEFAULT; @@ -641,6 +655,9 @@ public S3FileIOProperties(Map properties) { "Cannot support S3 CannedACL " + aclType); this.isChecksumEnabled = PropertyUtil.propertyAsBoolean(properties, CHECKSUM_ENABLED, CHECKSUM_ENABLED_DEFAULT); + this.isChunkedEncodingEnabled = + PropertyUtil.propertyAsBoolean( + properties, CHUNKED_ENCODING_ENABLED, CHUNKED_ENCODING_ENABLED_DEFAULT); this.deleteBatchSize = PropertyUtil.propertyAsInt(properties, DELETE_BATCH_SIZE, DELETE_BATCH_SIZE_DEFAULT); Preconditions.checkArgument( @@ -808,6 +825,10 @@ public boolean isChecksumEnabled() { return this.isChecksumEnabled; } + public boolean isChunkedEncodingEnabled() { + return this.isChunkedEncodingEnabled; + } + public boolean isRemoteSigningEnabled() { return this.isRemoteSigningEnabled; } @@ -994,6 +1015,7 @@ public void applyServiceConfigurations(T builder) { .pathStyleAccessEnabled(isPathStyleAccess) .useArnRegionEnabled(isUseArnRegionEnabled) .accelerateModeEnabled(isAccelerationEnabled) + .chunkedEncodingEnabled(isChunkedEncodingEnabled) .build()); } diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java index 1666de1f1d08..953f73d45d4a 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java @@ -566,4 +566,25 @@ public void testApplyRetryConfiguration() { RetryPolicy retryPolicy = builder.overrideConfiguration().retryPolicy().get(); assertThat(retryPolicy.numRetries()).as("retries was not set").isEqualTo(999); } + + @Test + public void testChunkedEncodingEnabledDefaultValue() { + Map properties = Maps.newHashMap(); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); + + assertThat(s3FileIOProperties.isChunkedEncodingEnabled()) + .as("chunked encoding should be enabled by default") + .isTrue(); + } + + @Test + public void testChunkedEncodingDisabled() { + Map properties = Maps.newHashMap(); + properties.put(S3FileIOProperties.CHUNKED_ENCODING_ENABLED, "false"); + S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); + + assertThat(s3FileIOProperties.isChunkedEncodingEnabled()) + .as("chunked encoding should be disabled when explicitly set to false") + .isFalse(); + } } From 5661827791e98d3029da724c06541cec8ca41e78 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 10 Feb 2026 14:24:42 +0800 Subject: [PATCH 02/13] add testMultipartUploadWithChunkedEncodingDisabled --- .../iceberg/aws/s3/TestS3MultipartUpload.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java index cbe3051a6711..8b3ad12da689 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java @@ -141,6 +141,37 @@ public void testParallelUpload() throws IOException { } } + @Test + public void testMultipartUploadWithChunkedEncodingDisabled() throws IOException { + // Create a new S3FileIO with chunked encoding disabled + S3FileIO ioWithoutChunkedEncoding = new S3FileIO(() -> s3); + ioWithoutChunkedEncoding.initialize( + ImmutableMap.of( + S3FileIOProperties.MULTIPART_SIZE, + Integer.toString(S3FileIOProperties.MULTIPART_SIZE_MIN), + S3FileIOProperties.CHECKSUM_ENABLED, + "true", + S3FileIOProperties.CHUNKED_ENCODING_ENABLED, + "false")); + + String testObjectUri = objectUri + "-no-chunked-encoding"; + int parts = 10; + + // Write data with chunked encoding disabled + try (PositionOutputStream outputStream = + ioWithoutChunkedEncoding.newOutputFile(testObjectUri).create()) { + for (int i = 0; i < parts; i++) { + for (long j = 0; j < S3FileIOProperties.MULTIPART_SIZE_MIN; j++) { + outputStream.write(random.nextInt()); + } + } + } + + // Verify the file was uploaded successfully + assertThat(ioWithoutChunkedEncoding.newInputFile(testObjectUri).getLength()) + .isEqualTo(parts * (long) S3FileIOProperties.MULTIPART_SIZE_MIN); + } + private void writeInts(String fileUri, int parts, Supplier writer) { writeInts(fileUri, parts, S3FileIOProperties.MULTIPART_SIZE_MIN, writer); } From 15ab7a5ae03d4223619b4508b7159fdcae9f4efa Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 10 Feb 2026 14:30:03 +0800 Subject: [PATCH 03/13] update open api define --- open-api/rest-catalog-open-api.py | 1 + open-api/rest-catalog-open-api.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 411881cb31ab..0c78b77a6198 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1478,6 +1478,7 @@ class LoadTableResult(BaseModel): - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + - `s3.chunked-encoding-enabled`: if `true`, chunked encoding is enabled for S3 requests ## Storage Credentials diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index fff71128e5e5..f8304dd0850a 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3478,6 +3478,7 @@ components: - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled + - `s3.chunked-encoding-enabled`: if `true`, chunked encoding is enabled for S3 requests ## Storage Credentials From 345231cf68c3e22324bcab944eab63b3120d9f70 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 10 Feb 2026 17:32:30 +0800 Subject: [PATCH 04/13] update --- .../iceberg/aws/s3/TestS3MultipartUpload.java | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java index 8b3ad12da689..d81d88b2ef52 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java @@ -36,6 +36,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariables; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import software.amazon.awssdk.services.s3.S3Client; /** Long-running tests to ensure multipart upload logic is resilient */ @@ -141,25 +143,27 @@ public void testParallelUpload() throws IOException { } } - @Test - public void testMultipartUploadWithChunkedEncodingDisabled() throws IOException { - // Create a new S3FileIO with chunked encoding disabled - S3FileIO ioWithoutChunkedEncoding = new S3FileIO(() -> s3); - ioWithoutChunkedEncoding.initialize( + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testMultipartUploadWithChunkedEncoding(boolean chunkedEncodingEnabled) + throws IOException { + // Create a new S3FileIO with specified chunked encoding setting + S3FileIO testIo = new S3FileIO(() -> s3); + testIo.initialize( ImmutableMap.of( S3FileIOProperties.MULTIPART_SIZE, Integer.toString(S3FileIOProperties.MULTIPART_SIZE_MIN), S3FileIOProperties.CHECKSUM_ENABLED, "true", S3FileIOProperties.CHUNKED_ENCODING_ENABLED, - "false")); + Boolean.toString(chunkedEncodingEnabled))); - String testObjectUri = objectUri + "-no-chunked-encoding"; + String testObjectUri = + objectUri + (chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled"); int parts = 10; - // Write data with chunked encoding disabled - try (PositionOutputStream outputStream = - ioWithoutChunkedEncoding.newOutputFile(testObjectUri).create()) { + // Write data with specified chunked encoding setting + try (PositionOutputStream outputStream = testIo.newOutputFile(testObjectUri).create()) { for (int i = 0; i < parts; i++) { for (long j = 0; j < S3FileIOProperties.MULTIPART_SIZE_MIN; j++) { outputStream.write(random.nextInt()); @@ -168,7 +172,7 @@ public void testMultipartUploadWithChunkedEncodingDisabled() throws IOException } // Verify the file was uploaded successfully - assertThat(ioWithoutChunkedEncoding.newInputFile(testObjectUri).getLength()) + assertThat(testIo.newInputFile(testObjectUri).getLength()) .isEqualTo(parts * (long) S3FileIOProperties.MULTIPART_SIZE_MIN); } From 4e0c68b598e4e1b881bdfb35f89933b350a49746 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 10 Feb 2026 20:19:04 +0800 Subject: [PATCH 05/13] update default value --- .../org/apache/iceberg/aws/s3/S3FileIOProperties.java | 4 ++-- .../apache/iceberg/aws/s3/TestS3FileIOProperties.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java index b2649b558ff4..9ad6f510f33f 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java @@ -298,14 +298,14 @@ public class S3FileIOProperties implements Serializable { /** * Enables or disables chunked encoding for S3 requests. * - *

This feature is enabled by default. + *

This feature is disabled by default. * *

For more details see: * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Configuration.html#chunkedEncodingEnabled() */ public static final String CHUNKED_ENCODING_ENABLED = "s3.chunked-encoding-enabled"; - public static final boolean CHUNKED_ENCODING_ENABLED_DEFAULT = true; + public static final boolean CHUNKED_ENCODING_ENABLED_DEFAULT = false; /** Configure the batch size used when deleting multiple files from a given S3 bucket */ public static final String DELETE_BATCH_SIZE = "s3.delete.batch-size"; diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java index 953f73d45d4a..7f9cfcffca8e 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java @@ -573,18 +573,18 @@ public void testChunkedEncodingEnabledDefaultValue() { S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); assertThat(s3FileIOProperties.isChunkedEncodingEnabled()) - .as("chunked encoding should be enabled by default") - .isTrue(); + .as("chunked encoding should be disabled by default") + .isFalse(); } @Test - public void testChunkedEncodingDisabled() { + public void testChunkedEncodingEnabled() { Map properties = Maps.newHashMap(); - properties.put(S3FileIOProperties.CHUNKED_ENCODING_ENABLED, "false"); + properties.put(S3FileIOProperties.CHUNKED_ENCODING_ENABLED, "true"); S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); assertThat(s3FileIOProperties.isChunkedEncodingEnabled()) - .as("chunked encoding should be disabled when explicitly set to false") + .as("chunked encoding should be enabled when explicitly set to true") .isFalse(); } } From ada810adacef37b02b3df0dfaa3dc9ed06747732 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Tue, 10 Feb 2026 20:53:34 +0800 Subject: [PATCH 06/13] update case --- .../java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java index 7f9cfcffca8e..b8d631952dd3 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java @@ -585,6 +585,6 @@ public void testChunkedEncodingEnabled() { assertThat(s3FileIOProperties.isChunkedEncodingEnabled()) .as("chunked encoding should be enabled when explicitly set to true") - .isFalse(); + .isTrue(); } } From 9c287e5b2ed723d587c218ba7a428f6aa660e025 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 27 Feb 2026 17:27:10 +0800 Subject: [PATCH 07/13] assert file contents in testMultipartUploadWithChunkedEncoding --- .../iceberg/aws/s3/TestS3MultipartUpload.java | 70 +++++++++++++++---- 1 file changed, 55 insertions(+), 15 deletions(-) diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java index d81d88b2ef52..6e2a4b76c816 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java @@ -21,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; +import java.util.Arrays; import java.util.Random; import java.util.UUID; import java.util.function.Supplier; @@ -158,22 +159,30 @@ public void testMultipartUploadWithChunkedEncoding(boolean chunkedEncodingEnable S3FileIOProperties.CHUNKED_ENCODING_ENABLED, Boolean.toString(chunkedEncodingEnabled))); - String testObjectUri = - objectUri + (chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled"); int parts = 10; - // Write data with specified chunked encoding setting - try (PositionOutputStream outputStream = testIo.newOutputFile(testObjectUri).create()) { - for (int i = 0; i < parts; i++) { - for (long j = 0; j < S3FileIOProperties.MULTIPART_SIZE_MIN; j++) { - outputStream.write(random.nextInt()); - } - } - } + // Verify write and read with int + int expectedInt = 42; + String intObjectUri = + objectUri + (chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled") + "-int"; + writeInts( + testIo, intObjectUri, parts, S3FileIOProperties.MULTIPART_SIZE_MIN, () -> expectedInt); - // Verify the file was uploaded successfully - assertThat(testIo.newInputFile(testObjectUri).getLength()) + assertThat(testIo.newInputFile(intObjectUri).getLength()) .isEqualTo(parts * (long) S3FileIOProperties.MULTIPART_SIZE_MIN); + verifyInts(testIo, intObjectUri, () -> expectedInt); + + // Verify write and read with bytes + byte expectedByte = 42; + String bytesObjectUri = + objectUri + (chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled") + "-bytes"; + byte[] expectedBytes = new byte[S3FileIOProperties.MULTIPART_SIZE_MIN]; + Arrays.fill(expectedBytes, expectedByte); + writeBytes(testIo, bytesObjectUri, parts, () -> expectedBytes); + + assertThat(testIo.newInputFile(bytesObjectUri).getLength()) + .isEqualTo(parts * (long) S3FileIOProperties.MULTIPART_SIZE_MIN); + verifyBytes(testIo, bytesObjectUri, () -> expectedByte); } private void writeInts(String fileUri, int parts, Supplier writer) { @@ -181,7 +190,12 @@ private void writeInts(String fileUri, int parts, Supplier writer) { } private void writeInts(String fileUri, int parts, long partSize, Supplier writer) { - try (PositionOutputStream outputStream = io.newOutputFile(fileUri).create()) { + writeInts(io, fileUri, parts, partSize, writer); + } + + private void writeInts( + S3FileIO fileIO, String fileUri, int parts, long partSize, Supplier writer) { + try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) { for (int i = 0; i < parts; i++) { for (long j = 0; j < partSize; j++) { outputStream.write(writer.get()); @@ -193,7 +207,11 @@ private void writeInts(String fileUri, int parts, long partSize, Supplier verifier) { - try (SeekableInputStream inputStream = io.newInputFile(fileUri).newStream()) { + verifyInts(io, fileUri, verifier); + } + + private void verifyInts(S3FileIO fileIO, String fileUri, Supplier verifier) { + try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) { int cur; while ((cur = inputStream.read()) != -1) { assertThat(cur).isEqualTo(verifier.get()); @@ -204,7 +222,11 @@ private void verifyInts(String fileUri, Supplier verifier) { } private void writeBytes(String fileUri, int parts, Supplier writer) { - try (PositionOutputStream outputStream = io.newOutputFile(fileUri).create()) { + writeBytes(io, fileUri, parts, writer); + } + + private void writeBytes(S3FileIO fileIO, String fileUri, int parts, Supplier writer) { + try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) { for (int i = 0; i < parts; i++) { outputStream.write(writer.get()); } @@ -212,4 +234,22 @@ private void writeBytes(String fileUri, int parts, Supplier writer) { throw new RuntimeException(e); } } + + private void verifyBytes(String fileUri, Supplier verifier) { + verifyBytes(io, fileUri, verifier); + } + + private void verifyBytes(S3FileIO fileIO, String fileUri, Supplier verifier) { + try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) { + byte[] readBuffer = new byte[S3FileIOProperties.MULTIPART_SIZE_MIN]; + int bytesRead; + while ((bytesRead = inputStream.read(readBuffer)) != -1) { + for (int i = 0; i < bytesRead; i++) { + assertThat(readBuffer[i]).isEqualTo(verifier.get()); + } + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } } From 496e8f5b3a65a63b7c12a78507e087ea343ebe02 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 27 Feb 2026 21:22:15 +0800 Subject: [PATCH 08/13] Remove s3.chunked-encoding-enabled config entry from REST catalog open API spec --- .../iceberg/aws/s3/TestS3MultipartUpload.java | 105 ++++++++++-------- open-api/rest-catalog-open-api.py | 1 - open-api/rest-catalog-open-api.yaml | 1 - 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java index 6e2a4b76c816..5cea95f8ac61 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java @@ -160,29 +160,16 @@ public void testMultipartUploadWithChunkedEncoding(boolean chunkedEncodingEnable Boolean.toString(chunkedEncodingEnabled))); int parts = 10; + long partSize = S3FileIOProperties.MULTIPART_SIZE_MIN; + String suffix = chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled"; - // Verify write and read with int - int expectedInt = 42; - String intObjectUri = - objectUri + (chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled") + "-int"; - writeInts( - testIo, intObjectUri, parts, S3FileIOProperties.MULTIPART_SIZE_MIN, () -> expectedInt); + String intObjectUri = objectUri + suffix + "-int"; + writeDistinctPartsWithInts(testIo, intObjectUri, parts, partSize); + verifyDistinctPartsWithInts(testIo, intObjectUri, parts, partSize); - assertThat(testIo.newInputFile(intObjectUri).getLength()) - .isEqualTo(parts * (long) S3FileIOProperties.MULTIPART_SIZE_MIN); - verifyInts(testIo, intObjectUri, () -> expectedInt); - - // Verify write and read with bytes - byte expectedByte = 42; - String bytesObjectUri = - objectUri + (chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled") + "-bytes"; - byte[] expectedBytes = new byte[S3FileIOProperties.MULTIPART_SIZE_MIN]; - Arrays.fill(expectedBytes, expectedByte); - writeBytes(testIo, bytesObjectUri, parts, () -> expectedBytes); - - assertThat(testIo.newInputFile(bytesObjectUri).getLength()) - .isEqualTo(parts * (long) S3FileIOProperties.MULTIPART_SIZE_MIN); - verifyBytes(testIo, bytesObjectUri, () -> expectedByte); + String bytesObjectUri = objectUri + suffix + "-bytes"; + writeDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize); + verifyDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize); } private void writeInts(String fileUri, int parts, Supplier writer) { @@ -190,12 +177,7 @@ private void writeInts(String fileUri, int parts, Supplier writer) { } private void writeInts(String fileUri, int parts, long partSize, Supplier writer) { - writeInts(io, fileUri, parts, partSize, writer); - } - - private void writeInts( - S3FileIO fileIO, String fileUri, int parts, long partSize, Supplier writer) { - try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) { + try (PositionOutputStream outputStream = io.newOutputFile(fileUri).create()) { for (int i = 0; i < parts; i++) { for (long j = 0; j < partSize; j++) { outputStream.write(writer.get()); @@ -207,11 +189,7 @@ private void writeInts( } private void verifyInts(String fileUri, Supplier verifier) { - verifyInts(io, fileUri, verifier); - } - - private void verifyInts(S3FileIO fileIO, String fileUri, Supplier verifier) { - try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) { + try (SeekableInputStream inputStream = io.newInputFile(fileUri).newStream()) { int cur; while ((cur = inputStream.read()) != -1) { assertThat(cur).isEqualTo(verifier.get()); @@ -222,11 +200,7 @@ private void verifyInts(S3FileIO fileIO, String fileUri, Supplier verif } private void writeBytes(String fileUri, int parts, Supplier writer) { - writeBytes(io, fileUri, parts, writer); - } - - private void writeBytes(S3FileIO fileIO, String fileUri, int parts, Supplier writer) { - try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) { + try (PositionOutputStream outputStream = io.newOutputFile(fileUri).create()) { for (int i = 0; i < parts; i++) { outputStream.write(writer.get()); } @@ -235,21 +209,60 @@ private void writeBytes(S3FileIO fileIO, String fileUri, int parts, Supplier verifier) { - verifyBytes(io, fileUri, verifier); + private void writeDistinctPartsWithInts(S3FileIO fileIO, String fileUri, int parts, long partSize) + throws IOException { + try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) { + for (int part = 0; part < parts; part++) { + int partByte = part + 1; + for (long j = 0; j < partSize; j++) { + outputStream.write(partByte); + } + } + } + + assertThat(fileIO.newInputFile(fileUri).getLength()).isEqualTo(parts * partSize); } - private void verifyBytes(S3FileIO fileIO, String fileUri, Supplier verifier) { + private void verifyDistinctPartsWithInts( + S3FileIO fileIO, String fileUri, int parts, long partSize) throws IOException { try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) { - byte[] readBuffer = new byte[S3FileIOProperties.MULTIPART_SIZE_MIN]; - int bytesRead; - while ((bytesRead = inputStream.read(readBuffer)) != -1) { + for (int part = 0; part < parts; part++) { + int expectedValue = part + 1; + for (long j = 0; j < partSize; j++) { + int actual = inputStream.read(); + assertThat(actual).as("part %d, offset %d", part, j).isEqualTo(expectedValue); + } + } + assertThat(inputStream.read()).as("expected end of stream").isEqualTo(-1); + } + } + + private void writeDistinctPartsWithBytes( + S3FileIO fileIO, String fileUri, int parts, long partSize) throws IOException { + try (PositionOutputStream outputStream = fileIO.newOutputFile(fileUri).create()) { + for (int part = 0; part < parts; part++) { + byte[] partBytes = new byte[(int) partSize]; + Arrays.fill(partBytes, (byte) (part + 1)); + outputStream.write(partBytes); + } + } + + assertThat(fileIO.newInputFile(fileUri).getLength()).isEqualTo(parts * partSize); + } + + private void verifyDistinctPartsWithBytes( + S3FileIO fileIO, String fileUri, int parts, long partSize) throws IOException { + try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) { + byte[] readBuffer = new byte[(int) partSize]; + for (int part = 0; part < parts; part++) { + byte expectedByte = (byte) (part + 1); + int bytesRead = inputStream.read(readBuffer); + assertThat(bytesRead).isEqualTo((int) partSize); for (int i = 0; i < bytesRead; i++) { - assertThat(readBuffer[i]).isEqualTo(verifier.get()); + assertThat(readBuffer[i]).as("part %d, offset %d", part, i).isEqualTo(expectedByte); } } - } catch (IOException e) { - throw new RuntimeException(e); + assertThat(inputStream.read()).as("expected end of stream").isEqualTo(-1); } } } diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 0c78b77a6198..411881cb31ab 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1478,7 +1478,6 @@ class LoadTableResult(BaseModel): - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled - - `s3.chunked-encoding-enabled`: if `true`, chunked encoding is enabled for S3 requests ## Storage Credentials diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index f8304dd0850a..fff71128e5e5 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3478,7 +3478,6 @@ components: - `s3.session-token`: if present, this value should be used for as the session token - `s3.remote-signing-enabled`: if `true` remote signing should be performed as described in the `s3-signer-open-api.yaml` specification - `s3.cross-region-access-enabled`: if `true`, S3 Cross-Region bucket access is enabled - - `s3.chunked-encoding-enabled`: if `true`, chunked encoding is enabled for S3 requests ## Storage Credentials From 604fe5983828e0f871489f72294a960c1b0adcf9 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Wed, 4 Mar 2026 23:32:14 +0800 Subject: [PATCH 09/13] Use IOUtil.readFully for reliable reads in TestS3MultipartUpload --- .../iceberg/aws/s3/TestS3MultipartUpload.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java index 5cea95f8ac61..d15828c4c040 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java @@ -28,6 +28,7 @@ import java.util.stream.IntStream; import org.apache.iceberg.aws.AwsClientFactories; import org.apache.iceberg.aws.AwsIntegTestUtil; +import org.apache.iceberg.io.IOUtil; import org.apache.iceberg.io.PositionOutputStream; import org.apache.iceberg.io.SeekableInputStream; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; @@ -226,11 +227,12 @@ private void writeDistinctPartsWithInts(S3FileIO fileIO, String fileUri, int par private void verifyDistinctPartsWithInts( S3FileIO fileIO, String fileUri, int parts, long partSize) throws IOException { try (SeekableInputStream inputStream = fileIO.newInputFile(fileUri).newStream()) { + byte[] readBuffer = new byte[(int) partSize]; for (int part = 0; part < parts; part++) { - int expectedValue = part + 1; - for (long j = 0; j < partSize; j++) { - int actual = inputStream.read(); - assertThat(actual).as("part %d, offset %d", part, j).isEqualTo(expectedValue); + byte expectedByte = (byte) (part + 1); + IOUtil.readFully(inputStream, readBuffer, 0, (int) partSize); + for (int i = 0; i < (int) partSize; i++) { + assertThat(readBuffer[i]).as("part %d, offset %d", part, i).isEqualTo(expectedByte); } } assertThat(inputStream.read()).as("expected end of stream").isEqualTo(-1); @@ -256,9 +258,8 @@ private void verifyDistinctPartsWithBytes( byte[] readBuffer = new byte[(int) partSize]; for (int part = 0; part < parts; part++) { byte expectedByte = (byte) (part + 1); - int bytesRead = inputStream.read(readBuffer); - assertThat(bytesRead).isEqualTo((int) partSize); - for (int i = 0; i < bytesRead; i++) { + IOUtil.readFully(inputStream, readBuffer, 0, (int) partSize); + for (int i = 0; i < (int) partSize; i++) { assertThat(readBuffer[i]).as("part %d, offset %d", part, i).isEqualTo(expectedByte); } } From 5948cebe90e4a55197fc514c8f9c16161b84402c Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 27 Mar 2026 00:31:10 +0800 Subject: [PATCH 10/13] ensure testIo is properly closed --- .../iceberg/aws/s3/TestS3MultipartUpload.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java index d15828c4c040..746015098a40 100644 --- a/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java +++ b/aws/src/integration/java/org/apache/iceberg/aws/s3/TestS3MultipartUpload.java @@ -150,27 +150,28 @@ public void testParallelUpload() throws IOException { public void testMultipartUploadWithChunkedEncoding(boolean chunkedEncodingEnabled) throws IOException { // Create a new S3FileIO with specified chunked encoding setting - S3FileIO testIo = new S3FileIO(() -> s3); - testIo.initialize( - ImmutableMap.of( - S3FileIOProperties.MULTIPART_SIZE, - Integer.toString(S3FileIOProperties.MULTIPART_SIZE_MIN), - S3FileIOProperties.CHECKSUM_ENABLED, - "true", - S3FileIOProperties.CHUNKED_ENCODING_ENABLED, - Boolean.toString(chunkedEncodingEnabled))); + try (S3FileIO testIo = new S3FileIO(() -> s3)) { + testIo.initialize( + ImmutableMap.of( + S3FileIOProperties.MULTIPART_SIZE, + Integer.toString(S3FileIOProperties.MULTIPART_SIZE_MIN), + S3FileIOProperties.CHECKSUM_ENABLED, + "true", + S3FileIOProperties.CHUNKED_ENCODING_ENABLED, + Boolean.toString(chunkedEncodingEnabled))); - int parts = 10; - long partSize = S3FileIOProperties.MULTIPART_SIZE_MIN; - String suffix = chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled"; + int parts = 10; + long partSize = S3FileIOProperties.MULTIPART_SIZE_MIN; + String suffix = chunkedEncodingEnabled ? "-chunked-enabled" : "-chunked-disabled"; - String intObjectUri = objectUri + suffix + "-int"; - writeDistinctPartsWithInts(testIo, intObjectUri, parts, partSize); - verifyDistinctPartsWithInts(testIo, intObjectUri, parts, partSize); + String intObjectUri = objectUri + suffix + "-int"; + writeDistinctPartsWithInts(testIo, intObjectUri, parts, partSize); + verifyDistinctPartsWithInts(testIo, intObjectUri, parts, partSize); - String bytesObjectUri = objectUri + suffix + "-bytes"; - writeDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize); - verifyDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize); + String bytesObjectUri = objectUri + suffix + "-bytes"; + writeDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize); + verifyDistinctPartsWithBytes(testIo, bytesObjectUri, parts, partSize); + } } private void writeInts(String fileUri, int parts, Supplier writer) { From 85ea1f0512d5199d24846bc77bf3f7b8d5134da2 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Fri, 27 Mar 2026 07:52:59 +0800 Subject: [PATCH 11/13] retrigger CI From 0592e17de455cb0e8b8dc9dec54c84123e895d95 Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 5 Apr 2026 09:34:45 +0800 Subject: [PATCH 12/13] Change chunked encoding default to true to match AWS SDK behavior --- .../java/org/apache/iceberg/aws/s3/S3FileIOProperties.java | 4 ++-- .../org/apache/iceberg/aws/s3/TestS3FileIOProperties.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java index 9ad6f510f33f..922010d61d27 100644 --- a/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java +++ b/aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIOProperties.java @@ -298,14 +298,14 @@ public class S3FileIOProperties implements Serializable { /** * Enables or disables chunked encoding for S3 requests. * - *

This feature is disabled by default. + *

This feature is enabled by default to match the AWS SDK default behavior. * *

For more details see: * https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Configuration.html#chunkedEncodingEnabled() */ public static final String CHUNKED_ENCODING_ENABLED = "s3.chunked-encoding-enabled"; - public static final boolean CHUNKED_ENCODING_ENABLED_DEFAULT = false; + public static final boolean CHUNKED_ENCODING_ENABLED_DEFAULT = true; /** Configure the batch size used when deleting multiple files from a given S3 bucket */ public static final String DELETE_BATCH_SIZE = "s3.delete.batch-size"; diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java index b8d631952dd3..5baa1607bf4e 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java @@ -573,8 +573,8 @@ public void testChunkedEncodingEnabledDefaultValue() { S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); assertThat(s3FileIOProperties.isChunkedEncodingEnabled()) - .as("chunked encoding should be disabled by default") - .isFalse(); + .as("chunked encoding should be enabled by default") + .isTrue(); } @Test From 8c001e1833a3b333d9f276617d658af782cdacdc Mon Sep 17 00:00:00 2001 From: Li Jiajia Date: Sun, 5 Apr 2026 09:39:01 +0800 Subject: [PATCH 13/13] Fix test to verify explicit disable of chunked encoding instead of duplicating default --- .../org/apache/iceberg/aws/s3/TestS3FileIOProperties.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java index 5baa1607bf4e..953f73d45d4a 100644 --- a/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java +++ b/aws/src/test/java/org/apache/iceberg/aws/s3/TestS3FileIOProperties.java @@ -578,13 +578,13 @@ public void testChunkedEncodingEnabledDefaultValue() { } @Test - public void testChunkedEncodingEnabled() { + public void testChunkedEncodingDisabled() { Map properties = Maps.newHashMap(); - properties.put(S3FileIOProperties.CHUNKED_ENCODING_ENABLED, "true"); + properties.put(S3FileIOProperties.CHUNKED_ENCODING_ENABLED, "false"); S3FileIOProperties s3FileIOProperties = new S3FileIOProperties(properties); assertThat(s3FileIOProperties.isChunkedEncodingEnabled()) - .as("chunked encoding should be enabled when explicitly set to true") - .isTrue(); + .as("chunked encoding should be disabled when explicitly set to false") + .isFalse(); } }