diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java index 514834d243d..d8d5a2438a5 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java @@ -1338,6 +1338,26 @@ public void testGetObject() throws Exception { } } + /** + * Adapted from ceph s3-tests test_object_read_unreadable. + */ + @Test + public void testGetObjectUnreadableKey() { + final String bucketName = getBucketName(); + s3Client.createBucket(bucketName); + + String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, + StandardCharsets.ISO_8859_1); + + AmazonServiceException ase = assertThrows(AmazonServiceException.class, + () -> s3Client.getObject(bucketName, unreadableKey)); + + assertEquals(ErrorType.Client, ase.getErrorType()); + assertEquals(400, ase.getStatusCode()); + assertEquals(S3ErrorTable.INVALID_URI.getCode(), ase.getErrorCode()); + assertEquals(S3ErrorTable.INVALID_URI.getErrorMessage(), ase.getErrorMessage()); + } + static Stream onlyTagKeyCasesV1() { Map fooBarEmptyBar = new HashMap<>(); fooBarEmptyBar.put("foo", "bar"); diff --git a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java index 34cccaca4a9..60ec4892d0a 100644 --- a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java +++ b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java @@ -588,6 +588,25 @@ public void testGetObjectIfModifiedSinceReturnsNotModified() { assertEquals(304, exception.statusCode()); } + /** + * Adapted from ceph s3-tests test_object_read_unreadable. + */ + @Test + public void testGetObjectUnreadableKey() { + final String bucketName = getBucketName(); + s3Client.createBucket(b -> b.bucket(bucketName)); + + String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, + StandardCharsets.ISO_8859_1); + + S3Exception exception = assertThrows(S3Exception.class, + () -> s3Client.getObjectAsBytes(b -> b.bucket(bucketName).key(unreadableKey))); + + assertEquals(400, exception.statusCode()); + assertEquals(S3ErrorTable.INVALID_URI.getCode(), exception.awsErrorDetails().errorCode()); + assertEquals(S3ErrorTable.INVALID_URI.getErrorMessage(), exception.awsErrorDetails().errorMessage()); + } + @Test public void testHeadObjectIfMatch() { final String bucketName = getBucketName(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java index 452303dc4c0..96bbff3baae 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java @@ -31,6 +31,7 @@ import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_REQUEST; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_TAG; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_URI; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.newError; import static org.apache.hadoop.ozone.s3.util.S3Consts.AWS_TAG_PREFIX; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; @@ -617,6 +618,23 @@ protected static boolean isAccessDenied(OMException ex) { || result == ResultCodes.INVALID_TOKEN; } + /** + * Reject object keys that cannot be represented in a valid URI. AWS S3 returns + * InvalidURI for unreadable keys (non-printable ASCII 128-255) before lookup. + */ + protected void validateObjectKeyUri(String keyPath) throws OS3Exception { + if (keyPath == null || keyPath.indexOf('\uFFFD') >= 0) { + throw newError(INVALID_URI, keyPath); + } + + for (int i = 0; i < keyPath.length(); i++) { + char c = keyPath.charAt(i); + if (c >= 0x80 && c <= 0xFF) { + throw newError(INVALID_URI, keyPath); + } + } + } + protected ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket) throws OS3Exception { String storageType = getHeaders().getHeaderString(STORAGE_CLASS_HEADER); String storageConfig = getHeaders().getHeaderString(CUSTOM_METADATA_HEADER_PREFIX + STORAGE_CONFIG_HEADER); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index e1687672eaf..ed9602d0fd0 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -364,6 +364,7 @@ public Response get( ) throws IOException, OS3Exception { ObjectRequestContext context = new ObjectRequestContext(S3GAction.GET_KEY, bucketName); try { + validateObjectKeyUri(keyPath); return handler.handleGetRequest(context, keyPath); } catch (OMException ex) { throw newError(bucketName, keyPath, ex); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java index e8a8473a815..4aecf70a32a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectGet.java @@ -17,12 +17,15 @@ package org.apache.hadoop.ozone.s3.endpoint; +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertSucceeds; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.get; import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.put; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT; +import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_URI; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_KEY; import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.PRECOND_FAILED; import static org.apache.hadoop.ozone.s3.util.S3Consts.CUSTOM_METADATA_HEADER_PREFIX; @@ -118,6 +121,15 @@ public void testGetWithNegativePartNumber() throws Exception { () -> get(rest, BUCKET_NAME, KEY_NAME)); } + @Test + public void testGetUnreadableKey() { + String unreadableKey = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, ISO_8859_1); + assertErrorResponse(INVALID_URI, () -> get(rest, BUCKET_NAME, unreadableKey)); + + String malformedUtf8Key = new String(new byte[] {(byte) 0xae, (byte) 0x8a, '-'}, UTF_8); + assertErrorResponse(INVALID_URI, () -> get(rest, BUCKET_NAME, malformedUtf8Key)); + } + @Test public void testGet() throws IOException, OS3Exception { //WHEN