diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java index 8da2be2755a3..463b9de0d95f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java @@ -396,6 +396,31 @@ public Builder toBuilder() { .setTags(tags); } + /** + * Returns a copy of this bucket with operational properties taken from + * {@code source}. Link identity fields (volume, name, owner, source path, + * ACLs, timestamps, object/update IDs) are unchanged. + * + *

When adding new operational bucket fields, update this method if they + * should be resolved from a link's source bucket. + */ + public OmBucketInfo withOperationalPropertiesFrom(OmBucketInfo source) { + return toBuilder() + .setDefaultReplicationConfig(source.getDefaultReplicationConfig()) + .setIsVersionEnabled(source.getIsVersionEnabled()) + .setStorageType(source.getStorageType()) + .setQuotaInBytes(source.getQuotaInBytes()) + .setQuotaInNamespace(source.getQuotaInNamespace()) + .setUsedBytes(source.getUsedBytes()) + .setUsedNamespace(source.getUsedNamespace()) + .setSnapshotUsedBytes(source.getSnapshotUsedBytes()) + .setSnapshotUsedNamespace(source.getSnapshotUsedNamespace()) + .addAllMetadata(source.getMetadata()) + .setBucketLayout(source.getBucketLayout()) + .setTags(source.getTags()) + .build(); + } + /** * Builder for OmBucketInfo. */ diff --git a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java index 857103a20c0d..5f816ff4c20a 100644 --- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java +++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmBucketInfo.java @@ -97,6 +97,56 @@ public void testClone() { cloneBucketInfo.getAcls().get(0)); } + @Test + public void testWithOperationalPropertiesFromPreservesLinkIdentity() { + ECReplicationConfig sourceReplication = new ECReplicationConfig(3, 2); + OmBucketInfo source = OmBucketInfo.newBuilder() + .setVolumeName("vol1") + .setBucketName("source") + .setBucketLayout(BucketLayout.OBJECT_STORE) + .setStorageType(StorageType.SSD) + .setIsVersionEnabled(true) + .setQuotaInBytes(1000) + .setQuotaInNamespace(10) + .setUsedBytes(500) + .setUsedNamespace(5) + .setDefaultReplicationConfig(new DefaultReplicationConfig(sourceReplication)) + .addAllMetadata(Collections.singletonMap("sourceKey", "sourceValue")) + .build(); + + OmBucketInfo link = OmBucketInfo.newBuilder() + .setVolumeName("vol1") + .setBucketName("link") + .setSourceVolume("vol1") + .setSourceBucket("source") + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .setCreationTime(123L) + .setModificationTime(456L) + .addAllMetadata(Collections.singletonMap("linkKey", "linkValue")) + .build(); + + OmBucketInfo resolvedLink = link.withOperationalPropertiesFrom(source); + + assertEquals("link", resolvedLink.getBucketName()); + assertEquals("vol1", resolvedLink.getVolumeName()); + assertEquals("vol1", resolvedLink.getSourceVolume()); + assertEquals("source", resolvedLink.getSourceBucket()); + assertEquals(123L, resolvedLink.getCreationTime()); + assertEquals(456L, resolvedLink.getModificationTime()); + + assertEquals(BucketLayout.OBJECT_STORE, resolvedLink.getBucketLayout()); + assertEquals(StorageType.SSD, resolvedLink.getStorageType()); + assertTrue(resolvedLink.getIsVersionEnabled()); + assertEquals(1000, resolvedLink.getQuotaInBytes()); + assertEquals(10, resolvedLink.getQuotaInNamespace()); + assertEquals(500, resolvedLink.getUsedBytes()); + assertEquals(5, resolvedLink.getUsedNamespace()); + assertEquals(sourceReplication, + resolvedLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals("sourceValue", resolvedLink.getMetadata().get("sourceKey")); + assertEquals("linkValue", resolvedLink.getMetadata().get("linkKey")); + } + @Test public void getProtobufMessageEC() { OmBucketInfo omBucketInfo = diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 1910d92e9690..642d6491baba 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2993,8 +2993,18 @@ public List listBuckets(String volumeName, String startKey, volumeName, null, null); } metrics.incNumBucketLists(); - return bucketManager.listBuckets(volumeName, + List buckets = bucketManager.listBuckets(volumeName, startKey, prefix, maxNumOfBuckets, hasSnapshot); + Map, OmBucketInfo> resolvedSourceCache = new HashMap<>(); + for (int i = 0; i < buckets.size(); i++) { + try { + buckets.set(i, enrichLinkBucketInfo(buckets.get(i), resolvedSourceCache)); + } catch (IOException e) { + LOG.debug("Failed to enrich listBuckets entry for {}/{}; returning raw entry", + volumeName, buckets.get(i).getBucketName(), e); + } + } + return buckets; } catch (IOException ex) { metrics.incNumBucketListFails(); auditSuccess = false; @@ -3009,6 +3019,62 @@ public List listBuckets(String volumeName, String startKey, } } + /** + * For link buckets, follows the link chain and overlays the source bucket's + * operational properties onto the link's {@link OmBucketInfo}. Non-link + * buckets and dangling links are returned unchanged. + */ + private OmBucketInfo enrichLinkBucketInfo(OmBucketInfo bucketInfo) + throws IOException { + return enrichLinkBucketInfo(bucketInfo, null); + } + + private OmBucketInfo enrichLinkBucketInfo( + OmBucketInfo bucketInfo, + Map, OmBucketInfo> resolvedSourceCache) + throws IOException { + if (!bucketInfo.isLink()) { + return bucketInfo; + } + // We already know that `bucketInfo` is a linked one, + // so we skip one `getBucketInfo` and start with the known link. + ResolvedBucket resolvedBucket = + resolveBucketLink(Pair.of( + bucketInfo.getSourceVolume(), + bucketInfo.getSourceBucket()), + true); + + // If it is a dangling link it means no real bucket exists, + // for example, it could have been deleted, but the links still present. + if (resolvedBucket.isDangling()) { + return bucketInfo; + } + OmBucketInfo realBucket = getResolvedSourceBucket(resolvedBucket, resolvedSourceCache); + return bucketInfo.withOperationalPropertiesFrom(realBucket); + } + + private OmBucketInfo getResolvedSourceBucket( + ResolvedBucket resolvedBucket, + Map, OmBucketInfo> resolvedSourceCache) + throws IOException { + Pair sourceKey = Pair.of( + resolvedBucket.realVolume(), + resolvedBucket.realBucket()); + if (resolvedSourceCache != null) { + OmBucketInfo cachedSource = resolvedSourceCache.get(sourceKey); + if (cachedSource != null) { + return cachedSource; + } + } + OmBucketInfo realBucket = bucketManager.getBucketInfo( + resolvedBucket.realVolume(), + resolvedBucket.realBucket()); + if (resolvedSourceCache != null) { + resolvedSourceCache.put(sourceKey, realBucket); + } + return realBucket; + } + /** * Gets the bucket information. * @@ -3030,46 +3096,8 @@ public OmBucketInfo getBucketInfo(String volume, String bucket) } metrics.incNumBucketInfos(); - OmBucketInfo bucketInfo = bucketManager.getBucketInfo(volume, bucket); - - // No links - return the bucket info right away. - if (!bucketInfo.isLink()) { - return bucketInfo; - } - // Otherwise follow the links to find the real bucket. - // We already know that `bucketInfo` is a linked one, - // so we skip one `getBucketInfo` and start with the known link. - ResolvedBucket resolvedBucket = - resolveBucketLink(Pair.of( - bucketInfo.getSourceVolume(), - bucketInfo.getSourceBucket()), - true); - - // If it is a dangling link it means no real bucket exists, - // for example, it could have been deleted, but the links still present. - if (!resolvedBucket.isDangling()) { - OmBucketInfo realBucket = - bucketManager.getBucketInfo( - resolvedBucket.realVolume(), - resolvedBucket.realBucket()); - // Pass the real bucket metadata in the link bucket info. - return bucketInfo.toBuilder() - .setDefaultReplicationConfig( - realBucket.getDefaultReplicationConfig()) - .setIsVersionEnabled(realBucket.getIsVersionEnabled()) - .setStorageType(realBucket.getStorageType()) - .setQuotaInBytes(realBucket.getQuotaInBytes()) - .setQuotaInNamespace(realBucket.getQuotaInNamespace()) - .setUsedBytes(realBucket.getUsedBytes()) - .setSnapshotUsedBytes(realBucket.getSnapshotUsedBytes()) - .setSnapshotUsedNamespace(realBucket.getSnapshotUsedNamespace()) - .setUsedNamespace(realBucket.getUsedNamespace()) - .addAllMetadata(realBucket.getMetadata()) - .setBucketLayout(realBucket.getBucketLayout()) - .build(); - } - // If no real bucket exists, return the requested one's info. - return bucketInfo; + return enrichLinkBucketInfo( + bucketManager.getBucketInfo(volume, bucket)); } catch (Exception ex) { metrics.incNumBucketInfoFails(); auditSuccess = false; diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java index 42d748607ac4..1755fb433c42 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestBucketManagerImpl.java @@ -31,10 +31,12 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.List; import org.apache.hadoop.crypto.key.KeyProvider; import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension; import org.apache.hadoop.hdds.client.DefaultReplicationConfig; import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; @@ -460,4 +462,82 @@ public void testLinkedBucketResolution() throws Exception { bucketInfo.getIsVersionEnabled(), storedLinkBucket.getIsVersionEnabled()); } + + @Test + public void testListBucketsResolvesFsoAndObsLinkLayouts() throws Exception { + String volume = volumeName(); + createSampleVol(volume); + + ECReplicationConfig fsoReplication = new ECReplicationConfig(3, 2); + OmBucketInfo fsoSource = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("fso-source") + .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED) + .setDefaultReplicationConfig(new DefaultReplicationConfig(fsoReplication)) + .build(); + writeClient.createBucket(fsoSource); + + RatisReplicationConfig obsReplication = + RatisReplicationConfig.getInstance(ReplicationFactor.THREE); + OmBucketInfo obsSource = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("obs-source") + .setBucketLayout(BucketLayout.OBJECT_STORE) + .setDefaultReplicationConfig(new DefaultReplicationConfig(obsReplication)) + .build(); + writeClient.createBucket(obsSource); + + OmBucketInfo fsoLink = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("link-fso") + .setSourceVolume(volume) + .setSourceBucket("fso-source") + .build(); + writeClient.createBucket(fsoLink); + + OmBucketInfo obsLink = OmBucketInfo.newBuilder() + .setVolumeName(volume) + .setBucketName("link-obs") + .setSourceVolume(volume) + .setSourceBucket("obs-source") + .build(); + writeClient.createBucket(obsLink); + + OmBucketInfo fsoLinkInfo = writeClient.getBucketInfo(volume, "link-fso"); + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, fsoLinkInfo.getBucketLayout()); + assertEquals(fsoReplication, + fsoLinkInfo.getDefaultReplicationConfig().getReplicationConfig()); + + OmBucketInfo obsLinkInfo = writeClient.getBucketInfo(volume, "link-obs"); + assertEquals(BucketLayout.OBJECT_STORE, obsLinkInfo.getBucketLayout()); + assertEquals(obsReplication, + obsLinkInfo.getDefaultReplicationConfig().getReplicationConfig()); + + List listedBuckets = + writeClient.listBuckets(volume, "", "", 100, false); + + OmBucketInfo listedFsoLink = null; + OmBucketInfo listedObsLink = null; + for (OmBucketInfo listedBucket : listedBuckets) { + if ("link-fso".equals(listedBucket.getBucketName())) { + listedFsoLink = listedBucket; + } else if ("link-obs".equals(listedBucket.getBucketName())) { + listedObsLink = listedBucket; + } + } + assertNotNull(listedFsoLink, "link-fso not found in listBuckets response"); + assertNotNull(listedObsLink, "link-obs not found in listBuckets response"); + + assertEquals(BucketLayout.FILE_SYSTEM_OPTIMIZED, listedFsoLink.getBucketLayout()); + assertEquals(fsoReplication, + listedFsoLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals(volume, listedFsoLink.getSourceVolume()); + assertEquals("fso-source", listedFsoLink.getSourceBucket()); + + assertEquals(BucketLayout.OBJECT_STORE, listedObsLink.getBucketLayout()); + assertEquals(obsReplication, + listedObsLink.getDefaultReplicationConfig().getReplicationConfig()); + assertEquals(volume, listedObsLink.getSourceVolume()); + assertEquals("obs-source", listedObsLink.getSourceBucket()); + } }