diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java index b4cc482500b56..60e7d67f088f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolServerSideTranslatorPB.java @@ -43,6 +43,8 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.LeaveSafeModeResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshMountTableEntriesRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshMountTableEntriesResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto; @@ -73,6 +75,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; @@ -100,6 +104,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntriesRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntriesResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl; @@ -168,6 +174,24 @@ public RemoveMountTableEntryResponseProto removeMountTableEntry( } } + @Override + public RemoveMountTableEntriesResponseProto removeMountTableEntries( + RpcController controller, + RemoveMountTableEntriesRequestProto request) + throws ServiceException { + try { + RemoveMountTableEntriesRequest req = + new RemoveMountTableEntriesRequestPBImpl(request); + RemoveMountTableEntriesResponse response = + server.removeMountTableEntries(req); + RemoveMountTableEntriesResponsePBImpl responsePB = + (RemoveMountTableEntriesResponsePBImpl)response; + return responsePB.getProto(); + } catch (IOException e) { + throw new ServiceException(e); + } + } + @Override public AddMountTableEntriesResponseProto addMountTableEntries(RpcController controller, AddMountTableEntriesRequestProto request) throws ServiceException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java index 0291b511b7d38..6095232804883 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/protocolPB/RouterAdminProtocolTranslatorPB.java @@ -46,6 +46,8 @@ import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshMountTableEntriesResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RefreshSuperUserGroupsConfigurationResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryRequestProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntryResponseProto; import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.UpdateMountTableEntryRequestProto; @@ -76,6 +78,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; @@ -99,6 +103,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshMountTableEntriesResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RefreshSuperUserGroupsConfigurationResponsePBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntriesRequestPBImpl; +import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntriesResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryRequestPBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.RemoveMountTableEntryResponsePBImpl; import org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb.UpdateMountTableEntryRequestPBImpl; @@ -207,6 +213,22 @@ public RemoveMountTableEntryResponse removeMountTableEntry( } } + @Override + public RemoveMountTableEntriesResponse removeMountTableEntries( + RemoveMountTableEntriesRequest request) throws IOException { + RemoveMountTableEntriesRequestPBImpl requestPB = + (RemoveMountTableEntriesRequestPBImpl)request; + RemoveMountTableEntriesRequestProto proto = requestPB.getProto(); + try { + RemoveMountTableEntriesResponseProto responseProto = + rpcProxy.removeMountTableEntries(null, proto); + return new RemoveMountTableEntriesResponsePBImpl(responseProto); + } catch (ServiceException e) { + + throw new IOException(getRemoteException(e).getMessage()); + } + } + @Override public GetMountTableEntriesResponse getMountTableEntries( GetMountTableEntriesRequest request) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java index f49d0407c174b..fa7dd36442bf8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/resolver/MountTableManager.java @@ -29,6 +29,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; @@ -76,12 +78,22 @@ UpdateMountTableEntryResponse updateMountTableEntry( * Remove an entry from the mount table. * * @param request Fully populated request object. - * @return True the mount table entry was removed from the data store. + * @return True if the mount table entry was removed from the data store. * @throws IOException Throws exception if the data store is not initialized. */ RemoveMountTableEntryResponse removeMountTableEntry( RemoveMountTableEntryRequest request) throws IOException; + /** + * Remove multiple entries from the mount table. + * + * @param request Fully populated request object. + * @return True if all paths were removed from the data store. + * @throws IOException if the data store is not initialized. + */ + RemoveMountTableEntriesResponse removeMountTableEntries(RemoveMountTableEntriesRequest request) + throws IOException; + /** * List all mount table entries present at or below the path. Fetches from the * state store. diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java index d9cacf2e75eba..c1c3f465c7b1b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterAdminServer.java @@ -34,6 +34,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.AddMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; import org.apache.hadoop.util.Preconditions; import org.apache.hadoop.conf.Configuration; @@ -515,6 +517,22 @@ public RemoveMountTableEntryResponse removeMountTableEntry( return getMountTableStore().removeMountTableEntry(request); } + @Override + public RemoveMountTableEntriesResponse removeMountTableEntries( + RemoveMountTableEntriesRequest request) throws IOException { + // clear sub-cluster's quota definition + for (String path : request.getSrcPaths()) { + try { + synchronizeQuota(path, HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_RESET, null); + } catch (Exception e) { + // Ignore exception, if any while reseting quota. Specifically to handle + // if the actual destination doesn't exist. + LOG.warn("Unable to clear quota at the destinations for {}: {}", path, e.getMessage()); + } + } + return getMountTableStore().removeMountTableEntries(request); + } + @Override public GetMountTableEntriesResponse getMountTableEntries( GetMountTableEntriesRequest request) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java index 3ace406e49a3e..f2cfde7612a0f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/impl/MountTableStoreImpl.java @@ -20,9 +20,11 @@ import static org.apache.hadoop.hdfs.DFSUtil.isParentEntry; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -44,6 +46,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; @@ -53,6 +57,7 @@ import org.apache.hadoop.security.AccessControlException; import org.apache.hadoop.util.Time; import static org.apache.hadoop.hdfs.server.federation.router.Quota.eachByStorageType; +import static org.apache.hadoop.hdfs.server.federation.store.StateStoreUtils.filterMultiple; /** * Implementation of the {@link MountTableStore} state store API. @@ -209,6 +214,56 @@ public RemoveMountTableEntryResponse removeMountTableEntry( return response; } + @Override + public RemoveMountTableEntriesResponse removeMountTableEntries( + RemoveMountTableEntriesRequest request) throws IOException { + List failedPaths = new ArrayList<>(); + List entriesToRemove = new ArrayList<>(); + List allEntries = getDriver().get(getRecordClass()).getRecords(); + for (String path : request.getSrcPaths()) { + final MountTable partial = MountTable.newInstance(); + partial.setSourcePath(path); + final Query query = new Query<>(partial); + List filtered = filterMultiple(query, allEntries); + MountTable deleteEntry = null; + if (filtered.size() == 1) { + deleteEntry = filtered.get(0); + } + + if (deleteEntry != null) { + RouterPermissionChecker pc = RouterAdminServer.getPermissionChecker(); + if (pc != null) { + try { + pc.checkPermission(deleteEntry, FsAction.WRITE); + entriesToRemove.add(deleteEntry); + } catch (IOException ioe) { + failedPaths.add(path); + } + } + } else { + failedPaths.add(path); + } + } + + boolean anyRemoved = false; + Map statuses = getDriver().removeMultiple(entriesToRemove); + for (Map.Entry mapEntry : statuses.entrySet()) { + if (!mapEntry.getValue()) { + failedPaths.add(mapEntry.getKey().getSourcePath()); + } else { + anyRemoved = true; + } + } + + RemoveMountTableEntriesResponse response = RemoveMountTableEntriesResponse.newInstance(); + response.setStatus(failedPaths.isEmpty()); + response.setFailedRecordsKeys(failedPaths); + if (anyRemoved) { + updateCacheAllRouters(); + } + return response; + } + @Override public GetMountTableEntriesResponse getMountTableEntries( GetMountTableEntriesRequest request) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntriesRequest.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntriesRequest.java new file mode 100644 index 0000000000000..5e31ff8834409 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntriesRequest.java @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer; + +/** + * API request for removing multiple mount table paths in the state store. + */ +public abstract class RemoveMountTableEntriesRequest { + + public static RemoveMountTableEntriesRequest newInstance() throws IOException { + return StateStoreSerializer.newRecord(RemoveMountTableEntriesRequest.class); + } + + public static RemoveMountTableEntriesRequest newInstance(List paths) throws IOException { + RemoveMountTableEntriesRequest request = newInstance(); + request.setSrcPaths(paths); + return request; + } + + @Public + @Unstable + public abstract List getSrcPaths(); + + @Public + @Unstable + public abstract void setSrcPaths(List paths); +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntriesResponse.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntriesResponse.java new file mode 100644 index 0000000000000..1d3d79d36d447 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/RemoveMountTableEntriesResponse.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.classification.InterfaceAudience.Public; +import org.apache.hadoop.classification.InterfaceStability.Unstable; +import org.apache.hadoop.hdfs.server.federation.store.driver.StateStoreSerializer; + +/** + * API response for removing multiple mount table paths in the state store. + */ +public abstract class RemoveMountTableEntriesResponse { + + public static RemoveMountTableEntriesResponse newInstance() throws IOException { + return StateStoreSerializer.newRecord(RemoveMountTableEntriesResponse.class); + } + + @Public + @Unstable + public abstract boolean getStatus(); + + @Public + @Unstable + public abstract List getFailedRecordsKeys(); + + @Public + @Unstable + public abstract void setStatus(boolean result); + + @Public + @Unstable + public abstract void setFailedRecordsKeys(List failedRecordsKeys); +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntriesRequestPBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntriesRequestPBImpl.java new file mode 100644 index 0000000000000..07f742feeb102 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntriesRequestPBImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.thirdparty.protobuf.Message; + +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesRequestProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesRequestProtoOrBuilder; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord; + +/** + * Protobuf implementation of the state store API object + * RemoveMountTableEntriesRequest. + */ +public class RemoveMountTableEntriesRequestPBImpl extends RemoveMountTableEntriesRequest + implements PBRecord { + + private FederationProtocolPBTranslator< + RemoveMountTableEntriesRequestProto, + RemoveMountTableEntriesRequestProto.Builder, + RemoveMountTableEntriesRequestProtoOrBuilder> + translator = new FederationProtocolPBTranslator<>(RemoveMountTableEntriesRequestProto.class); + + public RemoveMountTableEntriesRequestPBImpl() { + } + + public RemoveMountTableEntriesRequestPBImpl(RemoveMountTableEntriesRequestProto proto) { + this.setProto(proto); + } + + @Override + public RemoveMountTableEntriesRequestProto getProto() { + return this.translator.build(); + } + + @Override + public void setProto(Message proto) { + this.translator.setProto(proto); + } + + @Override + public void readInstance(String base64String) throws IOException { + this.translator.readInstance(base64String); + } + + @Override + public List getSrcPaths() { + return this.translator.getProtoOrBuilder().getSrcPathsList(); + } + + @Override + public void setSrcPaths(List paths) { + this.translator.getBuilder().addAllSrcPaths(paths); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntriesResponsePBImpl.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntriesResponsePBImpl.java new file mode 100644 index 0000000000000..9666042030145 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/store/protocol/impl/pb/RemoveMountTableEntriesResponsePBImpl.java @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.federation.store.protocol.impl.pb; + +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.thirdparty.protobuf.Message; + +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesResponseProto; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesResponseProto.Builder; +import org.apache.hadoop.hdfs.federation.protocol.proto.HdfsServerFederationProtos.RemoveMountTableEntriesResponseProtoOrBuilder; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.records.impl.pb.PBRecord; + +/** + * Protobuf implementation of the state store API object + * RemoveMountTableEntriesResponse. + */ +public class RemoveMountTableEntriesResponsePBImpl extends RemoveMountTableEntriesResponse + implements PBRecord { + + private FederationProtocolPBTranslator translator + = new FederationProtocolPBTranslator<>(RemoveMountTableEntriesResponseProto.class); + + public RemoveMountTableEntriesResponsePBImpl() { + } + + public RemoveMountTableEntriesResponsePBImpl(RemoveMountTableEntriesResponseProto proto) { + this.setProto(proto); + } + + @Override + public RemoveMountTableEntriesResponseProto getProto() { + return this.translator.build(); + } + + @Override + public void setProto(Message proto) { + this.translator.setProto(proto); + } + + @Override + public void readInstance(String base64String) throws IOException { + this.translator.readInstance(base64String); + } + + @Override + public boolean getStatus() { + return this.translator.getProtoOrBuilder().getStatus(); + } + + @Override + public List getFailedRecordsKeys() { + return this.translator.getProtoOrBuilder().getFailedEntriesKeysList(); + } + + @Override + public void setStatus(boolean result) { + this.translator.getBuilder().setStatus(result); + } + + @Override + public void setFailedRecordsKeys(List failedRecordsKeys) { + this.translator.getBuilder().addAllFailedEntriesKeys(failedRecordsKeys); + } +} \ No newline at end of file diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index c4c43326fa915..ea319228106c9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -77,6 +77,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.LeaveSafeModeResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RefreshMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; @@ -383,17 +385,12 @@ public int run(String[] argv) throws Exception { exitCode = -1; } } else if ("-rm".equals(cmd)) { - while (i < argv.length) { - try { - if (removeMount(argv[i])) { - System.out.println("Successfully removed mount point " + argv[i]); - } - } catch (IOException e) { - exitCode = -1; - System.err - .println(cmd.substring(1) + ": " + e.getLocalizedMessage()); - } - i++; + try { + removeMounts(argv, i); + } catch (IOException e) { + exitCode = -1; + System.err + .println(cmd.substring(1) + ": " + e.getLocalizedMessage()); } } else if ("-ls".equals(cmd)) { listMounts(argv, i); @@ -862,24 +859,44 @@ private MountTable getMountEntry(String mount, MountTableManager mountTable) } /** - * Remove mount point. + * Remove one or multiple mount points. * - * @param path Path to remove. - * @return If the mount point was removed successfully. + * @param params parameters, should contain paths to remove + * @param paramIdx starting param index * @throws IOException If it cannot be removed. */ - public boolean removeMount(String path) throws IOException { - path = normalizeFileSystemPath(path); + public void removeMounts(String[] params, int paramIdx) throws IOException { + List pathsToRemove = new ArrayList<>(); + while (paramIdx < params.length) { + pathsToRemove.add(normalizeFileSystemPath(params[paramIdx])); + paramIdx++; + } MountTableManager mountTable = client.getMountTableManager(); - RemoveMountTableEntryRequest request = - RemoveMountTableEntryRequest.newInstance(path); - RemoveMountTableEntryResponse response = - mountTable.removeMountTableEntry(request); - boolean removed = response.getStatus(); - if (!removed) { - System.out.println("Cannot remove mount point " + path); + if (pathsToRemove.isEmpty()) { + return; + } + if (pathsToRemove.size() == 1) { + String path = pathsToRemove.get(0); + RemoveMountTableEntryRequest request = RemoveMountTableEntryRequest.newInstance(path); + RemoveMountTableEntryResponse response = mountTable.removeMountTableEntry(request); + boolean removed = response.getStatus(); + if (!removed) { + System.out.println("Cannot remove mount point " + path); + } else { + System.out.println("Successfully removed mount point " + path); + } + return; + } + RemoveMountTableEntriesRequest request = + RemoveMountTableEntriesRequest.newInstance(pathsToRemove); + RemoveMountTableEntriesResponse response = mountTable.removeMountTableEntries(request); + for (String path : pathsToRemove) { + if (response.getFailedRecordsKeys().contains(path)) { + System.out.println("Cannot remove mount point " + path); + } else { + System.out.println("Successfully removed mount point " + path); + } } - return removed; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto index ee263c301ecd9..259722566cc82 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/FederationProtocol.proto @@ -189,6 +189,15 @@ message RemoveMountTableEntryResponseProto{ optional bool status = 1; } +message RemoveMountTableEntriesRequestProto { + repeated string srcPaths = 1; +} + +message RemoveMountTableEntriesResponseProto{ + optional bool status = 1; + repeated string failedEntriesKeys = 2; +} + message GetMountTableEntriesRequestProto { optional string srcPath = 1; } diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto index ea25d296b0b49..f596543b74354 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/proto/RouterProtocol.proto @@ -40,6 +40,11 @@ service RouterAdminProtocolService { */ rpc removeMountTableEntry(RemoveMountTableEntryRequestProto) returns(RemoveMountTableEntryResponseProto); + /** + * Remove multiple mount table entries. + */ + rpc removeMountTableEntries(RemoveMountTableEntriesRequestProto) returns(RemoveMountTableEntriesResponseProto); + /** * Add all mount table entries that are present in the request. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java index 8a4b06cc2f276..70ac046a53a49 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdmin.java @@ -22,11 +22,13 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.lang.reflect.Field; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -56,6 +58,8 @@ import org.apache.hadoop.hdfs.server.federation.store.protocol.GetDisabledNameservicesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.GetMountTableEntriesResponse; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesRequest; +import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntriesResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryRequest; import org.apache.hadoop.hdfs.server.federation.store.protocol.RemoveMountTableEntryResponse; import org.apache.hadoop.hdfs.server.federation.store.protocol.UpdateMountTableEntryRequest; @@ -328,6 +332,46 @@ public void testRemoveMountTable() throws IOException { assertEquals(entries2.size(), mockMountTable.size() - 1); } + @Test + public void testRemoveMountTables() throws IOException { + RouterClient client = routerContext.getAdminClient(); + MountTableManager mountTable = client.getMountTableManager(); + + // Add a few mounts + for (int i = 0; i < 5; i++) { + MountTable newEntry = + MountTable.newInstance("/batch-remove/" + i, Collections.singletonMap("ns0", "/testdir"), + Time.now(), Time.now()); + AddMountTableEntryRequest addRequest = AddMountTableEntryRequest.newInstance(newEntry); + AddMountTableEntryResponse addResponse = mountTable.addMountTableEntry(addRequest); + assertTrue(addResponse.getStatus()); + } + + assertEquals(getMountTableEntries(mountTable).size(), mockMountTable.size() + 5); + + // Remove existing entries + RemoveMountTableEntriesRequest removeRequest = RemoveMountTableEntriesRequest.newInstance( + Arrays.asList("/batch-remove/0", "/batch-remove/1")); + RemoveMountTableEntriesResponse res = mountTable.removeMountTableEntries(removeRequest); + assertTrue(res.getStatus()); + assertTrue(res.getFailedRecordsKeys().isEmpty()); + assertEquals(getMountTableEntries(mountTable).size(), mockMountTable.size() + 3); + + // Remove existing entry and non-existent entries + removeRequest = RemoveMountTableEntriesRequest.newInstance( + Arrays.asList("/batch-remove/2", "/batch-remove/20")); + res = mountTable.removeMountTableEntries(removeRequest); + assertFalse(res.getStatus()); + assertEquals(1, res.getFailedRecordsKeys().size()); + assertEquals("/batch-remove/20", res.getFailedRecordsKeys().get(0)); + assertEquals(getMountTableEntries(mountTable).size(), mockMountTable.size() + 2); + + // Remove non-existent entries + RemoveMountTableEntriesRequest removeRequest2 = RemoveMountTableEntriesRequest.newInstance( + Arrays.asList("/batch-remove/404", "/batch-remove/4000")); + assertThrows(IOException.class, () -> mountTable.removeMountTableEntries(removeRequest2)); + } + @Test public void testEditMountTable() throws IOException {