Skip to content

HDDS-15962. Remove leader readiness check on the bootstrap flow. #10862

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
sadanand48 wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

HDDS-15962. Remove leader readiness check on the bootstrap flow. #10862

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions ...p-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServlet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,10 +138,8 @@ public void init() throws ServletException {
public void processMetadataSnapshotRequest(HttpServletRequest request, HttpServletResponse response,
boolean isFormData, boolean flush) {
OzoneManager om = (OzoneManager) getServletContext().getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE);
boolean isOmLeader = om.isLeaderReady();
if (!isOmLeader) {
String msg = "Unable to process metadata snapshot request as "
+ "this OM is not the leader or not ready to serve requests";
if (!om.isLeader()) {
String msg = "Unable to process metadata snapshot request as this OM is not the leader";
LOG.warn(msg);
try {
response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, msg);
Expand Down
6 changes: 2 additions & 4 deletions ...manager/src/main/java/org/apache/hadoop/ozone/om/OMDBCheckpointServletInodeBasedXfer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,10 +144,8 @@ public BootstrapStateHandler.Lock getBootstrapStateLock() {
public void processMetadataSnapshotRequest(HttpServletRequest request, HttpServletResponse response,
boolean isFormData, boolean flush) {
OzoneManager om = (OzoneManager) getServletContext().getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE);
boolean isOmLeader = om.isLeaderReady();
if (!isOmLeader) {
String msg = "Unable to process metadata snapshot request as "
+ "this OM is not the leader or not ready to serve requests";
if (!om.isLeader()) {
String msg = "Unable to process metadata snapshot request as this OM is not the leader";
LOG.warn(msg);
try {
response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE, msg);
Expand Down
10 changes: 10 additions & 0 deletions hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.BUCKET_LOCK;
import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.LeveledResource.VOLUME_LOCK;
import static org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus.LEADER_AND_READY;
import static org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus.NOT_LEADER;
import static org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.getRaftGroupIdFromOmServiceId;
import static org.apache.hadoop.ozone.om.s3.S3SecretStoreConfigurationKeys.DEFAULT_SECRET_STORAGE_TYPE;
import static org.apache.hadoop.ozone.om.s3.S3SecretStoreConfigurationKeys.S3_SECRET_STORAGE_TYPE;
Expand Down Expand Up @@ -4615,6 +4616,15 @@ public boolean isLeaderReady() {
return ratisServer != null && ratisServer.getLeaderStatus() == LEADER_AND_READY;
}

/**
* Return true, if the current OM node is leader.
* Note that it also returns true if the OM is leader but is not ready.
*/
public boolean isLeader() {
final OzoneManagerRatisServer ratisServer = omRatisServer;
return ratisServer != null && ratisServer.getLeaderStatus() != NOT_LEADER;
}

/**
* Checks the leader status. Does nothing if this OM is leader and is ready.
* @throws OMLeaderNotReadyException if leader, but not ready
Expand Down
39 changes: 37 additions & 2 deletions ...est/java/org/apache/hadoop/ozone/om/TestOMDBCheckpointServletInodeBasedXferNonLeader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
Expand All @@ -31,6 +32,9 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import org.junit.jupiter.api.Test;

/**
Expand All @@ -43,7 +47,7 @@ void processMetadataSnapshotRequestReturns503WhenNotLeader() throws Exception {
OMDBCheckpointServletInodeBasedXfer servlet =
spy(new OMDBCheckpointServletInodeBasedXfer());
OzoneManager om = mock(OzoneManager.class);
when(om.isLeaderReady()).thenReturn(false);
when(om.isLeader()).thenReturn(false);

ServletContext ctx = mock(ServletContext.class);
when(ctx.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)).thenReturn(om);
Expand All @@ -62,7 +66,7 @@ void processMetadataSnapshotRequestSetsStatusWhenSendErrorFails() throws Excepti
OMDBCheckpointServletInodeBasedXfer servlet =
spy(new OMDBCheckpointServletInodeBasedXfer());
OzoneManager om = mock(OzoneManager.class);
when(om.isLeaderReady()).thenReturn(false);
when(om.isLeader()).thenReturn(false);

ServletContext ctx = mock(ServletContext.class);
when(ctx.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)).thenReturn(om);
Expand All @@ -77,4 +81,35 @@ void processMetadataSnapshotRequestSetsStatusWhenSendErrorFails() throws Excepti

verify(response).setStatus(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
}

@ParameterizedTest
@ValueSource(booleans = {false, true})
void processMetadataSnapshotRequestDoesNotReturn503WhenLeader(boolean isLeaderReady) throws Exception {
OMDBCheckpointServletInodeBasedXfer servlet =
spy(new OMDBCheckpointServletInodeBasedXfer());
OzoneManager om = mock(OzoneManager.class);
when(om.isLeader()).thenReturn(true);
when(om.isLeaderReady()).thenReturn(isLeaderReady);

ServletContext ctx = mock(ServletContext.class);
when(ctx.getAttribute(OzoneConsts.OM_CONTEXT_ATTRIBUTE)).thenReturn(om);
doReturn(ctx).when(servlet).getServletContext();
BootstrapStateHandler.Lock lock = mock(BootstrapStateHandler.Lock.class);
// Force a failure after leader check so this unit test can stay lightweight
// (no full servlet/bootstrap setup) while still proving that leader requests
// are not rejected with 503.
doThrow(new InterruptedException("test lock failure"))
.when(lock).acquireWriteLock();
doReturn(lock).when(servlet).getBootstrapStateLock();

HttpServletRequest request = mock(HttpServletRequest.class);
HttpServletResponse response = mock(HttpServletResponse.class);

servlet.processMetadataSnapshotRequest(request, response, false, true);

verify(response, never())
.sendError(eq(HttpServletResponse.SC_SERVICE_UNAVAILABLE), anyString());
// Internal error comes from the forced lock failure above.
verify(response).setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
}
}