Skip to content

HDDS-15918. Optimize TestSCMNodeManager execution time and stability by reducing heartbeat configs #10837

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rich7420 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

HDDS-15918. Optimize TestSCMNodeManager execution time and stability by reducing heartbeat configs #10837

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 72 additions & 75 deletions hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -694,16 +694,17 @@ public void testSetNodeOpStateAndCommandFired()
*/
@Test
public void testScmDetectStaleAndDeadNode()
throws IOException, InterruptedException, AuthenticationException {
throws IOException, InterruptedException, AuthenticationException,
TimeoutException {
final int interval = 100;
final int nodeCount = 10;

OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, interval,
MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, interval, MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 2, SECONDS);


try (SCMNodeManager nodeManager = createNodeManager(conf)) {
Expand All @@ -713,28 +714,20 @@ public void testScmDetectStaleAndDeadNode()
DatanodeDetails staleNode = HddsTestUtils.createRandomDatanodeAndRegister(
nodeManager);

// Heartbeat once
// Heartbeat the stale node once; it ages into stale then dead while the
// other nodes are kept alive.
nodeManager.processHeartbeat(staleNode);

// Heartbeat all other nodes.
for (DatanodeDetails dn : nodeList) {
nodeManager.processHeartbeat(dn);
}

// Wait for 2 seconds .. and heartbeat good nodes again.
Thread.sleep(2 * 1000);

for (DatanodeDetails dn : nodeList) {
nodeManager.processHeartbeat(dn);
}
// Keep the good nodes alive until the stale node moves into stale state.
GenericTestUtils.waitFor(() -> {
for (DatanodeDetails dn : nodeList) {
nodeManager.processHeartbeat(dn);
}
return nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1;
}, interval, 10 * 1000);

// Wait for 2 seconds, wait a total of 4 seconds to make sure that the
// node moves into stale state.
Thread.sleep(2 * 1000);
List<DatanodeDetails> staleNodeList =
nodeManager.getNodes(NodeStatus.inServiceStale());
assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceStale()),
"Expected to find 1 stale node");
assertEquals(1, staleNodeList.size(),
"Expected to find 1 stale node");
assertEquals(staleNode.getID(), staleNodeList.get(0).getID(),
Expand All @@ -745,23 +738,21 @@ public void testScmDetectStaleAndDeadNode()
nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name())
.get(HddsProtos.NodeState.STALE.name()).intValue());

Thread.sleep(1000);
// heartbeat good nodes again.
for (DatanodeDetails dn : nodeList) {
nodeManager.processHeartbeat(dn);
}

// 6 seconds is the dead window for this test , so we wait a total of
// 7 seconds to make sure that the node moves into dead state.
Thread.sleep(2 * 1000);
// Keep the good nodes alive until the stale node moves into dead state.
GenericTestUtils.waitFor(() -> {
for (DatanodeDetails dn : nodeList) {
nodeManager.processHeartbeat(dn);
}
return nodeManager.getNodeCount(NodeStatus.inServiceDead()) == 1;
}, interval, 10 * 1000);

// the stale node has been removed
// the stale node has moved on to dead
staleNodeList = nodeManager.getNodes(NodeStatus.inServiceStale());
nodeCounts = nodeManager.getNodeCount();
assertEquals(0, nodeManager.getNodeCount(NodeStatus.inServiceStale()),
"Expected to find 1 stale node");
"Expected to find 0 stale node");
assertEquals(0, staleNodeList.size(),
"Expected to find 1 stale node");
"Expected to find 0 stale node");
assertEquals(0,
nodeCounts.get(HddsProtos.NodeOperationalState.IN_SERVICE.name())
.get(HddsProtos.NodeState.STALE.name()).intValue());
Expand Down Expand Up @@ -789,21 +780,22 @@ public void testScmDetectStaleAndDeadNode()
*/
@Test
void testScmHandleJvmPause() throws Exception {
final int healthCheckInterval = 200; // milliseconds
final int heartbeatInterval = 1; // seconds
final int staleNodeInterval = 3; // seconds
final int deadNodeInterval = 6; // seconds
final int healthCheckInterval = 100; // milliseconds
final int heartbeatInterval = 100; // milliseconds
final int staleNodeInterval = 1000; // milliseconds
// Keep the dead window large so the stale node does not race into dead.
final int deadNodeInterval = 10000; // milliseconds
ScheduledFuture schedFuture;

OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
healthCheckInterval, MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL,
heartbeatInterval, SECONDS);
heartbeatInterval, MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL,
staleNodeInterval, SECONDS);
staleNodeInterval, MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL,
deadNodeInterval, SECONDS);
deadNodeInterval, MILLISECONDS);

try (SCMNodeManager nodeManager = createNodeManager(conf)) {
DatanodeDetails node1 =
Expand Down Expand Up @@ -835,7 +827,7 @@ void testScmHandleJvmPause() throws Exception {

// Step 1 : stop health check process (simulate JVM pause)
nodeManager.pauseHealthCheck();
Thread.sleep(MILLISECONDS.convert(staleNodeInterval, SECONDS));
Thread.sleep(staleNodeInterval);

// Step 2 : resume health check
assertEquals(0, nodeManager.getSkippedHealthChecks(),
Expand All @@ -856,8 +848,12 @@ void testScmHandleJvmPause() throws Exception {
// Step 5 : heartbeat for node1
nodeManager.processHeartbeat(node1);

// Step 6 : wait for health check process to run
Thread.sleep(1000);
// Step 6 : wait for node2 (not heartbeated) to transition to STALE while
// node1 (just heartbeated) stays HEALTHY.
GenericTestUtils.waitFor(
() -> nodeManager.getNodeCount(NodeStatus.inServiceHealthy()) == 1
&& nodeManager.getNodeCount(NodeStatus.inServiceStale()) == 1,
healthCheckInterval, staleNodeInterval);

// Step 7 : node2 should transition to STALE
assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
Expand Down Expand Up @@ -1159,9 +1155,9 @@ public void testScmClusterIsInExpectedState1()
OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100, MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 2, SECONDS);


/**
Expand All @@ -1186,10 +1182,11 @@ public void testScmClusterIsInExpectedState1()
assertEquals(3, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));

/**
* Cluster state: Quiesced: We are going to sleep for 3 seconds. Which
* means that no node is heartbeating. All nodes should move to Stale.
* Cluster state: Quiesced: We are going to sleep past the 1 second stale
* interval. Which means that no node is heartbeating. All nodes should
* move to Stale.
*/
Thread.sleep(3 * 1000);
Thread.sleep(1000);
assertEquals(3, nodeManager.getAllNodes().size());
assertEquals(3, nodeManager.getNodeCount(NodeStatus.inServiceStale()));

Expand All @@ -1198,21 +1195,21 @@ public void testScmClusterIsInExpectedState1()
* Cluster State : Move healthy node back to healthy state, move other 2
* nodes to Stale State.
*
* We heartbeat healthy node after 1 second and let other 2 nodes elapse
* the 3 second windows.
* We heartbeat healthy node partway through and let other 2 nodes elapse
* the 1 second stale window.
*/

nodeManager.processHeartbeat(healthyNode);
nodeManager.processHeartbeat(staleNode);
nodeManager.processHeartbeat(deadNode);

Thread.sleep(1500);
Thread.sleep(500);
nodeManager.processHeartbeat(healthyNode);
Thread.sleep(2 * 1000);
Thread.sleep(667);
assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));


// 3.5 seconds from last heartbeat for the stale and deadNode. So those
// ~1.17 seconds from last heartbeat for the stale and deadNode. So those
// 2 nodes must move to Stale state and the healthy node must
// remain in the healthy State.
List<DatanodeDetails> healthyList = nodeManager.getNodes(
Expand All @@ -1230,13 +1227,13 @@ public void testScmClusterIsInExpectedState1()

nodeManager.processHeartbeat(healthyNode);
nodeManager.processHeartbeat(staleNode);
Thread.sleep(1500);
Thread.sleep(500);
nodeManager.processHeartbeat(healthyNode);
Thread.sleep(2 * 1000);
Thread.sleep(667);

// 3.5 seconds have elapsed for stale node, so it moves into Stale.
// 7 seconds have elapsed for dead node, so it moves into dead.
// 2 Seconds have elapsed for healthy node, so it stays in healthy state.
// ~1.17 seconds have elapsed for stale node, so it moves into Stale.
// ~2.33 seconds have elapsed for dead node, so it moves into dead.
// ~0.67 seconds have elapsed for healthy node, so it stays healthy.
healthyList = nodeManager.getNodes((NodeStatus.inServiceHealthy()));
List<DatanodeDetails> staleList =
nodeManager.getNodes(NodeStatus.inServiceStale());
Expand Down Expand Up @@ -1339,9 +1336,9 @@ public void testScmClusterIsInExpectedState2()
OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100, MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 2, SECONDS);


try (SCMNodeManager nodeManager = createNodeManager(conf)) {
Expand All @@ -1354,17 +1351,17 @@ public void testScmClusterIsInExpectedState2()

Runnable healthyNodeTask = () -> {
try {
// 2 second heartbeat makes these nodes stay healthy.
heartbeatNodeSet(nodeManager, healthyNodeList, 2 * 1000);
// Sub-stale heartbeat keeps these nodes healthy.
heartbeatNodeSet(nodeManager, healthyNodeList, 600);
} catch (InterruptedException ignored) {
}
};

Runnable staleNodeTask = () -> {
try {
// 4 second heartbeat makes these nodes go to stale and back to
// healthy again.
heartbeatNodeSet(nodeManager, staleNodeList, 4 * 1000);
// Heartbeat between the stale and dead windows makes these nodes go
// to stale and back to healthy again.
heartbeatNodeSet(nodeManager, staleNodeList, 1500);
} catch (InterruptedException ignored) {
}
};
Expand All @@ -1385,7 +1382,7 @@ public void testScmClusterIsInExpectedState2()
thread2.setDaemon(true);
thread2.start();

Thread.sleep(10 * 1000);
Thread.sleep(4 * 1000);

// Assert all healthy nodes are healthy now, this has to be a greater
// than check since Stale nodes can be healthy when we check the state.
Expand Down Expand Up @@ -1433,11 +1430,11 @@ public void testScmCanHandleScale()
OzoneConfiguration conf = getConf();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, 100,
MILLISECONDS);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1,
SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3 * 1000,
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100,
MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6 * 1000,
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 1000,
MILLISECONDS);
conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 2000,
MILLISECONDS);

try (SCMNodeManager nodeManager = createNodeManager(conf)) {
Expand All @@ -1448,15 +1445,15 @@ public void testScmCanHandleScale()

Runnable healthyNodeTask = () -> {
try {
heartbeatNodeSet(nodeManager, healthyList, 2 * 1000);
heartbeatNodeSet(nodeManager, healthyList, 600);
} catch (InterruptedException ignored) {

}
};

Runnable staleNodeTask = () -> {
try {
heartbeatNodeSet(nodeManager, staleList, 4 * 1000);
heartbeatNodeSet(nodeManager, staleList, 1500);
} catch (InterruptedException ignored) {
}
};
Expand All @@ -1468,7 +1465,7 @@ public void testScmCanHandleScale()
Thread thread2 = new Thread(staleNodeTask);
thread2.setDaemon(true);
thread2.start();
Thread.sleep(3 * 1000);
Thread.sleep(1000);

GenericTestUtils.waitFor(() -> findNodes(nodeManager, staleCount, STALE),
500, 20 * 1000);
Expand Down
Loading