From a3137800e76d1e960d55366aadfe446f2730fa7e Mon Sep 17 00:00:00 2001 From: Ruizhe Zhou Date: Tue, 9 Dec 2025 14:29:45 +0800 Subject: [PATCH] perftest: Fix handshake deadlock by adding timeouts Currently, if one side (e.g., the server) fails during the handshake phase inside `ctx_hand_shake` (specifically in `rdma_write_keys`) and exits, the peer (client) remains blocked indefinitely. This deadlock occurs because `rdma_read_keys` function uses an infinite loop to poll the CQ without a timeout. This patch prevents the deadlock by: adding a timeout mechanism to the `ibv_poll_cq` loop in `rdma_read_keys`. This ensures that the waiting process terminates with an error if the peer fails to respond within a reasonable timeframe (default 60s). Signed-off-by: Ruizhe Zhou --- src/perftest_communication.c | 15 +++++++++++++-- src/perftest_communication.h | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/perftest_communication.c b/src/perftest_communication.c index 27439c2a..9b45a7b0 100755 --- a/src/perftest_communication.c +++ b/src/perftest_communication.c @@ -553,13 +553,24 @@ static int rdma_read_keys(struct pingpong_dest *rem_dest, struct ibv_wc wc; int ne; + unsigned long timeout_counter = 0; + do { - ne = ibv_poll_cq(comm->rdma_ctx->recv_cq,1,&wc); + ne = ibv_poll_cq(comm->rdma_ctx->recv_cq, 1, &wc); + if (ne == 0) { + timeout_counter++; + usleep(2000); /* sleep 2ms before polling cq again */ + if (timeout_counter > MAX_TIMEOUT_ITER) { + fprintf(stderr, + "Error: Timeout waiting for remote data.\n"); + return 1; + } + } } while (ne == 0); if (wc.status || !(wc.opcode & IBV_WC_RECV) || wc.wr_id != SYNC_SPEC_ID) { //coverity[uninit_use_in_call] - fprintf(stderr, "Bad wc status -- %d -- %d \n",(int)wc.status,(int)wc.wr_id); + fprintf(stderr, "Bad wc status -- %d -- %d\n", (int)wc.status, (int)wc.wr_id); return 1; } diff --git a/src/perftest_communication.h b/src/perftest_communication.h index 985464ec..947ffb21 100755 --- a/src/perftest_communication.h +++ b/src/perftest_communication.h @@ -83,6 +83,8 @@ #define SYNC_SPEC_ID (5) +#define MAX_TIMEOUT_ITER (30000) /* Max iterations to wait for client to poll cq during handshake. */ + #define KEY_PRINT_FMT "%04x:%04x:%06x:%06x:%08x:%016llx:%08x" #define KEY_PRINT_FMT_DV "%04x:%04x:%06x:%06x:%08x:%016llx:%016llx:%08x"