Fixes for the following bugs:

- Bug 1 fix (AgentManagerImpl.java) — GlobalLock.isLockAvailable() now only runs when the host status is not alive. This eliminates one IS_FREE_LOCK DB query per ping per healthy host, which is the direct cause of listHosts/listNetworks degradation.
- Bug 2 fix (HostConnectProcess.java) — shutdown() → shutdownNow(). Old thread pools from prior connect cycles are now interrupted immediately instead of draining their queued tasks, preventing thread accumulation during reconnect storms.
- Bug 3 fix (AgentManagerImpl.java) — Lock timeout in handleDisconnectWithoutInvestigation now logs a warn instead of silently discarding the disconnect event.
- Bug 4 fix (ServerAttache.java) — Alarm ScheduledFuture handles are now tracked in _alarmFutures and cancelled when the corresponding listener is unregistered or all commands are cancelled on disconnect.
- Bug 5 fix (AgentAttache.java) - Fix Alarm ScheduledFuture handles in AgentAttache as well

Author: sureshanaparti

agent/src/main/java/com/cloud/agent/HostConnectProcess.java

@@ -94,7 +94,7 @@ public boolean stop() {
 
     private void stopHostStatusExecutor() {
         if (hostStatusExecutor != null) {
-            hostStatusExecutor.shutdown();
+            hostStatusExecutor.shutdownNow();
             hostStatusExecutor = null;
         }
     }

agent/src/main/java/com/cloud/agent/ServerAttache.java

@@ -86,6 +86,7 @@ public class ServerAttache {
     protected String _name;
     private Link _link;
     protected ConcurrentHashMap<Long, ServerListener> _waitForList;
+    protected ConcurrentHashMap<Long, java.util.concurrent.ScheduledFuture<?>> _alarmFutures;
     protected LinkedList<Request> _requests;
     protected Long _currentSequence;
     protected long _nextSequence;
@@ -94,6 +95,7 @@ protected ServerAttache(Link link) {
         _name = link.getIpAddress();
         _link = link;
         _waitForList = new ConcurrentHashMap<>();
+        _alarmFutures = new ConcurrentHashMap<>();
         _requests = new LinkedList<>();
         _nextSequence = Long.valueOf(s_rand.nextInt(Short.MAX_VALUE)) << 48;
     }
@@ -148,7 +150,9 @@ protected void registerListener(long seq, ServerListener listener) {
             logger.trace(log(seq, "Registering listener"));
         }
         if (listener.getTimeout() != -1) {
-            s_listenerExecutor.schedule(new Alarm(seq), listener.getTimeout(), TimeUnit.SECONDS);
+            java.util.concurrent.ScheduledFuture<?> alarmFuture =
+                    s_listenerExecutor.schedule(new Alarm(seq), listener.getTimeout(), TimeUnit.SECONDS);
+            _alarmFutures.put(seq, alarmFuture);
         }
         _waitForList.put(seq, listener);
     }
@@ -157,6 +161,10 @@ protected ServerListener unregisterListener(long sequence) {
         if (logger.isTraceEnabled()) {
             logger.trace(log(sequence, "Unregistering listener"));
         }
+        java.util.concurrent.ScheduledFuture<?> alarmFuture = _alarmFutures.remove(sequence);
+        if (alarmFuture != null) {
+            alarmFuture.cancel(false);
+        }
         return _waitForList.remove(sequence);
     }
 
@@ -205,8 +213,13 @@ protected void cancelAllCommands() {
             Map.Entry<Long, ServerListener> entry = it.next();
             it.remove();
 
+            long seq = entry.getKey();
+            java.util.concurrent.ScheduledFuture<?> alarmFuture = _alarmFutures.remove(seq);
+            if (alarmFuture != null) {
+                alarmFuture.cancel(false);
+            }
             ServerListener monitor = entry.getValue();
-            logger.debug(log(entry.getKey(), "Sending disconnect to " + monitor.getClass()));
+            logger.debug(log(seq, "Sending disconnect to " + monitor.getClass()));
             monitor.processDisconnect();
         }
     }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentAttache.java

@@ -120,6 +120,7 @@ public int compare(final Object o1, final Object o2) {
     protected String _name = null;
     protected HypervisorType _hypervisorType;
     protected final ConcurrentHashMap<Long, Listener> _waitForList;
+    protected ConcurrentHashMap<Long, java.util.concurrent.ScheduledFuture<?>> _alarmFutures;
     protected final LinkedList<Request> _requests;
     protected Long _currentSequence;
     protected Status _status = Status.Connecting;
@@ -146,6 +147,7 @@ protected AgentAttache(final AgentManagerImpl agentMgr, final long id, final Str
         _name = name;
         _hypervisorType = hypervisorType;
         _waitForList = new ConcurrentHashMap<Long, Listener>();
+        _alarmFutures = new ConcurrentHashMap<>();
         _currentSequence = null;
         _maintenance = maintenance;
         _requests = new LinkedList<Request>();
@@ -241,13 +243,19 @@ protected synchronized int findRequest(final long seq) {
     protected void registerListener(final long seq, final Listener listener) {
         logger.trace(LOG_SEQ_FORMATTED_STRING, seq, "Registering listener");
         if (listener.getTimeout() != -1) {
-            s_listenerExecutor.schedule(new Alarm(seq), listener.getTimeout(), TimeUnit.SECONDS);
+            java.util.concurrent.ScheduledFuture<?> alarmFuture =
+                    s_listenerExecutor.schedule(new Alarm(seq), listener.getTimeout(), TimeUnit.SECONDS);
+            _alarmFutures.put(seq, alarmFuture);
         }
         _waitForList.put(seq, listener);
     }
 
     protected Listener unregisterListener(final long sequence) {
         logger.trace(LOG_SEQ_FORMATTED_STRING, sequence, "Unregistering listener");
+        java.util.concurrent.ScheduledFuture<?> alarmFuture = _alarmFutures.remove(sequence);
+        if (alarmFuture != null) {
+            alarmFuture.cancel(false);
+        }
         return _waitForList.remove(sequence);
     }
 
@@ -338,8 +346,13 @@ protected void cancelAllCommands(final Status state, final boolean cancelActive)
             while (it.hasNext()) {
                 final Map.Entry<Long, Listener> entry = it.next();
                 it.remove();
+                long seq = entry.getKey();
+                java.util.concurrent.ScheduledFuture<?> alarmFuture = _alarmFutures.remove(seq);
+                if (alarmFuture != null) {
+                    alarmFuture.cancel(false);
+                }
                 final Listener monitor = entry.getValue();
-                logger.debug(LOG_SEQ_FORMATTED_STRING, entry.getKey(), "Sending disconnect to " + monitor.getClass());
+                logger.debug(LOG_SEQ_FORMATTED_STRING, seq, "Sending disconnect to " + monitor.getClass());
                 monitor.processDisconnect(_id,  _uuid, _name, state);
             }
         }

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -1409,6 +1409,9 @@ protected boolean handleDisconnectWithoutInvestigation(final AgentAttache attach
             logger.debug("Handling disconnect without investigation for {}", attache);
             if (joinLock.lock(getTimeoutSec())) {
                 result = disconnectAgent(attache, event, transitState, hostId, joinLock);
+            } else {
+                logger.warn("Failed to acquire join lock for {} within timeout, " +
+                        "disconnect event {} may be lost; another MS node may be processing it", attache, event);
             }
         } finally {
             joinLock.releaseRef();
@@ -2269,8 +2272,8 @@ protected void processRequest(final Link link, final Request request) {
         private boolean sendRequestStartupCommand(long hostId, HostVO host) {
             boolean requestStartup = false;
             Set<Status> aliveStatuses = getAliveHostStatuses();
-            boolean lockAvailable = GlobalLock.isLockAvailable(getHostJoinLockName(hostId));
             if (host.getStatus() == null || !aliveStatuses.contains(host.getStatus())) {
+                boolean lockAvailable = GlobalLock.isLockAvailable(getHostJoinLockName(hostId));
                 if (!lockAvailable) {
                     logger.debug("Ping from host {}: requesting startup command " +
                             "due to host status ({}) is not considered as alive ({}), " +