FR77: Improve Management Server Maintenance (apache#480)

* Safely shutdown feature (ref: apache#6755)

* Updated version and some improvements

* Management Server Maintenance - Prepare and Cancel Maintenance changes

This is supported for the Cloudstack deployments with multiple management servers.
- During preparing for maintenance, MS waits for pending jobs to finish, and then Transfer/Migrate the agents to other available MS
- New APIs: prepareForMaintenance, cancelMaintenance
- New MS States: PreparingToMaintenance, Maintenance

* check for single active management server

* refactoring plugin name

* updated version, and cleanup

* code improvements

* support list hosts by management server id

* update ui with ms maintenance apis

* code improvements

* ui changes

* ui icons update

* ui fixes

* cond checks for maintenance and shutdown

* fix for management server not down issue on service stop

* continue with other components on error

* agent transfer fixes

* maintenance window timeout and fixes

* ui changes - added connected agents tab, and updated hosts & management servers fields

* marvin test update

* keep maintenance after shutdown/restart, do not update last_updated time in cluster heartbeat during maintenance (notifies node inactive/down after heartbeat threshold)

* listener for ms maintenance updates

* cleanup

* keep last msid in host table

* review comments

* allow only one mgmt server to prepare for maintenance

* added ms uuid in logs

* minor code improvements

* ui fields update

* fix systemvm navigation in connected agents

* algorithm check and input from ui

* check for active ms from host setting

* agent migration code improvements

* minor ui label fix

* fixes & code improvements

* agent reconnect fixes, consider avoid list

* ui fixes

* direct agents transfer and pending jobs timer task fixes

* close unclosed socket channels if any

* Updated pending jobs check timer task with ScheduledExecutorService

* fixes

* keep maintenance state on trigger shutdown call when ms is in maintenance

* direct agent transfer fixes

* add pending jobs count to ms response

* during ms heartbeat, update state to up only when it's down

* allow vm work jobs of async job created before prepare for maintenance

* Revert "keep maintenance state on trigger shutdown call when ms is in maintenance"

This reverts commit 4ebbea71ef20a65286bed41a517f03e253a8fe90.

* removed duplicate schema changes from schema-41800to41810.sql (already defined at schema-41811to41812.sql)

Author: sureshanaparti

agent/src/main/java/com/cloud/agent/Agent.java

@@ -27,6 +27,7 @@
 import java.nio.channels.ClosedChannelException;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -40,6 +41,8 @@
 
 import javax.naming.ConfigurationException;
 
+import com.cloud.agent.api.MigrateAgentConnectionAnswer;
+import com.cloud.agent.api.MigrateAgentConnectionCommand;
 import com.cloud.resource.AgentStatusUpdater;
 import com.cloud.resource.ResourceStatusUpdater;
 import com.cloud.agent.api.PingAnswer;
@@ -307,7 +310,6 @@ public void start() {
         }
         _shell.updateConnectedHost();
         scavengeOldAgentObjects();
-
     }
 
     public void stop(final String reason, final String detail) {
@@ -457,13 +459,18 @@ public synchronized void lockStartupTask(final Link link) {
     }
 
     public void sendStartup(final Link link) {
+        sendStartup(link, false);
+    }
+
+    public void sendStartup(final Link link, boolean transfer) {
         final StartupCommand[] startup = _resource.initialize();
         if (startup != null) {
             final String msHostList = _shell.getPersistentProperty(null, "host");
             final Command[] commands = new Command[startup.length];
             for (int i = 0; i < startup.length; i++) {
                 setupStartupCommand(startup[i]);
                 startup[i].setMSHostList(msHostList);
+                startup[i].setConnectionTransferred(transfer);
                 commands[i] = startup[i];
             }
             final Request request = new Request(_id != null ? _id : -1, -1, commands, false, false);
@@ -516,9 +523,14 @@ public Task create(final Task.Type type, final Link link, final byte[] data) {
     }
 
     protected void reconnect(final Link link) {
-        if (!_reconnectAllowed) {
+        reconnect(link, null, null, false);
+    }
+
+    protected void reconnect(final Link link, String preferredHost, List<String> avoidHostList, boolean forTransfer) {
+        if (!(forTransfer || _reconnectAllowed)) {
             return;
         }
+
         synchronized (this) {
             if (_startup != null) {
                 _startup.cancel();
@@ -550,22 +562,29 @@ protected void reconnect(final Link link) {
             _shell.getBackoffAlgorithm().waitBeforeRetry();
         }
 
+        String host = preferredHost;
+        if (StringUtils.isEmpty(host)) {
+            host = _shell.getNextHost();
+        }
+
         do {
-            final String host = _shell.getNextHost();
-            _connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), this);
-            s_logger.info("Reconnecting to host:" + host);
-            try {
-                _connection.start();
-            } catch (final NioConnectionException e) {
-                s_logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e);
-                _connection.stop();
+            if (CollectionUtils.isEmpty(avoidHostList) || !avoidHostList.contains(host)) {
+                _connection = new NioClient("Agent", host, _shell.getPort(), _shell.getWorkers(), this);
+                s_logger.info("Reconnecting to host:" + host);
                 try {
-                    _connection.cleanUp();
-                } catch (final IOException ex) {
-                    s_logger.warn("Fail to clean up old connection. " + ex);
+                    _connection.start();
+                } catch (final NioConnectionException e) {
+                    s_logger.info("Attempted to re-connect to the server, but received an unexpected exception, trying again...", e);
+                    _connection.stop();
+                    try {
+                        _connection.cleanUp();
+                    } catch (final IOException ex) {
+                        s_logger.warn("Fail to clean up old connection. " + ex);
+                    }
                 }
             }
             _shell.getBackoffAlgorithm().waitBeforeRetry();
+            host = _shell.getNextHost();
         } while (!_connection.isStartup());
         _shell.updateConnectedHost();
         s_logger.info("Connected to the host: " + _shell.getConnectedHost());
@@ -674,6 +693,8 @@ protected void processRequest(final Request request, final Link link) {
                         }
                     } else if (cmd instanceof SetupMSListCommand) {
                         answer = setupManagementServerList((SetupMSListCommand) cmd);
+                    } else if (cmd instanceof MigrateAgentConnectionCommand) {
+                        answer = migrateAgentToOtherMS((MigrateAgentConnectionCommand) cmd);
                     } else {
                         if (cmd instanceof ReadyCommand) {
                             processReadyCommand(cmd);
@@ -829,6 +850,53 @@ private Answer setupManagementServerList(final SetupMSListCommand cmd) {
         return new SetupMSListAnswer(true);
     }
 
+    private Answer migrateAgentToOtherMS(final MigrateAgentConnectionCommand cmd) {
+        try {
+            if (CollectionUtils.isNotEmpty(cmd.getMsList())) {
+                processManagementServerList(cmd.getMsList(), cmd.getLbAlgorithm(), cmd.getLbCheckInterval());
+            }
+            migrateAgentConnection(cmd.getAvoidMsList());
+        } catch (Exception e) {
+            String errMsg = "Migrate agent connection failed, due to " + e.getMessage();
+            s_logger.debug(errMsg, e);
+            return new MigrateAgentConnectionAnswer(errMsg);
+        }
+        return new MigrateAgentConnectionAnswer(true);
+    }
+
+    private void migrateAgentConnection(List<String> avoidMsList) {
+        final String[] msHosts = _shell.getHosts();
+        if (msHosts == null || msHosts.length < 1) {
+            throw new CloudRuntimeException("Management Server hosts empty, not properly configured in agent");
+        }
+
+        List<String> msHostsList = new ArrayList<>(Arrays.asList(msHosts));
+        msHostsList.removeAll(avoidMsList);
+        if (msHostsList.isEmpty() || StringUtils.isEmpty(msHostsList.get(0))) {
+            throw new CloudRuntimeException("No other Management Server hosts to migrate");
+        }
+
+        String preferredHost  = null;
+        for (String msHost : msHostsList) {
+            try (final Socket socket = new Socket()) {
+                socket.connect(new InetSocketAddress(msHost, _shell.getPort()), 5000);
+                preferredHost = msHost;
+                break;
+            } catch (final IOException e) {
+                throw new CloudRuntimeException("Management server host: " + msHost + " is not reachable, to migrate connection");
+            }
+        }
+
+        if (preferredHost == null) {
+            throw new CloudRuntimeException("Management server host(s) are not reachable, to migrate connection");
+        }
+
+        s_logger.debug("Management server host " + preferredHost + " is found to be reachable, trying to reconnect");
+        _shell.resetHostCounter();
+        _shell.setConnectionTransfer(true);
+        reconnect(_link, preferredHost, avoidMsList, true);
+    }
+
     public void processResponse(final Response response, final Link link) {
         final Answer answer = response.getAnswer();
         if (s_logger.isDebugEnabled()) {
@@ -1122,7 +1190,8 @@ public void doTask(final Task task) throws TaskExecutionException {
             if (task.getType() == Task.Type.CONNECT) {
                 _shell.getBackoffAlgorithm().reset();
                 setLink(task.getLink());
-                sendStartup(task.getLink());
+                sendStartup(task.getLink(), _shell.isConnectionTransfer());
+                _shell.setConnectionTransfer(false);
             } else if (task.getType() == Task.Type.DATA) {
                 Request request;
                 try {
@@ -1141,6 +1210,7 @@ public void doTask(final Task task) throws TaskExecutionException {
                     s_logger.error("Error parsing task", e);
                 }
             } else if (task.getType() == Task.Type.DISCONNECT) {
+                _shell.setConnectionTransfer(false);
                 reconnect(task.getLink());
                 return;
             } else if (task.getType() == Task.Type.OTHER) {

agent/src/main/java/com/cloud/agent/AgentShell.java

@@ -76,6 +76,7 @@ public class AgentShell implements IAgentShell, Daemon {
     private String hostToConnect;
     private String connectedHost;
     private Long preferredHostCheckInterval;
+    private boolean connectionTransfer = false;
     protected AgentProperties agentProperties = new AgentProperties();
 
     public AgentShell() {
@@ -214,6 +215,14 @@ public void setPersistentProperty(String prefix, String name, String value) {
             _storage.persist(name, value);
     }
 
+    public boolean isConnectionTransfer() {
+        return connectionTransfer;
+    }
+
+    public void setConnectionTransfer(boolean connectionTransfer) {
+        this.connectionTransfer = connectionTransfer;
+    }
+
     void loadProperties() throws ConfigurationException {
         final File file = PropertiesUtil.findConfigFile("agent.properties");
 

agent/src/main/java/com/cloud/agent/IAgentShell.java

@@ -70,4 +70,8 @@ public interface IAgentShell {
     String getConnectedHost();
 
     void launchNewAgent(ServerResource resource) throws ConfigurationException;
+
+    boolean isConnectionTransfer();
+
+    void setConnectionTransfer(boolean connectionTransfer);
 }

api/src/main/java/com/cloud/host/Host.java

@@ -175,6 +175,8 @@ public static String[] toStrings(Host.Type... types) {
      */
     Long getManagementServerId();
 
+    Long getLastManagementServerId();
+
     /*
      *@return removal date
      */

api/src/main/java/com/cloud/host/Status.java

@@ -127,6 +127,7 @@ public static String[] toStrings(Status... states) {
         s_fsm.addTransition(Status.Connecting, Event.HostDown, Status.Down);
         s_fsm.addTransition(Status.Connecting, Event.Ping, Status.Connecting);
         s_fsm.addTransition(Status.Connecting, Event.ManagementServerDown, Status.Disconnected);
+        s_fsm.addTransition(Status.Connecting, Event.StartAgentRebalance, Status.Rebalancing);
         s_fsm.addTransition(Status.Connecting, Event.AgentDisconnected, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.PingTimeout, Status.Alert);
         s_fsm.addTransition(Status.Up, Event.AgentDisconnected, Status.Alert);

api/src/main/java/com/cloud/resource/ResourceService.java

@@ -23,11 +23,11 @@
 import org.apache.cloudstack.api.command.admin.cluster.UpdateClusterCmd;
 import org.apache.cloudstack.api.command.admin.host.AddHostCmd;
 import org.apache.cloudstack.api.command.admin.host.AddSecondaryStorageCmd;
-import org.apache.cloudstack.api.command.admin.host.CancelMaintenanceCmd;
+import org.apache.cloudstack.api.command.admin.host.CancelHostMaintenanceCmd;
 import org.apache.cloudstack.api.command.admin.host.ReconnectHostCmd;
 import org.apache.cloudstack.api.command.admin.host.UpdateHostCmd;
 import org.apache.cloudstack.api.command.admin.host.UpdateHostPasswordCmd;
-import org.apache.cloudstack.api.command.admin.host.PrepareForMaintenanceCmd;
+import org.apache.cloudstack.api.command.admin.host.PrepareForHostMaintenanceCmd;
 import org.apache.cloudstack.api.command.admin.host.DeclareHostAsDegradedCmd;
 import org.apache.cloudstack.api.command.admin.host.CancelHostAsDegradedCmd;
 
@@ -51,7 +51,7 @@ public interface ResourceService {
 
     Host autoUpdateHostAllocationState(Long hostId, ResourceState.Event resourceEvent) throws NoTransitionException;
 
-    Host cancelMaintenance(CancelMaintenanceCmd cmd);
+    Host cancelMaintenance(CancelHostMaintenanceCmd cmd);
 
     Host reconnectHost(ReconnectHostCmd cmd) throws AgentUnavailableException;
 
@@ -69,7 +69,7 @@ public interface ResourceService {
 
     List<? extends Host> discoverHosts(AddSecondaryStorageCmd cmd) throws IllegalArgumentException, DiscoveryException, InvalidParameterValueException;
 
-    Host maintain(PrepareForMaintenanceCmd cmd);
+    Host maintain(PrepareForHostMaintenanceCmd cmd);
 
     Host declareHostAsDegraded(DeclareHostAsDegradedCmd cmd) throws NoTransitionException;
 

api/src/main/java/com/cloud/server/ManagementServerHostStats.java

@@ -19,6 +19,7 @@
 package com.cloud.server;
 
 import java.util.Date;
+import java.util.List;
 
 /**
  * management server related stats
@@ -70,6 +71,10 @@ public interface ManagementServerHostStats {
 
     String getOsDistribution();
 
+    List<String> getLastAgents();
+
+    List<String> getAgents();
+
     int getAgentCount();
 
     long getHeapMemoryUsed();

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -541,6 +541,8 @@ public class ApiConstants {
     public static final String PRIVATE_NETWORK_ID = "privatenetworkid";
     public static final String ALLOCATION_STATE = "allocationstate";
     public static final String MANAGED_STATE = "managedstate";
+    public static final String MANAGEMENT_SERVER_ID = "managementserverid";
+    public static final String MANAGEMENT_SERVER_NAME = "managementservername";
     public static final String STORAGE_ID = "storageid";
     public static final String PING_STORAGE_SERVER_IP = "pingstorageserverip";
     public static final String PING_DIR = "pingdir";
@@ -1038,6 +1040,13 @@ public class ApiConstants {
     public static final String LOGOUT = "logout";
     public static final String LIST_IDPS = "listIdps";
 
+    public static final String MAINTENANCE_INITIATED = "maintenanceinitiated";
+    public static final String SHUTDOWN_TRIGGERED = "shutdowntriggered";
+    public static final String READY_FOR_SHUTDOWN = "readyforshutdown";
+    public static final String PENDING_JOBS_COUNT = "pendingjobscount";
+    public static final String AGENTS_COUNT = "agentscount";
+    public static final String AGENTS = "agents";
+
     public static final String PUBLIC_MTU = "publicmtu";
     public static final String PRIVATE_MTU = "privatemtu";
     public static final String MTU = "mtu";

…and/admin/host/CancelMaintenanceCmd.java …admin/host/CancelHostMaintenanceCmd.javaapi/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelMaintenanceCmd.java renamed to api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelHostMaintenanceCmd.java api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelMaintenanceCmd.java renamed to api/src/main/java/org/apache/cloudstack/api/command/admin/host/CancelHostMaintenanceCmd.java

@@ -34,8 +34,8 @@
 
 @APICommand(name = "cancelHostMaintenance", description = "Cancels host maintenance.", responseObject = HostResponse.class,
         requestHasSensitiveInfo = false, responseHasSensitiveInfo = false)
-public class CancelMaintenanceCmd extends BaseAsyncCmd {
-    public static final Logger s_logger = Logger.getLogger(CancelMaintenanceCmd.class.getName());
+public class CancelHostMaintenanceCmd extends BaseAsyncCmd {
+    public static final Logger s_logger = Logger.getLogger(CancelHostMaintenanceCmd.class.getName());
 
 
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/admin/host/ListHostsCmd.java

@@ -21,6 +21,7 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.cloudstack.api.response.ManagementServerResponse;
 import org.apache.log4j.Logger;
 
 import org.apache.cloudstack.api.APICommand;
@@ -107,6 +108,9 @@ public class ListHostsCmd extends BaseListCmd {
     @Parameter(name = ApiConstants.HYPERVISOR, type = CommandType.STRING, description = "hypervisor type of host: XenServer,KVM,VMware,Hyperv,BareMetal,Simulator")
     private String hypervisor;
 
+    @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, entityType = ManagementServerResponse.class, description = "the id of the management server", since="4.18.1")
+    private Long managementServerId;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -178,7 +182,6 @@ public String getResourceState() {
         return resourceState;
     }
 
-
     public Boolean isOutOfBandManagementEnabled() {
         return outOfBandManagementEnabled;
     }
@@ -187,6 +190,10 @@ public String getHostOutOfBandManagementPowerState() {
         return outOfBandManagementPowerState;
     }
 
+    public Long getManagementServerId() {
+        return managementServerId;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////