tests: fix some Marvin smoketests (#2869)

Fixes intermittent failures:
- Add a pause to avoid  tests restarting before VRs recovering from HA have fully booted.
- Add some pauses to allow services to restart and hosts to reconnect before continuing tests.
- Adding a loop around this method because sometimes VRs are overloaded and just respond with exception, so it'll catch it and try 5 times with 30sec cooldown. If that fails as well it'll fail the test.
- wait until host is up using explicit check

Author: Paul Angus

test/integration/smoke/test_deploy_virtio_scsi_vm.py

@@ -96,6 +96,7 @@ def setUpClass(cls):
         cls.zone = get_zone(cls.apiclient, testClient.getZoneForTests())
         cls.pod = get_pod(cls.apiclient, cls.zone.id)
         cls.services['mode'] = cls.zone.networktype
+        cls.cleanup = []
         if cls.hypervisor.lower() not in ['kvm']:
             cls.hypervisorNotSupported = True
             return

test/integration/smoke/test_hostha_kvm.py

@@ -274,9 +274,12 @@ def test_hostha_enable_ha_when_host_in_maintenance(self):
             Tests Enable HA when host is in Maintenance mode, should be Ineligible
         """
         self.logger.debug("Starting test_hostha_enable_ha_when_host_in_maintenance")
-
+        self.logger.debug("Pausing to wait for VMs to have finished starting")
+        time.sleep(300)
+        
         # Enable HA
         self.configureAndEnableHostHa()
+        
 
         # Prepare for maintenance Host
         self.setHostToMaintanance(self.host.id)

test/integration/smoke/test_vm_life_cycle.py

@@ -900,16 +900,34 @@ def migrate_and_check(self, vm, src_host, dest_host, proto='tls'):
         vm_response = VirtualMachine.list(self.apiclient, id=vm.id)[0]
         self.assertEqual(vm_response.hostid, dest_host.id, "Check destination host ID of migrated VM")
 
+    def waitUntilHostInState(self, hostId, state="Up", interval=5, retries=20):
+        while retries > -1:
+            print("Waiting for host: %s to be %s. %s retries left." % (hostId, state, retries))
+            time.sleep(interval)
+            host = Host.list(
+                self.apiclient,
+                hostid=hostId,
+                type='Routing'
+            )[0]
+            if host.state != state:
+                if retries >= 0:
+                    retries = retries - 1
+                    continue
+            else:
+                print("Host %s now showing as %s" % (hostId, state))
+                return
+
     def unsecure_host(self, host):
         SshClient(host.ipaddress, port=22, user=self.hostConfig["username"], passwd=self.hostConfig["password"])\
             .execute("rm -f /etc/cloudstack/agent/cloud* && \
                       sed -i 's/listen_tls.*/listen_tls=0/g' /etc/libvirt/libvirtd.conf && \
                       sed -i 's/listen_tcp.*/listen_tcp=1/g' /etc/libvirt/libvirtd.conf && \
                       sed -i '/.*_file=.*/d' /etc/libvirt/libvirtd.conf && \
                       service libvirtd restart && \
+                      sleep 30 && \
                       service cloudstack-agent restart")
-
-        time.sleep(10)
+        print("Unsecuring Host: %s" % (host.name))
+        self.waitUntilHostInState(hostId=host.id, state="Up") 
         self.check_connection(host=host, secured='false')
         return host
 
@@ -921,6 +939,8 @@ def secure_all_hosts(self):
             self.apiclient.provisionCertificate(cmd)
 
         for host in self.hosts:
+            print("Securing Host %s" % host.name)
+            self.waitUntilHostInState(hostId=host.id, state="Up")
             self.check_connection(secured='true', host=host)
 
     def deploy_vm(self, origin_host):
@@ -971,6 +991,7 @@ def test_01_secure_vm_migration(self):
         vm = self.deploy_vm(src_host)
         self.cleanup.append(vm)
 
+        self.debug("Securing Host(s)")
         dest_host = self.get_target_host(secured='true', virtualmachineid=vm.id)
         self.migrate_and_check(vm, src_host, dest_host)
 

tools/marvin/marvin/lib/base.py

@@ -605,22 +605,34 @@ def create(cls, apiclient, services, templateid=None, accountid=None,
             return VirtualMachine(virtual_machine.__dict__, services)
 
         # program ssh access over NAT via PF
-        if mode.lower() == 'advanced':
-            cls.access_ssh_over_nat(
-                apiclient,
-                services,
-                virtual_machine,
-                allow_egress=allow_egress,
-                networkid=cmd.networkids[0] if cmd.networkids else None)
-        elif mode.lower() == 'basic':
-            if virtual_machine.publicip is not None:
-                # EIP/ELB (netscaler) enabled zone
-                vm_ssh_ip = virtual_machine.publicip
-            else:
-                # regular basic zone with security group
-                vm_ssh_ip = virtual_machine.nic[0].ipaddress
-            virtual_machine.ssh_ip = vm_ssh_ip
-            virtual_machine.public_ip = vm_ssh_ip
+        retries = 5
+        interval = 30
+        while retries > 0:
+            time.sleep(interval)
+            try:
+                if mode.lower() == 'advanced':
+                    cls.access_ssh_over_nat(
+                        apiclient,
+                        services,
+                        virtual_machine,
+                        allow_egress=allow_egress,
+                        networkid=cmd.networkids[0] if cmd.networkids else None)
+                elif mode.lower() == 'basic':
+                    if virtual_machine.publicip is not None:
+                        # EIP/ELB (netscaler) enabled zone
+                        vm_ssh_ip = virtual_machine.publicip
+                    else:
+                        # regular basic zone with security group
+                        vm_ssh_ip = virtual_machine.nic[0].ipaddress
+                    virtual_machine.ssh_ip = vm_ssh_ip
+                    virtual_machine.public_ip = vm_ssh_ip
+                break
+            except Exception as e:
+                if retries >= 0:
+                    retries = retries - 1
+                    continue
+                raise Exception(
+                    "The following exception appeared while programming ssh access - %s" % e)
 
         return VirtualMachine(virtual_machine.__dict__, services)