DNM: kola testiso folding proof of concept

.cci.jenkinsfile

@@ -38,26 +38,15 @@ cosaPod(cpu: "${cpuCount}",
     // Run stage Build FCOS (init, fetch and build)
     cosaBuild(skipKola: 1, cosaDir: "/srv", noForce: true)
 
-    // Run stage Kola QEMU (basic-qemu-scenarios, upgrade and self tests)
-    kola(cosaDir: "/srv", addExtTests: ["${env.WORKSPACE}/ci/run-kola-self-tests"])
-
     stage("Build Artifacts") {
-        def artifacts = ["aliyun", "applehv", "aws", "azure", "azurestack",
-                         "digitalocean", "exoscale", "gcp", "hetzner", "hyperv",
-                         "ibmcloud", "kubevirt", "live", "metal", "metal4k",
-                         "nutanix", "openstack", "vultr"]
+        def artifacts = ["qemu", "live", "metal", "metal4k"]
         utils.cosaCmd(cosaDir: "/srv", args: "osbuild ${artifacts.join(' ')}")
-        cosaParallelCmds(cosaDir: "/srv", commands: ["vmware", "virtualbox"])
         // quick schema validation
         utils.cosaCmd(cosaDir: "/srv", args: "meta --get name")
     }
 
-    kolaTestIso(cosaDir: "/srv")
-
-    stage("Compress") {
-        // Test compression but only a few of the artifacts. This should test xz, zip, gzip
-        utils.cosaCmd(cosaDir: "/srv", args: "compress --fast --artifact=qemu --artifact=applehv --artifact=hyperv")
-    }
+    // Run stage Kola QEMU (basic-qemu-scenarios, upgrade and self tests)
+    kola(cosaDir: "/srv", addExtTests: ["${env.WORKSPACE}/ci/run-kola-self-tests"])
 
     stage("Upload Dry Run") {
         utils.cosaCmd(cosaDir: "/srv", args: "buildupload --dry-run s3 --acl=public-read my-nonexistent-bucket/my/prefix")

mantle/cmd/kola/spawn.go

@@ -32,7 +32,6 @@ import (
 	"github.com/coreos/coreos-assembler/mantle/kola"
 	"github.com/coreos/coreos-assembler/mantle/platform"
 	"github.com/coreos/coreos-assembler/mantle/platform/conf"
-	"github.com/coreos/coreos-assembler/mantle/platform/machine/qemu"
 )
 
 var (
@@ -162,9 +161,8 @@ func runSpawn(cmd *cobra.Command, args []string) error {
 		if spawnVerbose {
 			fmt.Println("Spawning machine...")
 		}
-		// use qemu-specific interface only if needed
-		if strings.HasPrefix(kolaPlatform, "qemu") && (spawnMachineOptions != "" || !spawnRemove) {
-			machineOpts := platform.QemuMachineOptions{
+		if spawnMachineOptions != "" || !spawnRemove {
+			machineOpts := platform.MachineOptions{
 				DisablePDeathSig: !spawnRemove,
 			}
 			if spawnMachineOptions != "" {
@@ -178,13 +176,7 @@ func runSpawn(cmd *cobra.Command, args []string) error {
 					return errors.Wrapf(err, "Could not unmarshal machine options")
 				}
 			}
-
-			switch qc := cluster.(type) {
-			case *qemu.Cluster:
-				mach, err = qc.NewMachineWithQemuOptions(userdata, machineOpts)
-			default:
-				plog.Fatalf("unreachable: qemu cluster %v unknown type", qc)
-			}
+			mach, err = cluster.NewMachineWithOptions(userdata, machineOpts)
 		} else {
 			mach, err = cluster.NewMachine(userdata)
 		}

mantle/harness/harness.go

@@ -139,13 +139,22 @@ func (t *H) StartExecTimer() {
 
 func (t *H) RunWithExecTimeoutCheck(f func(), errMsg string) {
 	if t.execTimer == nil {
-		// Some subtests do not explcitly start timer, since timer is started in
-		// kola/harness.go: runTest. So we will assign a timer in that case.
+		// If the timer is not started go ahead and start it now.
 		t.StartExecTimer()
 	}
 	t.runTimeoutCheck(t.timeoutContext, t.timeout, f, errMsg)
 }
 
+// TimeoutContext returns a context that is cancelled when the test
+// execution timer fires. If the timer has not been started yet it
+// is started now.
+func (t *H) TimeoutContext() context.Context {
+	if t.execTimer == nil {
+		t.StartExecTimer()
+	}
+	return t.timeoutContext
+}
+
 func (t *H) StopExecTimer() {
 	if t.execTimer == nil {
 		return

mantle/kola/cluster/cluster.go

@@ -136,16 +136,7 @@ func DropFile(machines []platform.Machine, localPath string) error {
 // This ensures the output will be correctly accumulated under the correct
 // test.
 func (t *TestCluster) SSH(m platform.Machine, cmd string) ([]byte, error) {
-	var stdout, stderr []byte
-	var err error
-	f := func() {
-		stdout, stderr, err = m.SSH(cmd)
-	}
-
-	errMsg := fmt.Sprintf("ssh: %s", cmd)
-	// If f does not before the test timeout, the RunWithExecTimeoutCheck
-	// will end this goroutine and mark the test as failed
-	t.H.RunWithExecTimeoutCheck(f, errMsg)
+	stdout, stderr, err := m.SSH(cmd)
 	if len(stderr) > 0 {
 		for _, line := range strings.Split(string(stderr), "\n") {
 			t.Log(line)

mantle/kola/harness.go

@@ -1238,18 +1238,20 @@ ExecStart=%s
 		DependencyDir: destDirs,
 		Tags:          []string{"external"},
 
-		AdditionalDisks:           targetMeta.AdditionalDisks,
-		PrimaryDisk:               targetMeta.PrimaryDisk,
-		InjectContainer:           targetMeta.InjectContainer,
-		MinMemory:                 targetMeta.MinMemory,
-		NumaNodes:                 targetMeta.NumaNodes,
-		MinDiskSize:               targetMeta.MinDiskSize,
-		AdditionalNics:            targetMeta.AdditionalNics,
-		AppendKernelArgs:          targetMeta.AppendKernelArgs,
-		AppendFirstbootKernelArgs: targetMeta.AppendFirstbootKernelArgs,
-		InstanceType:              targetMeta.InstanceType,
-		NonExclusive:              !targetMeta.Exclusive,
-		Conflicts:                 targetMeta.Conflicts,
+		MachineOptions: platform.MachineOptions{
+			AdditionalDisks:           targetMeta.AdditionalDisks,
+			PrimaryDisk:               targetMeta.PrimaryDisk,
+			MinMemory:                 targetMeta.MinMemory,
+			NumaNodes:                 targetMeta.NumaNodes,
+			MinDiskSize:               targetMeta.MinDiskSize,
+			AdditionalNics:            targetMeta.AdditionalNics,
+			AppendKernelArgs:          targetMeta.AppendKernelArgs,
+			AppendFirstbootKernelArgs: targetMeta.AppendFirstbootKernelArgs,
+			InstanceType:              targetMeta.InstanceType,
+		},
+		InjectContainer: targetMeta.InjectContainer,
+		NonExclusive:    !targetMeta.Exclusive,
+		Conflicts:       targetMeta.Conflicts,
 
 		Run: func(c cluster.TestCluster) {
 			mach := c.Machines()[0]
@@ -1620,7 +1622,7 @@ func makeNonExclusiveTest(bucket int, tests []*register.Test, flight platform.Fl
 		if test.HasFlag(register.AllowConfigWarnings) {
 			plog.Fatalf("Non-exclusive test %v cannot have AllowConfigWarnings flag", test.Name)
 		}
-		if test.AppendKernelArgs != "" {
+		if test.MachineOptions.AppendKernelArgs != "" {
 			plog.Fatalf("Non-exclusive test %v cannot have AppendKernelArgs", test.Name)
 		}
 		if !internetAccess && testRequiresInternet(test) {
@@ -1759,11 +1761,11 @@ func waitForMemory(h *harness.H, flight platform.Flight, t *register.Test) {
 // pool. This should be called after the test's QEMU VM has been started and
 // the memory allocated for the VM.
 func releaseMemoryCount(flight platform.Flight, t *register.Test) {
+	reservedMemoryCountMutex.Lock()
+	defer reservedMemoryCountMutex.Unlock()
 	if t.ReservedMemoryCountMiB == 0 {
 		return // memory count already released
 	}
-	reservedMemoryCountMutex.Lock()
-	defer reservedMemoryCountMutex.Unlock()
 	reservedMemoryCountMiB -= t.ReservedMemoryCountMiB
 	t.ReservedMemoryCountMiB = 0
 	if reservedMemoryCountMiB < 0 {
@@ -1782,8 +1784,8 @@ func getNeededMemoryMiB(t *register.Test) int {
 		}
 	}
 	// If the test specifies MinMemory, use that.
-	if t.MinMemory != 0 {
-		return t.MinMemory
+	if t.MachineOptions.MinMemory != 0 {
+		return t.MachineOptions.MinMemory
 	}
 	// Fall back to architecture-specific defaults from the QEMU platform.
 	return platform.DefaultMemoryMiB(Options.CosaBuildArch)
@@ -1811,6 +1813,7 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig
 		SSHOnTestFailure:   Options.SSHOnTestFailure,
 		WarningsAction:     conf.FailWarnings,
 		EarlyRelease:       h.Release,
+		TestExecTimeout:    h.TimeoutContext(),
 	}
 	if t.HasFlag(register.AllowConfigWarnings) {
 		rconf.WarningsAction = conf.IgnoreWarnings
@@ -1823,6 +1826,8 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig
 	}
 	defer func() {
 		h.StopExecTimer()
+		// give some time for the remote journal to be flushed before we Destroy()
+		time.Sleep(2 * time.Second)
 		c.Destroy()
 		// Release the memory reservation (if there was one) now that the VM is gone.
 		releaseMemoryCount(flight, t)
@@ -1858,19 +1863,7 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig
 	if t.ClusterSize > 0 {
 		var userdata *conf.UserData = t.UserData
 
-		options := platform.MachineOptions{
-			MultiPathDisk:             t.MultiPathDisk,
-			PrimaryDisk:               t.PrimaryDisk,
-			AdditionalDisks:           t.AdditionalDisks,
-			MinMemory:                 t.MinMemory,
-			MinDiskSize:               t.MinDiskSize,
-			NumaNodes:                 t.NumaNodes,
-			AdditionalNics:            t.AdditionalNics,
-			AppendKernelArgs:          t.AppendKernelArgs,
-			AppendFirstbootKernelArgs: t.AppendFirstbootKernelArgs,
-			SkipStartMachine:          true,
-			InstanceType:              t.InstanceType,
-		}
+		options := t.MachineOptions
 
 		if testSecureBoot(t) {
 			options.Firmware = "uefi-secure"
@@ -1913,32 +1906,19 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig
 		tcluster.H.WarningOnFailure()
 	}
 
-	// Note that we passed in SkipStartMachine=true in our machine
-	// options. This means NewMachines() didn't block on the machines
-	// being up with SSH access before returning; i.e. it skipped running
-	// platform.StartMachines(). The machines should now be booting.
-	// Let's start the test execution timer and then run mach.Start()
-	// (wrapper for platform.StartMachine()) which sets up the journal
-	// forwarding and runs machine checks, both of which require SSH
-	// to be up, which implies Ignition has completed successfully.
-	//
-	// We do all of this so that the time it takes to run Ignition can
-	// be included in our test execution timeout.
-	h.StartExecTimer()
-	for _, mach := range tcluster.Machines() {
-		plog.Debugf("Trying to StartMachine() %v", mach.ID())
-		var err error
-		tcluster.RunWithExecTimeoutCheck(func() {
-			err = mach.Start()
-		}, fmt.Sprintf("SSH unsuccessful within allotted timeframe for %v.", mach.ID()))
-		if err != nil {
-			h.Fatal(errors.Wrapf(err, "mach.Start() failed"))
+	// Machines may be created directly by the test (not via the
+	// harness with NewMachines above), so we poll asynchronously
+	// via a goroutine until at least one shows up and then release
+	// the temporary memory reservation.
+	go func() {
+		// Wait for at least one machine in the cluster to exist.
+		// At the point machines show up in tcluster.Machines() they've
+		// already been contacted successfully via SSH in StartMachine().
+		for len(tcluster.Machines()) == 0 {
+			time.Sleep(1 * time.Second)
 		}
-	}
-
-	// Release the temporary memory reservation now that the VM is up and should
-	// be using it's allotted memory (preallocation). Applicable on qemu only.
-	releaseMemoryCount(flight, t)
+		releaseMemoryCount(flight, t)
+	}()
 
 	// drop kolet binary on machines
 	if t.ExternalTest != "" || t.NativeFuncs != nil {
@@ -1999,12 +1979,6 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig
 		}
 	}
 
-	defer func() {
-		// give some time for the remote journal to be flushed so it can be read
-		// before we run the deferred machine destruction
-		time.Sleep(2 * time.Second)
-	}()
-
 	// run test
 	t.Run(tcluster)
 }

mantle/kola/register/register.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"github.com/coreos/coreos-assembler/mantle/kola/cluster"
+	"github.com/coreos/coreos-assembler/mantle/platform"
 	"github.com/coreos/coreos-assembler/mantle/platform/conf"
 )
 
@@ -70,45 +71,20 @@ type Test struct {
 	Timeout              time.Duration // the duration for which a test will be allowed to run
 	RequiredTag          string        // if specified, test is filtered by default unless tag is provided -- defaults to none
 	Description          string        // test description
-	NumaNodes            bool          // simulate two NUMA nodes
 
-	// Whether the primary disk is multipathed. Deprecated in favour of PrimaryDisk.
-	MultiPathDisk bool
-
-	// Sizes of additional empty disks to attach to the node, followed by
-	// comma-separated list of optional options (e.g. ["1G",
-	// "5G:mpath,foo,bar"]) -- defaults to none.
-	AdditionalDisks []string
-
-	// Size of primary disk to attach to the node, followed by
-	// comma-separated list of optional options (e.g. "20G:mpath"]).
-	PrimaryDisk string
+	// MachineOptions contains options for machine creation (disks, memory,
+	// kernel args, etc.). The test harness passes these to
+	// NewMachineWithOptions when ClusterSize > 0.
+	MachineOptions platform.MachineOptions
 
 	// InjectContainer will cause the ostree base image to be injected into the target
 	InjectContainer bool
 
-	// Minimum amount of memory in MB required for test.
-	MinMemory int
-
 	// The artificially reserved memory count in MiB for the test. This is used
 	// for budgeting memory usage for tests prior to the VMs starting up on the
 	// QEMU platform.
 	ReservedMemoryCountMiB int
 
-	// Minimum amount of primary disk in GB required for test. Deprecated in favour
-	// of PrimaryDisk.
-	MinDiskSize int
-
-	// Additional amount of NICs required for test.
-	AdditionalNics int
-
-	// Additional kernel arguments to append to the defaults.
-	AppendKernelArgs string
-
-	// Additional first boot kernel arguments to append to the defaults.
-	AppendFirstbootKernelArgs string
-
-	// ExternalTest is a path to a binary that will be uploaded
 	ExternalTest string
 	// DependencyDir is a path to directory that will be uploaded, normally used by external tests
 	DependencyDir DepDirMap
@@ -124,10 +100,6 @@ type Test struct {
 	// Conflicts is non-empty iff nonexclusive is true
 	// Contains the tests that conflict with this particular test
 	Conflicts []string
-
-	// If provided, this test will be run on the target instance type.
-	// This overrides the instance type set with `kola run`
-	InstanceType string
 }
 
 // Registered tests that run as part of `kola run` live here. Mapping of names

mantle/kola/registry/registry.go

@@ -7,6 +7,7 @@ import (
 	_ "github.com/coreos/coreos-assembler/mantle/kola/tests/etcd"
 	_ "github.com/coreos/coreos-assembler/mantle/kola/tests/fips"
 	_ "github.com/coreos/coreos-assembler/mantle/kola/tests/ignition"
+	_ "github.com/coreos/coreos-assembler/mantle/kola/tests/install-media"
 	_ "github.com/coreos/coreos-assembler/mantle/kola/tests/metadata"
 	_ "github.com/coreos/coreos-assembler/mantle/kola/tests/misc"
 	_ "github.com/coreos/coreos-assembler/mantle/kola/tests/ostree"