diff --git a/.cci.jenkinsfile b/.cci.jenkinsfile index 08b63a6e0a..ca408f2423 100644 --- a/.cci.jenkinsfile +++ b/.cci.jenkinsfile @@ -38,26 +38,15 @@ cosaPod(cpu: "${cpuCount}", // Run stage Build FCOS (init, fetch and build) cosaBuild(skipKola: 1, cosaDir: "/srv", noForce: true) - // Run stage Kola QEMU (basic-qemu-scenarios, upgrade and self tests) - kola(cosaDir: "/srv", addExtTests: ["${env.WORKSPACE}/ci/run-kola-self-tests"]) - stage("Build Artifacts") { - def artifacts = ["aliyun", "applehv", "aws", "azure", "azurestack", - "digitalocean", "exoscale", "gcp", "hetzner", "hyperv", - "ibmcloud", "kubevirt", "live", "metal", "metal4k", - "nutanix", "openstack", "vultr"] + def artifacts = ["qemu", "live", "metal", "metal4k"] utils.cosaCmd(cosaDir: "/srv", args: "osbuild ${artifacts.join(' ')}") - cosaParallelCmds(cosaDir: "/srv", commands: ["vmware", "virtualbox"]) // quick schema validation utils.cosaCmd(cosaDir: "/srv", args: "meta --get name") } - kolaTestIso(cosaDir: "/srv") - - stage("Compress") { - // Test compression but only a few of the artifacts. This should test xz, zip, gzip - utils.cosaCmd(cosaDir: "/srv", args: "compress --fast --artifact=qemu --artifact=applehv --artifact=hyperv") - } + // Run stage Kola QEMU (basic-qemu-scenarios, upgrade and self tests) + kola(cosaDir: "/srv", addExtTests: ["${env.WORKSPACE}/ci/run-kola-self-tests"]) stage("Upload Dry Run") { utils.cosaCmd(cosaDir: "/srv", args: "buildupload --dry-run s3 --acl=public-read my-nonexistent-bucket/my/prefix") diff --git a/mantle/cmd/kola/spawn.go b/mantle/cmd/kola/spawn.go index c71964e065..264cb76dae 100644 --- a/mantle/cmd/kola/spawn.go +++ b/mantle/cmd/kola/spawn.go @@ -32,7 +32,6 @@ import ( "github.com/coreos/coreos-assembler/mantle/kola" "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" ) var ( @@ -162,9 +161,8 @@ func runSpawn(cmd *cobra.Command, args []string) error { if spawnVerbose { fmt.Println("Spawning machine...") } - // use qemu-specific interface only if needed - if strings.HasPrefix(kolaPlatform, "qemu") && (spawnMachineOptions != "" || !spawnRemove) { - machineOpts := platform.QemuMachineOptions{ + if spawnMachineOptions != "" || !spawnRemove { + machineOpts := platform.MachineOptions{ DisablePDeathSig: !spawnRemove, } if spawnMachineOptions != "" { @@ -178,13 +176,7 @@ func runSpawn(cmd *cobra.Command, args []string) error { return errors.Wrapf(err, "Could not unmarshal machine options") } } - - switch qc := cluster.(type) { - case *qemu.Cluster: - mach, err = qc.NewMachineWithQemuOptions(userdata, machineOpts) - default: - plog.Fatalf("unreachable: qemu cluster %v unknown type", qc) - } + mach, err = cluster.NewMachineWithOptions(userdata, machineOpts) } else { mach, err = cluster.NewMachine(userdata) } diff --git a/mantle/harness/harness.go b/mantle/harness/harness.go index 1edb6a5738..8b2d56a8da 100644 --- a/mantle/harness/harness.go +++ b/mantle/harness/harness.go @@ -139,13 +139,22 @@ func (t *H) StartExecTimer() { func (t *H) RunWithExecTimeoutCheck(f func(), errMsg string) { if t.execTimer == nil { - // Some subtests do not explcitly start timer, since timer is started in - // kola/harness.go: runTest. So we will assign a timer in that case. + // If the timer is not started go ahead and start it now. t.StartExecTimer() } t.runTimeoutCheck(t.timeoutContext, t.timeout, f, errMsg) } +// TimeoutContext returns a context that is cancelled when the test +// execution timer fires. If the timer has not been started yet it +// is started now. +func (t *H) TimeoutContext() context.Context { + if t.execTimer == nil { + t.StartExecTimer() + } + return t.timeoutContext +} + func (t *H) StopExecTimer() { if t.execTimer == nil { return diff --git a/mantle/kola/cluster/cluster.go b/mantle/kola/cluster/cluster.go index 931e78f63a..5c22c0981b 100644 --- a/mantle/kola/cluster/cluster.go +++ b/mantle/kola/cluster/cluster.go @@ -136,16 +136,7 @@ func DropFile(machines []platform.Machine, localPath string) error { // This ensures the output will be correctly accumulated under the correct // test. func (t *TestCluster) SSH(m platform.Machine, cmd string) ([]byte, error) { - var stdout, stderr []byte - var err error - f := func() { - stdout, stderr, err = m.SSH(cmd) - } - - errMsg := fmt.Sprintf("ssh: %s", cmd) - // If f does not before the test timeout, the RunWithExecTimeoutCheck - // will end this goroutine and mark the test as failed - t.H.RunWithExecTimeoutCheck(f, errMsg) + stdout, stderr, err := m.SSH(cmd) if len(stderr) > 0 { for _, line := range strings.Split(string(stderr), "\n") { t.Log(line) diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index 711b81f213..e0690853f0 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -1238,18 +1238,20 @@ ExecStart=%s DependencyDir: destDirs, Tags: []string{"external"}, - AdditionalDisks: targetMeta.AdditionalDisks, - PrimaryDisk: targetMeta.PrimaryDisk, - InjectContainer: targetMeta.InjectContainer, - MinMemory: targetMeta.MinMemory, - NumaNodes: targetMeta.NumaNodes, - MinDiskSize: targetMeta.MinDiskSize, - AdditionalNics: targetMeta.AdditionalNics, - AppendKernelArgs: targetMeta.AppendKernelArgs, - AppendFirstbootKernelArgs: targetMeta.AppendFirstbootKernelArgs, - InstanceType: targetMeta.InstanceType, - NonExclusive: !targetMeta.Exclusive, - Conflicts: targetMeta.Conflicts, + MachineOptions: platform.MachineOptions{ + AdditionalDisks: targetMeta.AdditionalDisks, + PrimaryDisk: targetMeta.PrimaryDisk, + MinMemory: targetMeta.MinMemory, + NumaNodes: targetMeta.NumaNodes, + MinDiskSize: targetMeta.MinDiskSize, + AdditionalNics: targetMeta.AdditionalNics, + AppendKernelArgs: targetMeta.AppendKernelArgs, + AppendFirstbootKernelArgs: targetMeta.AppendFirstbootKernelArgs, + InstanceType: targetMeta.InstanceType, + }, + InjectContainer: targetMeta.InjectContainer, + NonExclusive: !targetMeta.Exclusive, + Conflicts: targetMeta.Conflicts, Run: func(c cluster.TestCluster) { mach := c.Machines()[0] @@ -1620,7 +1622,7 @@ func makeNonExclusiveTest(bucket int, tests []*register.Test, flight platform.Fl if test.HasFlag(register.AllowConfigWarnings) { plog.Fatalf("Non-exclusive test %v cannot have AllowConfigWarnings flag", test.Name) } - if test.AppendKernelArgs != "" { + if test.MachineOptions.AppendKernelArgs != "" { plog.Fatalf("Non-exclusive test %v cannot have AppendKernelArgs", test.Name) } if !internetAccess && testRequiresInternet(test) { @@ -1759,11 +1761,11 @@ func waitForMemory(h *harness.H, flight platform.Flight, t *register.Test) { // pool. This should be called after the test's QEMU VM has been started and // the memory allocated for the VM. func releaseMemoryCount(flight platform.Flight, t *register.Test) { + reservedMemoryCountMutex.Lock() + defer reservedMemoryCountMutex.Unlock() if t.ReservedMemoryCountMiB == 0 { return // memory count already released } - reservedMemoryCountMutex.Lock() - defer reservedMemoryCountMutex.Unlock() reservedMemoryCountMiB -= t.ReservedMemoryCountMiB t.ReservedMemoryCountMiB = 0 if reservedMemoryCountMiB < 0 { @@ -1782,8 +1784,8 @@ func getNeededMemoryMiB(t *register.Test) int { } } // If the test specifies MinMemory, use that. - if t.MinMemory != 0 { - return t.MinMemory + if t.MachineOptions.MinMemory != 0 { + return t.MachineOptions.MinMemory } // Fall back to architecture-specific defaults from the QEMU platform. return platform.DefaultMemoryMiB(Options.CosaBuildArch) @@ -1811,6 +1813,7 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig SSHOnTestFailure: Options.SSHOnTestFailure, WarningsAction: conf.FailWarnings, EarlyRelease: h.Release, + TestExecTimeout: h.TimeoutContext(), } if t.HasFlag(register.AllowConfigWarnings) { rconf.WarningsAction = conf.IgnoreWarnings @@ -1823,6 +1826,8 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig } defer func() { h.StopExecTimer() + // give some time for the remote journal to be flushed before we Destroy() + time.Sleep(2 * time.Second) c.Destroy() // Release the memory reservation (if there was one) now that the VM is gone. releaseMemoryCount(flight, t) @@ -1858,19 +1863,7 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig if t.ClusterSize > 0 { var userdata *conf.UserData = t.UserData - options := platform.MachineOptions{ - MultiPathDisk: t.MultiPathDisk, - PrimaryDisk: t.PrimaryDisk, - AdditionalDisks: t.AdditionalDisks, - MinMemory: t.MinMemory, - MinDiskSize: t.MinDiskSize, - NumaNodes: t.NumaNodes, - AdditionalNics: t.AdditionalNics, - AppendKernelArgs: t.AppendKernelArgs, - AppendFirstbootKernelArgs: t.AppendFirstbootKernelArgs, - SkipStartMachine: true, - InstanceType: t.InstanceType, - } + options := t.MachineOptions if testSecureBoot(t) { options.Firmware = "uefi-secure" @@ -1913,32 +1906,19 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig tcluster.H.WarningOnFailure() } - // Note that we passed in SkipStartMachine=true in our machine - // options. This means NewMachines() didn't block on the machines - // being up with SSH access before returning; i.e. it skipped running - // platform.StartMachines(). The machines should now be booting. - // Let's start the test execution timer and then run mach.Start() - // (wrapper for platform.StartMachine()) which sets up the journal - // forwarding and runs machine checks, both of which require SSH - // to be up, which implies Ignition has completed successfully. - // - // We do all of this so that the time it takes to run Ignition can - // be included in our test execution timeout. - h.StartExecTimer() - for _, mach := range tcluster.Machines() { - plog.Debugf("Trying to StartMachine() %v", mach.ID()) - var err error - tcluster.RunWithExecTimeoutCheck(func() { - err = mach.Start() - }, fmt.Sprintf("SSH unsuccessful within allotted timeframe for %v.", mach.ID())) - if err != nil { - h.Fatal(errors.Wrapf(err, "mach.Start() failed")) + // Machines may be created directly by the test (not via the + // harness with NewMachines above), so we poll asynchronously + // via a goroutine until at least one shows up and then release + // the temporary memory reservation. + go func() { + // Wait for at least one machine in the cluster to exist. + // At the point machines show up in tcluster.Machines() they've + // already been contacted successfully via SSH in StartMachine(). + for len(tcluster.Machines()) == 0 { + time.Sleep(1 * time.Second) } - } - - // Release the temporary memory reservation now that the VM is up and should - // be using it's allotted memory (preallocation). Applicable on qemu only. - releaseMemoryCount(flight, t) + releaseMemoryCount(flight, t) + }() // drop kolet binary on machines if t.ExternalTest != "" || t.NativeFuncs != nil { @@ -1999,12 +1979,6 @@ func runTest(h *harness.H, t *register.Test, pltfrm string, flight platform.Flig } } - defer func() { - // give some time for the remote journal to be flushed so it can be read - // before we run the deferred machine destruction - time.Sleep(2 * time.Second) - }() - // run test t.Run(tcluster) } diff --git a/mantle/kola/register/register.go b/mantle/kola/register/register.go index ccb5d6ce26..6c5ade0c52 100644 --- a/mantle/kola/register/register.go +++ b/mantle/kola/register/register.go @@ -19,6 +19,7 @@ import ( "time" "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" ) @@ -70,45 +71,20 @@ type Test struct { Timeout time.Duration // the duration for which a test will be allowed to run RequiredTag string // if specified, test is filtered by default unless tag is provided -- defaults to none Description string // test description - NumaNodes bool // simulate two NUMA nodes - // Whether the primary disk is multipathed. Deprecated in favour of PrimaryDisk. - MultiPathDisk bool - - // Sizes of additional empty disks to attach to the node, followed by - // comma-separated list of optional options (e.g. ["1G", - // "5G:mpath,foo,bar"]) -- defaults to none. - AdditionalDisks []string - - // Size of primary disk to attach to the node, followed by - // comma-separated list of optional options (e.g. "20G:mpath"]). - PrimaryDisk string + // MachineOptions contains options for machine creation (disks, memory, + // kernel args, etc.). The test harness passes these to + // NewMachineWithOptions when ClusterSize > 0. + MachineOptions platform.MachineOptions // InjectContainer will cause the ostree base image to be injected into the target InjectContainer bool - // Minimum amount of memory in MB required for test. - MinMemory int - // The artificially reserved memory count in MiB for the test. This is used // for budgeting memory usage for tests prior to the VMs starting up on the // QEMU platform. ReservedMemoryCountMiB int - // Minimum amount of primary disk in GB required for test. Deprecated in favour - // of PrimaryDisk. - MinDiskSize int - - // Additional amount of NICs required for test. - AdditionalNics int - - // Additional kernel arguments to append to the defaults. - AppendKernelArgs string - - // Additional first boot kernel arguments to append to the defaults. - AppendFirstbootKernelArgs string - - // ExternalTest is a path to a binary that will be uploaded ExternalTest string // DependencyDir is a path to directory that will be uploaded, normally used by external tests DependencyDir DepDirMap @@ -124,10 +100,6 @@ type Test struct { // Conflicts is non-empty iff nonexclusive is true // Contains the tests that conflict with this particular test Conflicts []string - - // If provided, this test will be run on the target instance type. - // This overrides the instance type set with `kola run` - InstanceType string } // Registered tests that run as part of `kola run` live here. Mapping of names diff --git a/mantle/kola/registry/registry.go b/mantle/kola/registry/registry.go index c22c474e86..13a3b3d62d 100644 --- a/mantle/kola/registry/registry.go +++ b/mantle/kola/registry/registry.go @@ -7,6 +7,7 @@ import ( _ "github.com/coreos/coreos-assembler/mantle/kola/tests/etcd" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/fips" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ignition" + _ "github.com/coreos/coreos-assembler/mantle/kola/tests/install-media" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/metadata" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/misc" _ "github.com/coreos/coreos-assembler/mantle/kola/tests/ostree" diff --git a/mantle/kola/tests/coretest/core.go b/mantle/kola/tests/coretest/core.go index 4e27a89d10..2d8a3592cf 100644 --- a/mantle/kola/tests/coretest/core.go +++ b/mantle/kola/tests/coretest/core.go @@ -11,11 +11,8 @@ import ( "github.com/pborman/uuid" - "github.com/coreos/coreos-assembler/mantle/kola" - "github.com/coreos/coreos-assembler/mantle/kola/cluster" "github.com/coreos/coreos-assembler/mantle/kola/register" "github.com/coreos/coreos-assembler/mantle/platform" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" ) const ( @@ -64,31 +61,40 @@ func init() { register.RegisterTest(®ister.Test{ Name: "basic.uefi", Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with UEFI enabled", - Run: uefiWithBasicTests, + Run: LocalTests, Platforms: []string{"qemu"}, - ClusterSize: 0, + ClusterSize: 1, NativeFuncs: nativeFuncs, Architectures: []string{"x86_64", "aarch64"}, + MachineOptions: platform.MachineOptions{ + Firmware: uefi, + }, }) register.RegisterTest(®ister.Test{ Name: "basic.uefi-secure", Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with UEFI Secure Boot enabled", - Run: uefiSecureWithBasicTests, + Run: LocalTests, Platforms: []string{"qemu"}, - ClusterSize: 0, + ClusterSize: 1, NativeFuncs: nativeFuncs, Architectures: []string{"x86_64"}, + MachineOptions: platform.MachineOptions{ + Firmware: uefiSecure, + }, }) register.RegisterTest(®ister.Test{ Name: "basic.nvme", Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with nvme enabled", - Run: nvmeBasicTests, + Run: LocalTests, Platforms: []string{"qemu"}, - ClusterSize: 0, + ClusterSize: 1, NativeFuncs: nativeFuncs, // NVMe in theory is supported on all arches, but the way we test it seems to // only work on x86_64 and aarch64. Architectures: []string{"x86_64", "aarch64"}, + MachineOptions: platform.MachineOptions{ + Nvme: true, + }, }) register.RegisterTest(®ister.Test{ Name: "rootfs.uuid", @@ -113,47 +119,6 @@ func init() { }) } -func uefiWithBasicTests(c cluster.TestCluster) { - runBasicTests(c, uefi, false) -} - -func uefiSecureWithBasicTests(c cluster.TestCluster) { - runBasicTests(c, uefiSecure, false) -} - -func nvmeBasicTests(c cluster.TestCluster) { - runBasicTests(c, "", true) -} - -func runBasicTests(c cluster.TestCluster, firmware string, nvme bool) { - var err error - var m platform.Machine - - options := platform.QemuMachineOptions{ - Firmware: firmware, - Nvme: nvme, - } - switch pc := c.Cluster.(type) { - // These cases have to be separated because when put together to the same case statement - // the golang compiler no longer checks that the individual types in the case have the - // NewMachineWithQemuOptions function, but rather whether platform.Cluster - // does which fails - case *qemu.Cluster: - m, err = pc.NewMachineWithQemuOptions(nil, options) - default: - panic("Unsupported cluster type") - } - if err != nil { - c.Fatal(err) - } - - // copy over kolet into the machine - if err := kola.ScpKolet([]platform.Machine{m}); err != nil { - c.Fatal(err) - } - LocalTests(c) -} - func TestPortSsh() error { //t.Parallel() err := CheckPort("tcp", "127.0.0.1:22", PortTimeout) diff --git a/mantle/kola/tests/ignition/kdump.go b/mantle/kola/tests/ignition/kdump.go index 2429504ced..f58352ee11 100644 --- a/mantle/kola/tests/ignition/kdump.go +++ b/mantle/kola/tests/ignition/kdump.go @@ -11,7 +11,6 @@ import ( "github.com/coreos/coreos-assembler/mantle/kola/register" "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" "github.com/coreos/coreos-assembler/mantle/util" ) @@ -106,7 +105,7 @@ func setupSSHMachine(c cluster.TestCluster) SshServer { var address string var port string - options := platform.QemuMachineOptions{ + options := platform.MachineOptions{ HostForwardPorts: []platform.HostForwardPort{ {Service: "ssh", HostPort: 0, GuestPort: 22}, }, @@ -146,16 +145,7 @@ func setupSSHMachine(c cluster.TestCluster) SshServer { }`, strings.TrimSpace(string(pubkeyBuf)))) // start the machine - switch c := c.Cluster.(type) { - // These cases have to be separated because when put together to the same case statement - // the golang compiler no longer checks that the individual types in the case have the - // NewMachineWithQemuOptions function, but rather whether platform.Cluster - // does which fails - case *qemu.Cluster: - m, err = c.NewMachineWithQemuOptions(ignition, options) - default: - panic("unreachable") - } + m, err = c.Cluster.NewMachineWithOptions(ignition, options) if err != nil { c.Fatal(err) } @@ -253,7 +243,7 @@ func setupNFSMachine(c cluster.TestCluster) NfsServer { var m platform.Machine var err error - options := platform.QemuMachineOptions{ + options := platform.MachineOptions{ HostForwardPorts: []platform.HostForwardPort{ {Service: "ssh", HostPort: 0, GuestPort: 22}, // Kdump NFS option does not allow a custom port @@ -280,16 +270,7 @@ storage: - path: /var/nfs/crash`) // start the machine - switch c := c.Cluster.(type) { - // These cases have to be separated because when put together to the same case statement - // the golang compiler no longer checks that the individual types in the case have the - // NewMachineWithQemuOptions function, but rather whether platform.Cluster - // does which fails - case *qemu.Cluster: - m, err = c.NewMachineWithQemuOptions(nfs_server_butane, options) - default: - panic("unreachable") - } + m, err = c.Cluster.NewMachineWithOptions(nfs_server_butane, options) if err != nil { c.Fatal(err) } diff --git a/mantle/kola/tests/ignition/luks.go b/mantle/kola/tests/ignition/luks.go index 2efb066678..d601e7f1b0 100644 --- a/mantle/kola/tests/ignition/luks.go +++ b/mantle/kola/tests/ignition/luks.go @@ -74,7 +74,7 @@ func setupTangMachine(c cluster.TestCluster) ut.TangServer { var thumbprint []byte var tangAddress string - options := platform.QemuMachineOptions{ + options := platform.MachineOptions{ HostForwardPorts: []platform.HostForwardPort{ {Service: "ssh", HostPort: 0, GuestPort: 22}, {Service: "tang", HostPort: 0, GuestPort: 80}, @@ -87,20 +87,16 @@ func setupTangMachine(c cluster.TestCluster) ut.TangServer { } }`) - switch pc := c.Cluster.(type) { - // These cases have to be separated because when put together to the same case statement - // the golang compiler no longer checks that the individual types in the case have the - // NewMachineWithQemuOptions function, but rather whether platform.Cluster - // does which fails + switch c.Cluster.(type) { case *qemu.Cluster: - m, err = pc.NewMachineWithQemuOptions(ignition, options) + m, err = c.Cluster.NewMachineWithOptions(ignition, options) for _, hfp := range options.HostForwardPorts { if hfp.Service == "tang" { tangAddress = fmt.Sprintf("10.0.2.2:%d", hfp.HostPort) } } default: - m, err = pc.NewMachine(ignition) + m, err = c.Cluster.NewMachine(ignition) tangAddress = fmt.Sprintf("%s:80", m.PrivateIP()) } if err != nil { @@ -234,19 +230,14 @@ func runCexTest(c cluster.TestCluster) { } }`) - opts := platform.QemuMachineOptions{ - Cex: true, + opts := platform.MachineOptions{ + Cex: true, + MinMemory: 8192, } - opts.MinMemory = 8192 - switch pc := c.Cluster.(type) { - case *qemu.Cluster: - m, err = pc.NewMachineWithQemuOptions(ignition, opts) - if err != nil { - c.Fatalf("Unable to create test machine: %v", err) - } - default: - panic("Unsupported cluster type") + m, err = c.Cluster.NewMachineWithOptions(ignition, opts) + if err != nil { + c.Fatalf("Unable to create test machine: %v", err) } // copy over kolet into the machine diff --git a/mantle/cmd/kola/resources/iscsi_butane_setup.yaml b/mantle/kola/tests/install-media/resources/iscsi_butane_setup.yaml similarity index 100% rename from mantle/cmd/kola/resources/iscsi_butane_setup.yaml rename to mantle/kola/tests/install-media/resources/iscsi_butane_setup.yaml diff --git a/mantle/cmd/kola/testiso.go b/mantle/kola/tests/install-media/test-install-media.go similarity index 81% rename from mantle/cmd/kola/testiso.go rename to mantle/kola/tests/install-media/test-install-media.go index 199091e26b..74d2d0f3a7 100644 --- a/mantle/cmd/kola/testiso.go +++ b/mantle/kola/tests/install-media/test-install-media.go @@ -1,23 +1,4 @@ -// Copyright 2020 Red Hat, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// TODO: -// - Support testing the "just run Live" case - maybe try to figure out -// how to have main `kola` tests apply? -// - Test `coreos-install iso embed` path - -package main +package testinstall import ( "bufio" @@ -31,30 +12,25 @@ import ( "strings" "time" - "github.com/coreos/coreos-assembler/mantle/harness" + "github.com/coreos/pkg/capnslog" + + ////"github.com/coreos/coreos-assembler/mantle/harness" "github.com/coreos/coreos-assembler/mantle/harness/reporters" - "github.com/coreos/coreos-assembler/mantle/harness/testresult" + ////"github.com/coreos/coreos-assembler/mantle/harness/testresult" "github.com/coreos/coreos-assembler/mantle/platform/conf" "github.com/coreos/coreos-assembler/mantle/util" coreosarch "github.com/coreos/stream-metadata-go/arch" "github.com/pkg/errors" - "github.com/spf13/cobra" + ////"github.com/spf13/cobra" "github.com/coreos/coreos-assembler/mantle/kola" + "github.com/coreos/coreos-assembler/mantle/kola/cluster" + "github.com/coreos/coreos-assembler/mantle/kola/register" "github.com/coreos/coreos-assembler/mantle/platform" ) var ( - cmdTestIso = &cobra.Command{ - RunE: runTestIso, - PreRunE: preRun, - Use: "testiso [glob pattern...]", - Short: "Test a CoreOS PXE boot or ISO install path", - - SilenceUsage: true, - } - instInsecure bool pxeKernelArgs []string @@ -151,6 +127,7 @@ var ( //"iso-offline-install-iscsi.ibft-with-mpath.uefi", //"iso-offline-install-iscsi.manual.uefi", } + plog = capnslog.NewPackageLogger("github.com/coreos/coreos-assembler/mantle", "kola/tests/install") ) const ( @@ -354,11 +331,17 @@ RequiredBy=multi-user.target`, nmConnectionId, nmConnectionFile) var iscsi_butane_config string func init() { - cmdTestIso.Flags().BoolVarP(&instInsecure, "inst-insecure", "S", false, "Do not verify signature on metal image") - cmdTestIso.Flags().BoolVar(&console, "console", false, "Connect qemu console to terminal, turn off automatic initramfs failure checking") - cmdTestIso.Flags().StringSliceVar(&pxeKernelArgs, "pxe-kargs", nil, "Additional kernel arguments for PXE") - - root.AddCommand(cmdTestIso) + for _, test := range getAllTests(kola.CosaBuild) { + register.RegisterTest(®ister.Test{ + Name: "install-media." + test, + Description: "The %s install test", + Run: runTestInstall, + ClusterSize: 0, + Tags: []string{"reprovision"}, + Platforms: []string{"qemu"}, + Timeout: installTimeoutMins * time.Minute, + }) + } } func liveArtifactExistsInBuild() error { @@ -382,9 +365,10 @@ func getAllTests(build *util.LocalBuild) []string { case "aarch64": tests = tests_aarch64 } - if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { - tests = append(tests, tests_RHCOS_uefi...) - } + //// XXX FIX + ////if kola.CosaBuild.Meta.Name == "rhcos" && arch != "s390x" && arch != "ppc64le" { + //// tests = append(tests, tests_RHCOS_uefi...) + ////} return tests } @@ -488,68 +472,34 @@ func newQemuBuilderWithDisk(outdir string) (*platform.QemuBuilder, *conf.Conf, e return builder, config, nil } -// See similar semantics in the `filterTests` of `kola.go`. -func filterTests(tests []string, patterns []string) ([]string, error) { - r := []string{} - for _, test := range tests { - if matches, err := kola.MatchesPatterns(test, patterns); err != nil { - return nil, err - } else if matches { - r = append(r, test) - } - } - return r, nil -} +func runTestInstall(c cluster.TestCluster) { + var err error + var outputDir string + + // Grab the test name from cluster.TestCluster + test := c.H.Name() -func runTestIso(cmd *cobra.Command, args []string) (err error) { if kola.CosaBuild == nil { - return fmt.Errorf("Must provide --build") - } - tests := getAllTests(kola.CosaBuild) - if len(args) != 0 { - if tests, err = filterTests(tests, args); err != nil { - return err - } else if len(tests) == 0 { - return harness.SuiteEmpty - } + c.Fatal(fmt.Errorf("Must provide --build")) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // Call `ParseDenyListYaml` to populate the `kola.DenylistedTests` var - err = kola.ParseDenyListYaml("qemu") - if err != nil { - plog.Fatal(err) - } - - finalTests := []string{} - for _, test := range tests { - if !kola.HasString(test, kola.DenylistedTests) { - matchTest, err := kola.MatchesPatterns(test, kola.DenylistedTests) - if err != nil { - return err - - } - if !matchTest { - finalTests = append(finalTests, test) - } - } - } - - // note this reassigns a *global* - outputDir, err = kola.SetupOutputDir(outputDir, "testiso") + // XXX Previously had set up an outputdir but couldn't access + // the varible here so passing in "" as first argument. + outputDir, err = kola.SetupOutputDir("", "testinstall") if err != nil { - return err + c.Fatal(err) } // see similar code in suite.go reportDir := filepath.Join(outputDir, "reports") - if err := os.Mkdir(reportDir, 0777); err != nil { - return err + if err = os.Mkdir(reportDir, 0777); err != nil { + c.Fatal(err) } - reporter := reporters.NewJSONReporter("report.json", "testiso", "") + reporter := reporters.NewJSONReporter("report.json", "testinstall", "") defer func() { if reportErr := reporter.Output(reportDir); reportErr != nil && err != nil { err = reportErr @@ -561,120 +511,101 @@ func runTestIso(cmd *cobra.Command, args []string) (err error) { NmKeyfiles: make(map[string]string), } - if instInsecure { - baseInst.Insecure = true - fmt.Printf("Ignoring verification of signature on metal image\n") - } + // TODO Fix later + ////if instInsecure { + //// baseInst.Insecure = true + //// fmt.Printf("Ignoring verification of signature on metal image\n") + ////} // Ignore signing verification by default when running with development build // https://github.com/coreos/fedora-coreos-tracker/issues/908 - if !baseInst.Insecure && strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { - baseInst.Insecure = true - fmt.Printf("Detected development build; disabling signature verification\n") - } - - var duration time.Duration - - atLeastOneFailed := false - for _, test := range finalTests { - - // All of these tests require buildextend-live to have been run - err = liveArtifactExistsInBuild() - if err != nil { - return err - } - - addNmKeyfile = false - enable4k = false - enableMultipath = false - enableUefi = false - enableUefiSecure = false - isOffline = false - inst := baseInst // Pretend this is Rust and I wrote .copy() - - fmt.Printf("Running test: %s\n", test) - components := strings.Split(test, ".") - - inst.PxeAppendRootfs = kola.HasString("rootfs-appended", components) - - if kola.HasString("4k", components) { - enable4k = true - inst.Native4k = true - } - if kola.HasString("nm", components) { - addNmKeyfile = true - } - if kola.HasString("mpath", components) { - enableMultipath = true - inst.MultiPathDisk = true - } - if kola.HasString("uefi-secure", components) { - enableUefiSecure = true - } else if kola.HasString("uefi", components) { - enableUefi = true - } - // For offline it is a part of the first component. i.e. for - // iso-offline-install.bios we need to search for 'offline' in - // iso-offline-install, which is currently in components[0]. - if kola.HasString("offline", strings.Split(components[0], "-")) { - isOffline = true - } - // For fromram it is a part of the first component. i.e. for - // iso-offline-install-fromram.uefi we need to search for 'fromram' in - // iso-offline-install-fromram, which is currently in components[0]. - if kola.HasString("fromram", strings.Split(components[0], "-")) { - isISOFromRAM = true - } - - switch components[0] { - case "pxe-offline-install", "pxe-online-install": - duration, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) - case "iso-as-disk": - duration, err = testAsDisk(ctx, filepath.Join(outputDir, test)) - case "iso-live-login": - duration, err = testLiveLogin(ctx, filepath.Join(outputDir, test)) - case "iso-fips": - duration, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) - case "iso-install", "iso-offline-install", "iso-offline-install-fromram": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) - case "miniso-install": - duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) - case "iso-offline-install-iscsi": - var butane_config string - switch components[1] { - case "ibft": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") - case "manual": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") - case "ibft-with-mpath": - butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") - default: - plog.Fatalf("Unknown test name:%s", test) - } - duration, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test), butane_config) + ////if !baseInst.Insecure && strings.Contains(kola.CosaBuild.Meta.BuildID, ".dev.") { + //// baseInst.Insecure = true + //// fmt.Printf("Detected development build; disabling signature verification\n") + ////} + // XXX for now just set this to true + baseInst.Insecure = true + + // Verify Live Artifacts exist in build + err = liveArtifactExistsInBuild() + if err != nil { + c.Fatal(err) + } + + addNmKeyfile = false + enable4k = false + enableMultipath = false + enableUefi = false + enableUefiSecure = false + isOffline = false + inst := baseInst // Pretend this is Rust and I wrote .copy() + + components := strings.Split(test, ".") + + inst.PxeAppendRootfs = kola.HasString("rootfs-appended", components) + + if kola.HasString("4k", components) { + enable4k = true + inst.Native4k = true + } + if kola.HasString("nm", components) { + addNmKeyfile = true + } + if kola.HasString("mpath", components) { + enableMultipath = true + inst.MultiPathDisk = true + } + if kola.HasString("uefi-secure", components) { + enableUefiSecure = true + } else if kola.HasString("uefi", components) { + enableUefi = true + } + // For offline it is a part of the first component. i.e. for + // iso-offline-install.bios we need to search for 'offline' in + // iso-offline-install, which is currently in components[1]. + if kola.HasString("offline", strings.Split(components[1], "-")) { + isOffline = true + } + // For fromram it is a part of the first component. i.e. for + // iso-offline-install-fromram.uefi we need to search for 'fromram' in + // iso-offline-install-fromram, which is currently in components[1]. + if kola.HasString("fromram", strings.Split(components[1], "-")) { + isISOFromRAM = true + } + + switch components[1] { + case "pxe-offline-install", "pxe-online-install": + _, err = testPXE(ctx, inst, filepath.Join(outputDir, test)) + case "iso-as-disk": + _, err = testAsDisk(ctx, filepath.Join(outputDir, test)) + case "iso-live-login": + _, err = testLiveLogin(ctx, filepath.Join(outputDir, test)) + case "iso-fips": + _, err = testLiveFIPS(ctx, filepath.Join(outputDir, test)) + case "iso-install", "iso-offline-install", "iso-offline-install-fromram": + _, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false) + case "miniso-install": + _, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true) + case "iso-offline-install-iscsi": + var butane_config string + switch components[2] { + case "ibft": + butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1") + case "manual": + butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg netroot=iscsi:10.0.2.15::::iqn.2024-05.com.coreos:0") + case "ibft-with-mpath": + butane_config = strings.ReplaceAll(iscsi_butane_config, "COREOS_INSTALLER_KARGS", "--append-karg rd.iscsi.firmware=1 --append-karg rd.multipath=default --append-karg root=/dev/disk/by-label/dm-mpath-root --append-karg rw") default: - plog.Fatalf("Unknown test name:%s", test) - } - - result := testresult.Pass - output := []byte{} - if err != nil { - result = testresult.Fail - output = []byte(err.Error()) - } - reporter.ReportTest(test, []string{}, result, duration, output) - if printResult(test, duration, err) { - atLeastOneFailed = true + c.Fatalf("Unknown test name:%s", test) } + _, err = testLiveInstalliscsi(ctx, inst, filepath.Join(outputDir, test), butane_config) + default: + c.Fatalf("Unknown test:%s", test) } - reporter.SetResult(testresult.Pass) - if atLeastOneFailed { - reporter.SetResult(testresult.Fail) - return harness.SuiteFailed + if err != nil { + c.Fatalf("Test %s failed with err: %v", test, err) } - - return nil } func awaitCompletion(ctx context.Context, inst *platform.QemuInstance, outdir string, qchan *os.File, booterrchan chan error, expected []string) (time.Duration, error) { diff --git a/mantle/kola/tests/misc/boot-mirror.go b/mantle/kola/tests/misc/boot-mirror.go index e246ac3714..38a366ddd7 100644 --- a/mantle/kola/tests/misc/boot-mirror.go +++ b/mantle/kola/tests/misc/boot-mirror.go @@ -26,7 +26,6 @@ import ( "github.com/coreos/coreos-assembler/mantle/kola/tests/util" "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" ut "github.com/coreos/coreos-assembler/mantle/util" ) @@ -94,11 +93,9 @@ func init() { func runBootMirrorTest(c cluster.TestCluster) { var m platform.Machine var err error - options := platform.QemuMachineOptions{ - MachineOptions: platform.MachineOptions{ - AdditionalDisks: []string{"5G", "5G"}, - MinMemory: 4096, - }, + options := platform.MachineOptions{ + AdditionalDisks: []string{"5G", "5G"}, + MinMemory: 4096, } // ppc64le uses 64K pages; see similar logic in harness.go and luks.go switch coreosarch.CurrentRpmArch() { @@ -108,7 +105,7 @@ func runBootMirrorTest(c cluster.TestCluster) { // FIXME: for QEMU tests kola currently assumes the host CPU architecture // matches the one under test userdata := bootmirror.Subst("LAYOUT", coreosarch.CurrentRpmArch()) - m, err = c.Cluster.(*qemu.Cluster).NewMachineWithQemuOptions(userdata, options) + m, err = c.Cluster.NewMachineWithOptions(userdata, options) if err != nil { c.Fatal(err) } @@ -146,11 +143,9 @@ func runBootMirrorTest(c cluster.TestCluster) { func runBootMirrorLUKSTest(c cluster.TestCluster) { var m platform.Machine var err error - options := platform.QemuMachineOptions{ - MachineOptions: platform.MachineOptions{ - AdditionalDisks: []string{"5G"}, - MinMemory: 4096, - }, + options := platform.MachineOptions{ + AdditionalDisks: []string{"5G"}, + MinMemory: 4096, } // ppc64le uses 64K pages; see similar logic in harness.go and luks.go switch coreosarch.CurrentRpmArch() { @@ -160,7 +155,7 @@ func runBootMirrorLUKSTest(c cluster.TestCluster) { // FIXME: for QEMU tests kola currently assumes the host CPU architecture // matches the one under test userdata := bootmirrorluks.Subst("LAYOUT", coreosarch.CurrentRpmArch()) - m, err = c.Cluster.(*qemu.Cluster).NewMachineWithQemuOptions(userdata, options) + m, err = c.Cluster.NewMachineWithOptions(userdata, options) if err != nil { c.Fatal(err) } diff --git a/mantle/kola/tests/misc/multipath.go b/mantle/kola/tests/misc/multipath.go index 088837e59a..cc643078c4 100644 --- a/mantle/kola/tests/misc/multipath.go +++ b/mantle/kola/tests/misc/multipath.go @@ -115,40 +115,48 @@ kernel_arguments: func init() { register.RegisterTest(®ister.Test{ - Name: "multipath.day1", - Description: "Verify that multipath can be configured day 1 through Ignition.", - Run: runMultipathDay1, - ClusterSize: 1, - Platforms: []string{"qemu"}, - UserData: mpath_on_boot_day1, - MultiPathDisk: true, + Name: "multipath.day1", + Description: "Verify that multipath can be configured day 1 through Ignition.", + Run: runMultipathDay1, + ClusterSize: 1, + Platforms: []string{"qemu"}, + UserData: mpath_on_boot_day1, + MachineOptions: platform.MachineOptions{ + MultiPathDisk: true, + }, }) register.RegisterTest(®ister.Test{ - Name: "multipath.day2", - Description: "Verify that multipath can be configured day 2 through Ignition.", - Run: runMultipathDay2, - ClusterSize: 1, - Platforms: []string{"qemu"}, - MultiPathDisk: true, + Name: "multipath.day2", + Description: "Verify that multipath can be configured day 2 through Ignition.", + Run: runMultipathDay2, + ClusterSize: 1, + Platforms: []string{"qemu"}, + MachineOptions: platform.MachineOptions{ + MultiPathDisk: true, + }, }) register.RegisterTest(®ister.Test{ - Name: "multipath.partition", - Description: "Verify that multipath can be configured for a partition.", - Run: runMultipathPartition, - ClusterSize: 1, - Platforms: []string{"qemu"}, - UserData: mpath_on_var_lib_containers, - AdditionalDisks: []string{"1G:mpath,wwn=1"}, + Name: "multipath.partition", + Description: "Verify that multipath can be configured for a partition.", + Run: runMultipathPartition, + ClusterSize: 1, + Platforms: []string{"qemu"}, + UserData: mpath_on_var_lib_containers, + MachineOptions: platform.MachineOptions{ + AdditionalDisks: []string{"1G:mpath,wwn=1"}, + }, }) // See https://issues.redhat.com/browse/OCPBUGS-56597 register.RegisterTest(®ister.Test{ - Name: "multipath.single-disk", - Description: "Verify that multipath can be reduced to one path", - Run: runMultipathReduceDisk, - ClusterSize: 1, - Platforms: []string{"qemu"}, - UserData: mpath_single_disk, - MultiPathDisk: true, + Name: "multipath.single-disk", + Description: "Verify that multipath can be reduced to one path", + Run: runMultipathReduceDisk, + ClusterSize: 1, + Platforms: []string{"qemu"}, + UserData: mpath_single_disk, + MachineOptions: platform.MachineOptions{ + MultiPathDisk: true, + }, }) } diff --git a/mantle/kola/tests/misc/network.go b/mantle/kola/tests/misc/network.go index cee6b8a57a..f073f0a19b 100644 --- a/mantle/kola/tests/misc/network.go +++ b/mantle/kola/tests/misc/network.go @@ -27,7 +27,6 @@ import ( "github.com/coreos/coreos-assembler/mantle/kola/register" "github.com/coreos/coreos-assembler/mantle/platform" "github.com/coreos/coreos-assembler/mantle/platform/conf" - "github.com/coreos/coreos-assembler/mantle/platform/machine/qemu" "github.com/coreos/coreos-assembler/mantle/util" ) @@ -68,17 +67,19 @@ func init() { kargs = "net.ifnames=0" } register.RegisterTest(®ister.Test{ - Run: InitInterfacesTest, - ClusterSize: 1, - Name: "rhcos.network.init-interfaces-test", - Description: "Verify init-interfaces script works in both fresh setup and reboot.", - Timeout: 40 * time.Minute, - Distros: []string{"rhcos"}, - Platforms: []string{"qemu"}, - RequiredTag: "openshift", - AdditionalNics: 2, - AppendKernelArgs: kargs, - UserData: userdata, + Run: InitInterfacesTest, + ClusterSize: 1, + Name: "rhcos.network.init-interfaces-test", + Description: "Verify init-interfaces script works in both fresh setup and reboot.", + Timeout: 40 * time.Minute, + Distros: []string{"rhcos"}, + Platforms: []string{"qemu"}, + RequiredTag: "openshift", + MachineOptions: platform.MachineOptions{ + AdditionalNics: 2, + AppendKernelArgs: kargs, + }, + UserData: userdata, }) } @@ -505,10 +506,8 @@ func setupMultipleNetworkTest(c cluster.TestCluster, primaryMac, secondaryMac st var m platform.Machine var err error - options := platform.QemuMachineOptions{ - MachineOptions: platform.MachineOptions{ - AdditionalNics: 2, - }, + options := platform.MachineOptions{ + AdditionalNics: 2, } // On s390x, multiple NICs are ordered by the CCW device number. Use classic ethX names to ensure consistent and ordered naming. if runtime.GOARCH == "s390x" { @@ -539,16 +538,7 @@ func setupMultipleNetworkTest(c cluster.TestCluster, primaryMac, secondaryMac st } }`, base64.StdEncoding.EncodeToString([]byte(captureMacsScript)))) - switch pc := c.Cluster.(type) { - // These cases have to be separated because when put together to the same case statement - // the golang compiler no longer checks that the individual types in the case have the - // NewMachineWithQemuOptions function, but rather whether platform.Cluster - // does which fails - case *qemu.Cluster: - m, err = pc.NewMachineWithQemuOptions(userdata, options) - default: - panic("unreachable") - } + m, err = c.Cluster.NewMachineWithOptions(userdata, options) if err != nil { c.Fatal(err) } diff --git a/mantle/kola/tests/ostree/sync.go b/mantle/kola/tests/ostree/sync.go index 9253e08209..085afb9fa6 100644 --- a/mantle/kola/tests/ostree/sync.go +++ b/mantle/kola/tests/ostree/sync.go @@ -96,24 +96,20 @@ func setupNFSMachine(c cluster.TestCluster) NfsServer { var err error var nfs_server string - options := platform.QemuMachineOptions{ + options := platform.MachineOptions{ HostForwardPorts: []platform.HostForwardPort{ {Service: "ssh", HostPort: 0, GuestPort: 22}, {Service: "nfs", HostPort: 2049, GuestPort: 2049}, }, + MinMemory: 2048, } - options.MinMemory = 2048 // start the machine - switch c := c.Cluster.(type) { - // These cases have to be separated because when put together to the same case statement - // the golang compiler no longer checks that the individual types in the case have the - // NewMachineWithQemuOptions function, but rather whether platform.Cluster - // does which fails + switch c.Cluster.(type) { case *qemu.Cluster: - m, err = c.NewMachineWithQemuOptions(nfs_server_butane, options) + m, err = c.Cluster.NewMachineWithOptions(nfs_server_butane, options) nfs_server = "10.0.2.2" default: - m, err = c.NewMachine(nfs_server_butane) + m, err = c.Cluster.NewMachine(nfs_server_butane) nfs_server = m.PrivateIP() } if err != nil { diff --git a/mantle/kola/tests/rhcos/upgrade.go b/mantle/kola/tests/rhcos/upgrade.go index 8f435c3354..ca5d895294 100644 --- a/mantle/kola/tests/rhcos/upgrade.go +++ b/mantle/kola/tests/rhcos/upgrade.go @@ -205,14 +205,14 @@ func rhcosUpgradeBasic(c cluster.TestCluster) { // no downgraded packages func rhcosUpgradeFromOcpRhcos(c cluster.TestCluster) { var m platform.Machine - options := platform.QemuMachineOptions{} + options := platform.MachineOptions{} ignition := conf.Ignition(`{ "ignition": { "version": "3.0.0" } }`) - switch pc := c.Cluster.(type) { + switch c.Cluster.(type) { case *qemu.Cluster: ostreeCommit := kola.CosaBuild.Meta.OstreeCommit temp := os.TempDir() @@ -228,7 +228,7 @@ func rhcosUpgradeFromOcpRhcos(c cluster.TestCluster) { defer os.Remove(rhcosQcow2) options.OverrideBackingFile = rhcosQcow2 - m, err = pc.NewMachineWithQemuOptions(ignition, options) + m, err = c.Cluster.NewMachineWithOptions(ignition, options) if err != nil { c.Fatal(err) } diff --git a/mantle/platform/cluster.go b/mantle/platform/cluster.go index a574850694..96ac4ba5a3 100644 --- a/mantle/platform/cluster.go +++ b/mantle/platform/cluster.go @@ -17,6 +17,7 @@ package platform import ( "bufio" "bytes" + "context" "fmt" "net" "os" @@ -91,6 +92,11 @@ func (bc *BaseCluster) PasswordSSHClient(ip string, user string, password string // SSH executes the given command, cmd, on the given Machine, m. It returns the // stdout and stderr of the command and an error. // Leading and trailing whitespace is trimmed from each. +// +// If the cluster's RuntimeConfig has a TestExecTimeout context set +// (e.g. by the test harness), SSH commands will be cancelled when that +// context is done. This allows the test execution timeout to kill hung +// SSH sessions without requiring callers to pass a context explicitly. func (bc *BaseCluster) SSH(m Machine, cmd string) ([]byte, []byte, error) { var stdout bytes.Buffer var stderr bytes.Buffer @@ -108,7 +114,31 @@ func (bc *BaseCluster) SSH(m Machine, cmd string) ([]byte, []byte, error) { session.Stdout = &stdout session.Stderr = &stderr - err = session.Run(cmd) + + ctx := bc.rconf.TestExecTimeout + if ctx == nil { + ctx = context.Background() + } + + // Use Start()+Wait() so we can select on context cancellation. + if err := session.Start(cmd); err != nil { + return nil, nil, err + } + + done := make(chan error, 1) + go func() { done <- session.Wait() }() + + select { + case err = <-done: + // Command finished normally (or with an error). + case <-ctx.Done(): + // Context cancelled/timed out. Close the session to + // kill the remote command, then drain the Wait goroutine. + session.Close() + <-done + err = ctx.Err() + } + plog.Debugf("Running cmd=%v res=%v", cmd, err) outBytes := bytes.TrimSpace(stdout.Bytes()) errBytes := bytes.TrimSpace(stderr.Bytes()) diff --git a/mantle/platform/machine/aws/cluster.go b/mantle/platform/machine/aws/cluster.go index e455840d7e..9ef5d29771 100644 --- a/mantle/platform/machine/aws/cluster.go +++ b/mantle/platform/machine/aws/cluster.go @@ -37,26 +37,12 @@ func (ac *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { + if err := options.EnsureNoQEMUOnlyOptions("aws"); err != nil { + return nil, err + } if len(options.AdditionalDisks) > 0 { return nil, errors.New("platform aws does not yet support additional disks") } - - if options.MultiPathDisk { - return nil, errors.New("platform aws does not support multipathed disks") - } - - if options.AdditionalNics > 0 { - return nil, errors.New("platform aws does not support additional nics") - } - - if options.AppendKernelArgs != "" { - return nil, errors.New("platform aws does not support appending kernel arguments") - } - - if options.AppendFirstbootKernelArgs != "" { - return nil, errors.New("platform aws does not support appending firstboot kernel arguments") - } - if options.InstanceType != "" { return nil, errors.New("platform aws does not support changing instance types") } @@ -115,12 +101,10 @@ func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(mach, mach.journal); err != nil { - mach.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(mach, mach.journal); err != nil { + mach.Destroy() + return nil, err } ac.AddMach(mach) diff --git a/mantle/platform/machine/azure/cluster.go b/mantle/platform/machine/azure/cluster.go index 2b1f846050..89a443d0d8 100644 --- a/mantle/platform/machine/azure/cluster.go +++ b/mantle/platform/machine/azure/cluster.go @@ -16,7 +16,6 @@ package azure import ( "crypto/rand" - "errors" "fmt" "os" "path/filepath" @@ -46,17 +45,8 @@ func (ac *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { - if options.MultiPathDisk { - return nil, errors.New("platform azure does not support multipathed disks") - } - if options.AdditionalNics > 0 { - return nil, errors.New("platform azure does not support additional nics") - } - if options.AppendKernelArgs != "" { - return nil, errors.New("platform azure does not support appending kernel arguments") - } - if options.AppendFirstbootKernelArgs != "" { - return nil, errors.New("platform azure does not support appending firstboot kernel arguments") + if err := options.EnsureNoQEMUOnlyOptions("azure"); err != nil { + return nil, err } conf, err := ac.RenderUserData(userdata, map[string]string{ @@ -94,12 +84,10 @@ func (ac *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(mach, mach.journal); err != nil { - mach.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(mach, mach.journal); err != nil { + mach.Destroy() + return nil, err } ac.AddMach(mach) diff --git a/mantle/platform/machine/do/cluster.go b/mantle/platform/machine/do/cluster.go index 77d39c234b..e88aad2f1d 100644 --- a/mantle/platform/machine/do/cluster.go +++ b/mantle/platform/machine/do/cluster.go @@ -37,21 +37,12 @@ func (dc *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (dc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { + if err := options.EnsureNoQEMUOnlyOptions("do"); err != nil { + return nil, err + } if len(options.AdditionalDisks) > 0 { return nil, errors.New("platform do does not yet support additional disks") } - if options.MultiPathDisk { - return nil, errors.New("platform do does not support multipathed disks") - } - if options.AdditionalNics > 0 { - return nil, errors.New("platform do does not support additional nics") - } - if options.AppendKernelArgs != "" { - return nil, errors.New("platform do does not support appending kernel arguments") - } - if options.AppendFirstbootKernelArgs != "" { - return nil, errors.New("platform do does not support appending firstboot kernel arguments") - } if options.InstanceType != "" { return nil, errors.New("platform do does not support changing instance types") } @@ -102,12 +93,10 @@ func (dc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(mach, mach.journal); err != nil { - mach.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(mach, mach.journal); err != nil { + mach.Destroy() + return nil, err } dc.AddMach(mach) diff --git a/mantle/platform/machine/esx/cluster.go b/mantle/platform/machine/esx/cluster.go index 86135e4700..dc1008064c 100644 --- a/mantle/platform/machine/esx/cluster.go +++ b/mantle/platform/machine/esx/cluster.go @@ -43,21 +43,12 @@ func (ec *cluster) NewMachine(userdata *platformConf.UserData) (platform.Machine } func (ec *cluster) NewMachineWithOptions(userdata *platformConf.UserData, options platform.MachineOptions) (platform.Machine, error) { + if err := options.EnsureNoQEMUOnlyOptions("esx"); err != nil { + return nil, err + } if len(options.AdditionalDisks) > 0 { return nil, errors.New("platform esx does not yet support additional disks") } - if options.MultiPathDisk { - return nil, errors.New("platform esx does not support multipathed disks") - } - if options.AdditionalNics > 0 { - return nil, errors.New("platform esx does not support additional nics") - } - if options.AppendKernelArgs != "" { - return nil, errors.New("platform esx does not support appending kernel arguments") - } - if options.AppendFirstbootKernelArgs != "" { - return nil, errors.New("platform esx does not support appending firstboot kernel arguments") - } if options.InstanceType != "" { return nil, errors.New("platform esx does not support changing instance types") } @@ -107,12 +98,10 @@ ExecStart=/usr/bin/bash -c 'echo "COREOS_ESX_IPV4_PRIVATE_0=$(ip addr show ens19 } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(mach, mach.journal); err != nil { - mach.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(mach, mach.journal); err != nil { + mach.Destroy() + return nil, err } ec.AddMach(mach) diff --git a/mantle/platform/machine/gcloud/cluster.go b/mantle/platform/machine/gcloud/cluster.go index 0eabc741f1..8d46821618 100644 --- a/mantle/platform/machine/gcloud/cluster.go +++ b/mantle/platform/machine/gcloud/cluster.go @@ -37,17 +37,8 @@ func (gc *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (gc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { - if options.MultiPathDisk { - return nil, errors.New("platform gcp does not support multipathed disks") - } - if options.AdditionalNics > 0 { - return nil, errors.New("platform gcp does not support additional nics") - } - if options.AppendKernelArgs != "" { - return nil, errors.New("platform gcp does not support appending kernel arguments") - } - if options.AppendFirstbootKernelArgs != "" { - return nil, errors.New("platform gcp does not support appending firstboot kernel arguments") + if err := options.EnsureNoQEMUOnlyOptions("gcp"); err != nil { + return nil, err } if options.InstanceType != "" { return nil, errors.New("platform gcp does not support changing instance types") @@ -101,12 +92,10 @@ func (gc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(gm, gm.journal); err != nil { - gm.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(gm, gm.journal); err != nil { + gm.Destroy() + return nil, err } gc.AddMach(gm) diff --git a/mantle/platform/machine/openstack/cluster.go b/mantle/platform/machine/openstack/cluster.go index 60255fe63a..a78b29a938 100644 --- a/mantle/platform/machine/openstack/cluster.go +++ b/mantle/platform/machine/openstack/cluster.go @@ -35,21 +35,12 @@ func (oc *cluster) NewMachine(userdata *conf.UserData) (platform.Machine, error) } func (oc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platform.MachineOptions) (platform.Machine, error) { + if err := options.EnsureNoQEMUOnlyOptions("openstack"); err != nil { + return nil, err + } if len(options.AdditionalDisks) > 0 { return nil, errors.New("platform openstack does not yet support additional disks") } - if options.MultiPathDisk { - return nil, errors.New("platform openstack does not support multipathed disks") - } - if options.AdditionalNics > 0 { - return nil, errors.New("platform openstack does not support additional nics") - } - if options.AppendKernelArgs != "" { - return nil, errors.New("platform openstack does not support appending kernel arguments") - } - if options.AppendFirstbootKernelArgs != "" { - return nil, errors.New("platform openstack does not support appending firstboot kernel arguments") - } if options.InstanceType != "" { return nil, errors.New("platform openstack does not support changing instance types") } @@ -94,12 +85,10 @@ func (oc *cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(mach, mach.journal); err != nil { - mach.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(mach, mach.journal); err != nil { + mach.Destroy() + return nil, err } oc.AddMach(mach) diff --git a/mantle/platform/machine/qemu/cluster.go b/mantle/platform/machine/qemu/cluster.go index fcad73122b..40481c4ea1 100644 --- a/mantle/platform/machine/qemu/cluster.go +++ b/mantle/platform/machine/qemu/cluster.go @@ -51,13 +51,6 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.InstanceType != "" { return nil, errors.New("platform qemu does not support changing instance types") } - return qc.NewMachineWithQemuOptions(userdata, platform.QemuMachineOptions{ - MachineOptions: options, - Firmware: options.Firmware, - }) -} - -func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options platform.QemuMachineOptions) (platform.Machine, error) { id := uuid.New() dir := filepath.Join(qc.RuntimeConf().OutputDir, id) @@ -215,12 +208,10 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(qm, qm.journal); err != nil { - qm.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(qm, qm.journal); err != nil { + qm.Destroy() + return nil, err } qc.AddMach(qm) diff --git a/mantle/platform/machine/qemuiso/cluster.go b/mantle/platform/machine/qemuiso/cluster.go index 35d2c07cae..f56a47bc5e 100644 --- a/mantle/platform/machine/qemuiso/cluster.go +++ b/mantle/platform/machine/qemuiso/cluster.go @@ -49,12 +49,6 @@ func (qc *Cluster) NewMachineWithOptions(userdata *conf.UserData, options platfo if options.MultiPathDisk { return nil, errors.New("platform qemu-iso does not support multipathed primary disks") } - return qc.NewMachineWithQemuOptions(userdata, platform.QemuMachineOptions{ - MachineOptions: options, - }) -} - -func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options platform.QemuMachineOptions) (platform.Machine, error) { id := uuid.New() dir := filepath.Join(qc.RuntimeConf().OutputDir, id) @@ -154,12 +148,10 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl } // Run StartMachine, which blocks on the machine being booted up enough - // for SSH access, but only if the caller didn't tell us not to. - if !options.SkipStartMachine { - if err := platform.StartMachine(qm, qm.journal); err != nil { - qm.Destroy() - return nil, err - } + // for SSH access. + if err := platform.StartMachine(qm, qm.journal); err != nil { + qm.Destroy() + return nil, err } qc.AddMach(qm) diff --git a/mantle/platform/platform.go b/mantle/platform/platform.go index fcde8df89f..b43b032a09 100644 --- a/mantle/platform/platform.go +++ b/mantle/platform/platform.go @@ -157,18 +157,71 @@ type Flight interface { } type MachineOptions struct { + AdditionalDisks []string + MinDiskSize int + InstanceType string + + // Fields below are only supported on QEMU-based platforms. + // Non-QEMU platforms call EnsureNoQEMUOnlyOptions() to reject them. MultiPathDisk bool PrimaryDisk string - AdditionalDisks []string MinMemory int - MinDiskSize int NumaNodes bool AdditionalNics int AppendKernelArgs string AppendFirstbootKernelArgs string - SkipStartMachine bool // Skip platform.StartMachine on machine bringup - InstanceType string Firmware string + HostForwardPorts []HostForwardPort + DisablePDeathSig bool + OverrideBackingFile string + Nvme bool + Cex bool +} + +// EnsureNoQEMUOnlyOptions returns an error if any QEMU-only options +// are set. Non-QEMU platforms should call this to reject unsupported +// options early. +func (m *MachineOptions) EnsureNoQEMUOnlyOptions(platformName string) error { + if m.MultiPathDisk { + return fmt.Errorf("platform %s does not support multipathed disks", platformName) + } + if m.PrimaryDisk != "" { + return fmt.Errorf("platform %s does not support custom primary disks", platformName) + } + if m.MinMemory != 0 { + return fmt.Errorf("platform %s does not support setting minimum memory", platformName) + } + if m.NumaNodes { + return fmt.Errorf("platform %s does not support NUMA node simulation", platformName) + } + if m.AdditionalNics > 0 { + return fmt.Errorf("platform %s does not support additional NICs", platformName) + } + if m.AppendKernelArgs != "" { + return fmt.Errorf("platform %s does not support appending kernel arguments", platformName) + } + if m.AppendFirstbootKernelArgs != "" { + return fmt.Errorf("platform %s does not support appending firstboot kernel arguments", platformName) + } + if m.Firmware != "" { + return fmt.Errorf("platform %s does not support setting firmware", platformName) + } + if len(m.HostForwardPorts) > 0 { + return fmt.Errorf("platform %s does not support host forward ports", platformName) + } + if m.DisablePDeathSig { + return fmt.Errorf("platform %s does not support DisablePDeathSig", platformName) + } + if m.OverrideBackingFile != "" { + return fmt.Errorf("platform %s does not support OverrideBackingFile", platformName) + } + if m.Nvme { + return fmt.Errorf("platform %s does not support NVMe", platformName) + } + if m.Cex { + return fmt.Errorf("platform %s does not support Cex", platformName) + } + return nil } // SystemdDropin is a userdata type agnostic struct representing a systemd dropin @@ -221,6 +274,12 @@ type RuntimeConfig struct { // whether a Manhole into a machine should be created on detected failure SSHOnTestFailure bool + + // TestExecTimeout is a context that is cancelled when the test + // execution timeout fires. BaseCluster.SSH uses it to terminate + // in-flight SSH commands when the test times out. If nil, + // context.Background() is used (no timeout). + TestExecTimeout context.Context } // Wrap a StdoutPipe as a io.ReadCloser diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 3bc0fd7cb7..916a3e3b2f 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -66,17 +66,6 @@ type HostForwardPort struct { GuestPort int } -// QemuMachineOptions is specialized MachineOption struct for QEMU. -type QemuMachineOptions struct { - MachineOptions - HostForwardPorts []HostForwardPort - DisablePDeathSig bool - OverrideBackingFile string - Firmware string - Nvme bool - Cex bool -} - // QEMUMachine represents a qemu instance. type QEMUMachine interface { // Embedding the Machine interface