OCPQE-32065: Proposed Implementation for Spot Check Jobs

apm.lock.yaml

@@ -92,8 +92,8 @@ local_deployed_files:
 - .opencode/commands/sippy-update-ga-release-views.md
 - .opencode/commands/sippy-update-job-variant.md
 local_deployed_file_hashes:
-  .claude/commands/agentic-followup.md: sha256:3195764952aac75e75d85a861ec58d9f2cbb697735b9446eecbf50762241255d
-  .claude/commands/agentic-solve.md: sha256:cba8ca0bf1e1ee0d1175138e6648e6344d337615bf906909034b9a55b5531465
+  .claude/commands/agentic-followup.md: sha256:d40313dcf0721911cae14a4af9774c021b0f16d24662a1b547524a5eef56ca86
+  .claude/commands/agentic-solve.md: sha256:8df186e128d0ffac40f9494c8d46275ea12af43921981f05d52944ba072f40cd
   .claude/commands/sippy-dev-app.md: sha256:656276ed961940c137dde32ecdb0501427d4d811502a27125ba073adc770d266
   .claude/commands/sippy-dev-frontend.md: sha256:42eae4b3bc610c9fcb43533a6fed229a6d1c409d279f3d6f93672986ede62e3a
   .claude/commands/sippy-dev-migrate.md: sha256:80160e88e0cc0fc09ab3dd9cc6fc496fe87dd8873800eb65d700868034d59da2
@@ -111,8 +111,8 @@ local_deployed_file_hashes:
   .claude/rules/general.md: sha256:997f68e86cb43485ec5f108be3417f9bbb43ae1faffd660d598f18260f5df3ce
   .claude/rules/mcp.md: sha256:ddfe965e7cf8cddbba1374c6ae582a20ac0af17c958bf10e1a4edff6ff2ad0b8
   .claude/rules/testing.md: sha256:57834092f6732d17f8c1812d25290cfc1cfbbbeb6eae1561ba2240973c651866
-  .cursor/commands/agentic-followup.md: sha256:3195764952aac75e75d85a861ec58d9f2cbb697735b9446eecbf50762241255d
-  .cursor/commands/agentic-solve.md: sha256:cba8ca0bf1e1ee0d1175138e6648e6344d337615bf906909034b9a55b5531465
+  .cursor/commands/agentic-followup.md: sha256:d40313dcf0721911cae14a4af9774c021b0f16d24662a1b547524a5eef56ca86
+  .cursor/commands/agentic-solve.md: sha256:8df186e128d0ffac40f9494c8d46275ea12af43921981f05d52944ba072f40cd
   .cursor/commands/sippy-dev-app.md: sha256:656276ed961940c137dde32ecdb0501427d4d811502a27125ba073adc770d266
   .cursor/commands/sippy-dev-frontend.md: sha256:42eae4b3bc610c9fcb43533a6fed229a6d1c409d279f3d6f93672986ede62e3a
   .cursor/commands/sippy-dev-migrate.md: sha256:80160e88e0cc0fc09ab3dd9cc6fc496fe87dd8873800eb65d700868034d59da2
@@ -131,8 +131,8 @@ local_deployed_file_hashes:
   .cursor/rules/general.mdc: sha256:5bc6e1e12d53d85656248c9dc1239c74bcc0df29d5987f3b08e3d79e3df413b7
   .cursor/rules/mcp.mdc: sha256:c02472afd46e4c89f71d4487dcd5da98b0c1bcbcf7f9cbc4d7ed4e7d3a206ec1
   .cursor/rules/testing.mdc: sha256:e5ce80313a812750404d45355b462c2e3a6458f5bc20ad7996a5da1b169a4703
-  .gemini/commands/agentic-followup.toml: sha256:cb017c493fb79dcfbc6eda8d2d058cd951a2aea700887e8754f0103858fd2089
-  .gemini/commands/agentic-solve.toml: sha256:613044d7539a4550102760b434bad50fb8eaac76ba4a3721e7f6265dbbf2e8ed
+  .gemini/commands/agentic-followup.toml: sha256:81b2f8f0ddb3757d564bb4f35c4610d8cb466068a068296222b7fc01ef253977
+  .gemini/commands/agentic-solve.toml: sha256:33a3850f284398fe4080c63e2501afbf2bd0fa2475fb922400111a8294e650df
   .gemini/commands/sippy-dev-app.toml: sha256:fc28174eeab4e440694a823bd838d429241997a018d8a13f32e0f67ca4d973c5
   .gemini/commands/sippy-dev-frontend.toml: sha256:ec4ab5e1fb7581f09473e33b3ed4f53ce40509f23aac581e3b100ad1f59de5e5
   .gemini/commands/sippy-dev-migrate.toml: sha256:25c98ba4bfdb95270dfcb4238ae688f9a66f0e645d8a4a6c5e03b1cf8db5cb7e
@@ -143,8 +143,8 @@ local_deployed_file_hashes:
   .gemini/commands/sippy-generate-release-views.toml: sha256:9173d7507f469eda21a19f7e5ec47bce0c6ce87a73c1f5c10594a89417332927
   .gemini/commands/sippy-update-ga-release-views.toml: sha256:662ac8d503d883cf6142c801b79e46616afb3d806e1d936e952d918b257ddf73
   .gemini/commands/sippy-update-job-variant.toml: sha256:3b044dfaddeafa1a6d375918d5860785f8d315b593c355bbb7293a87c50361b8
-  .opencode/commands/agentic-followup.md: sha256:3195764952aac75e75d85a861ec58d9f2cbb697735b9446eecbf50762241255d
-  .opencode/commands/agentic-solve.md: sha256:cba8ca0bf1e1ee0d1175138e6648e6344d337615bf906909034b9a55b5531465
+  .opencode/commands/agentic-followup.md: sha256:d40313dcf0721911cae14a4af9774c021b0f16d24662a1b547524a5eef56ca86
+  .opencode/commands/agentic-solve.md: sha256:8df186e128d0ffac40f9494c8d46275ea12af43921981f05d52944ba072f40cd
   .opencode/commands/sippy-dev-app.md: sha256:656276ed961940c137dde32ecdb0501427d4d811502a27125ba073adc770d266
   .opencode/commands/sippy-dev-frontend.md: sha256:42eae4b3bc610c9fcb43533a6fed229a6d1c409d279f3d6f93672986ede62e3a
   .opencode/commands/sippy-dev-migrate.md: sha256:80160e88e0cc0fc09ab3dd9cc6fc496fe87dd8873800eb65d700868034d59da2

config/views.yaml

@@ -9,6 +9,13 @@ component_readiness:
     release: "5.0"
     relative_start: now-7d
     relative_end: now
+  spot_check_job_samples:
+    spotcheck-30d:
+      relative_start: now-30d
+      relative_end: now
+      include_variants:
+        JobTier:
+          - spotcheck-30d
   variant_options:
     column_group_by:
       Network: { }

pkg/api/componentreadiness/component_report.go

@@ -3,7 +3,6 @@ package componentreadiness
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"maps"
 	"os"
 	"reflect"
@@ -12,11 +11,13 @@ import (
 	"sync"
 	"time"
 
-	"github.com/apache/thrift/lib/go/thrift"
-	fischer "github.com/glycerine/golang-fisher-exact"
+	"github.com/openshift/sippy/pkg/api/componentreadiness/middleware/alltestspassrate"
+	"github.com/openshift/sippy/pkg/api/componentreadiness/middleware/fisherexact"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/middleware/linkinjector"
+	"github.com/openshift/sippy/pkg/api/componentreadiness/middleware/newtestpassrate"
 	regressionallowances2 "github.com/openshift/sippy/pkg/api/componentreadiness/middleware/regressionallowances"
 	"github.com/openshift/sippy/pkg/api/componentreadiness/middleware/regressiontracker"
+	"github.com/openshift/sippy/pkg/api/componentreadiness/middleware/spotcheckjobs"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
@@ -37,7 +38,6 @@ import (
 )
 
 const (
-	explanationNoRegression         = "No significant regressions found"
 	ComponentReportCacheKeyPrefix   = "ComponentReport~"
 	TestDetailsReportCacheKeyPrefix = "TestDetailsReport~"
 )
@@ -276,6 +276,14 @@ func (c *ComponentReportGenerator) getCache() cache.Cache {
 
 func (c *ComponentReportGenerator) initializeMiddleware() {
 	c.middlewares = middleware.List{}
+
+	// Initialize all our middleware applicable to this request.
+
+	// middlewares that inject synthetic tests must run first so results are in place for other middleware.
+	if len(c.ReqOptions.SpotCheckJobSamples) > 0 {
+		c.middlewares = append(c.middlewares, spotcheckjobs.NewSpotCheckJobsMiddleware(c.dataProvider, c.ReqOptions))
+	}
+
 	// Initialize all our middleware applicable to this request.
 	if c.ReqOptions.AdvancedOption.IncludeMultiReleaseAnalysis && c.ReqOptions.SampleRelease.PullRequestOptions == nil {
 		c.middlewares = append(c.middlewares, releasefallback.NewReleaseFallbackMiddleware(c.dataProvider, c.ReqOptions, c.releaseConfigs))
@@ -287,6 +295,11 @@ func (c *ComponentReportGenerator) initializeMiddleware() {
 	}
 	c.middlewares = append(c.middlewares, regressionallowances2.NewRegressionAllowancesMiddleware(c.ReqOptions, c.releaseConfigs))
 
+	// Analysis middleware ordered by priority — first responder wins.
+	c.middlewares = append(c.middlewares, newtestpassrate.NewNewTestPassRateMiddleware(c.ReqOptions))
+	c.middlewares = append(c.middlewares, alltestspassrate.NewAllTestsPassRateMiddleware(c.ReqOptions))
+	c.middlewares = append(c.middlewares, fisherexact.NewFisherExactMiddleware(c.ReqOptions))
+
 	// Initialize LinkInjector middleware
 	linkInjector := linkinjector.NewLinkInjectorMiddleware(c.ReqOptions, c.baseURL)
 	c.middlewares = append(c.middlewares, linkInjector)
@@ -644,7 +657,9 @@ func (c *ComponentReportGenerator) generateComponentTestReport(basisStatusMap, s
 				return crtype.ComponentReport{}, err
 			}
 
-			c.assessComponentStatus(&cellReport, log.NewEntry(log.New()))
+			if _, err := c.middlewares.Analyze(testKey, &cellReport); err != nil {
+				return crtype.ComponentReport{}, err
+			}
 			if lastFailure := sampleStatus.LastFailure; !lastFailure.IsZero() {
 				cellReport.LastFailure = &lastFailure // it's a copy, for pointer hygiene
 			}
@@ -747,172 +762,6 @@ func buildReport(sortedRows []crtest.RowIdentification, sortedColumns []crtest.C
 	return regressionRows, nil
 }
 
-func getRegressionStatus(basisPassPercentage, samplePassPercentage float64) crtest.Status {
-	if (basisPassPercentage - samplePassPercentage) > 0.15 {
-		return crtest.ExtremeRegression
-	}
-
-	return crtest.SignificantRegression
-}
-
-// TODO: this will eventually become the analyze step on a Middleware, or possibly a separate
-// set of objects relating to analysis, as there's not a lot of overlap between the analyzers
-// (fishers, pass rate, bayes (future)) and the middlewares (fallback, intentional regressions,
-// cross variant compare, rarely run jobs, etc.)
-func (c *ComponentReportGenerator) assessComponentStatus(testStats *testdetails.TestComparison, logger *log.Entry) {
-	// Catch unset required confidence, typically unit tests
-	opts := c.ReqOptions.AdvancedOption
-	if testStats.RequiredConfidence == 0 {
-		testStats.RequiredConfidence = opts.Confidence
-	}
-
-	if (testStats.BaseStats == nil || testStats.BaseStats.Total() == 0) && opts.PassRateRequiredNewTests > 0 {
-		// If we have no base stats, fall back to a raw pass rate comparison for new or improperly renamed tests:
-		c.buildPassRateTestStats(testStats, float64(opts.PassRateRequiredNewTests))
-		// If a new test reports no regression, and we're not using pass rate mode for all tests, we alter
-		// status to be missing basis for the pre-existing Fisher Exact behavior:
-		if testStats.ReportStatus == crtest.NotSignificant && opts.PassRateRequiredAllTests == 0 {
-			testStats.ReportStatus = crtest.MissingBasis
-		}
-		return
-	} else if opts.PassRateRequiredAllTests > 0 {
-		// If requested, switch to pass rate only testing to see what does not meet the criteria:
-		c.buildPassRateTestStats(testStats, float64(opts.PassRateRequiredAllTests))
-		return
-	}
-
-	// Otherwise we fall back to default behavior of Fishers Exact test:
-	c.buildFisherExactTestStats(testStats, logger)
-}
-
-func (c *ComponentReportGenerator) buildFisherExactTestStats(testStats *testdetails.TestComparison, logger *log.Entry) {
-
-	fisherExact := 0.0
-	testStats.Comparison = crtest.FisherExact
-
-	status := crtest.MissingBasis
-	opts := c.ReqOptions.AdvancedOption
-	if testStats.SampleStats.Total() == 0 {
-		if opts.IgnoreMissing {
-			status = crtest.NotSignificant
-		} else {
-			status = crtest.MissingSample
-		}
-		testStats.ReportStatus = status
-		testStats.FisherExact = thrift.Float64Ptr(0.0)
-		testStats.Explanations = append(testStats.Explanations, explanationNoRegression)
-	} else if testStats.BaseStats != nil && testStats.BaseStats.Total() != 0 {
-		samplePass := testStats.SampleStats.Passes(opts.FlakeAsFailure)
-		basePass := testStats.BaseStats.Passes(opts.FlakeAsFailure)
-		basisPassPercentage := float64(basePass) / float64(testStats.BaseStats.Total())
-		effectivePityFactor := float64(opts.PityFactor) + testStats.PityAdjustment
-		effectiveMinimumFailure := opts.MinimumFailure
-
-		// default starting status now that we know we have basis and sample
-		status = crtest.NotSignificant
-
-		// now that we know sampleTotal is non zero
-		samplePassPercentage := float64(samplePass) / float64(testStats.SampleStats.Total())
-
-		// are we below the MinimumFailure threshold?
-		if effectiveMinimumFailure != 0 &&
-			(testStats.SampleStats.Total()-samplePass) < effectiveMinimumFailure {
-			if status <= crtest.SignificantTriagedRegression {
-				testStats.Explanations = append(testStats.Explanations,
-					fmt.Sprintf("%s regression detected.", crtest.StringForStatus(status)))
-			}
-			testStats.ReportStatus = status
-			testStats.FisherExact = thrift.Float64Ptr(0.0)
-			return
-		}
-		significant := false
-		improved := samplePassPercentage >= basisPassPercentage
-
-		if improved {
-			// flip base and sample when improved
-			significant, fisherExact = c.fischerExactTest(testStats.RequiredConfidence, testStats.BaseStats.Total()-basePass, basePass, testStats.SampleStats.Total()-samplePass, samplePass)
-		} else if basisPassPercentage-samplePassPercentage > effectivePityFactor/100 {
-			significant, fisherExact = c.fischerExactTest(testStats.RequiredConfidence, testStats.SampleStats.Total()-samplePass, samplePass, testStats.BaseStats.Total()-basePass, basePass)
-		}
-		logger.Debugf("computed Fisher info: signifcant: %v, fisherExact: %v", significant, fisherExact)
-		if significant {
-			if improved {
-				status = crtest.SignificantImprovement
-			} else {
-				status = getRegressionStatus(basisPassPercentage, samplePassPercentage)
-			}
-		}
-	}
-	logger.Debugf("computed status: %d", int(status))
-	testStats.ReportStatus = status
-	testStats.FisherExact = thrift.Float64Ptr(fisherExact)
-
-	baseRelease := "no basis"
-	if testStats.BaseStats != nil {
-		baseRelease = testStats.BaseStats.Release
-	}
-	// If we have a regression, include explanations:
-	if testStats.ReportStatus <= crtest.SignificantTriagedRegression {
-		logger.Debugf("regression detected against: %s", baseRelease)
-
-		if testStats.ReportStatus <= crtest.SignificantRegression {
-			testStats.Explanations = append(testStats.Explanations,
-				fmt.Sprintf("%s regression detected.", crtest.StringForStatus(testStats.ReportStatus)))
-			testStats.Explanations = append(testStats.Explanations,
-				fmt.Sprintf("Fishers Exact probability of a regression: %.2f%%.", float64(100)-*testStats.FisherExact))
-			testStats.Explanations = append(testStats.Explanations,
-				fmt.Sprintf("Test pass rate dropped from %.2f%% to %.2f%%.",
-					testStats.BaseStats.SuccessRate*float64(100),
-					testStats.SampleStats.SuccessRate*float64(100)))
-		} else {
-			testStats.Explanations = append(testStats.Explanations,
-				fmt.Sprintf("%s regression detected.", crtest.StringForStatus(testStats.ReportStatus)))
-		}
-	} else {
-		logger.Debugf("NO regression detected against: %s", baseRelease)
-	}
-}
-
-func (c *ComponentReportGenerator) buildPassRateTestStats(testStats *testdetails.TestComparison, requiredSuccessRate float64) {
-
-	effectiveSuccessReq := requiredSuccessRate + testStats.RequiredPassRateAdjustment
-
-	// Assume 2x our allowed failure rate = an extreme regression.
-	// i.e. if we require 90%, extreme is anything below 80%
-	//      if we require 95%, extreme is anything below 90%
-	// if an adjustment is applied, still use the configured success rate to define extreme regression.
-	severeRegressionSuccessRate := effectiveSuccessReq - (100 - requiredSuccessRate)
-
-	// Require 6 runs in the sample (typically 1 week for daily jobs) for us to consider a pass rate requirement for a new test:
-	sufficientRuns := testStats.SampleStats.Total() >= 6
-
-	opts := c.ReqOptions.AdvancedOption
-	successRate := testStats.SampleStats.PassRate(opts.FlakeAsFailure)
-	if sufficientRuns && successRate*100 < effectiveSuccessReq && testStats.SampleStats.FailureCount >= opts.MinimumFailure {
-		rStatus := crtest.SignificantRegression
-		if successRate*100 < severeRegressionSuccessRate {
-			rStatus = crtest.ExtremeRegression
-		}
-		testStats.ReportStatus = rStatus
-		testStats.Explanations = append(testStats.Explanations,
-			fmt.Sprintf("Test has a %.2f%% pass rate, but %.2f%% is required.", successRate*100, effectiveSuccessReq))
-		testStats.Comparison = crtest.PassRate
-		testStats.SampleStats.SuccessRate = successRate
-		return
-	}
-
-	testStats.ReportStatus = crtest.NotSignificant
-	testStats.Explanations = append(testStats.Explanations, explanationNoRegression)
-}
-
-func (c *ComponentReportGenerator) fischerExactTest(confidenceRequired, sampleFailure, sampleSuccess, baseFailure, baseSuccess int) (bool, float64) {
-	_, _, r, _ := fischer.FisherExactTest(sampleFailure,
-		sampleSuccess,
-		baseFailure,
-		baseSuccess)
-	return r < 1-float64(confidenceRequired)/100, r
-}
-
 func (c *ComponentReportGenerator) getUniqueJUnitColumnValuesLast60Days(ctx context.Context, field string,
 	nested bool) ([]string,
 	error) {

pkg/api/componentreadiness/component_report_test.go

@@ -13,7 +13,6 @@ import (
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/crtest"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/reqopts"
 	"github.com/openshift/sippy/pkg/apis/api/componentreport/testdetails"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/openshift/sippy/pkg/api/componentreadiness/utils"
@@ -1196,6 +1195,7 @@ func TestGenerateComponentReport(t *testing.T) {
 	componentAndCapabilityGetter = fakeComponentAndCapabilityGetter
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
+			tc.generator.initializeMiddleware()
 			report, err := tc.generator.generateComponentTestReport(tc.baseStatus, tc.sampleStatus)
 			assert.NoError(t, err, "error generating component report")
 
@@ -1622,6 +1622,7 @@ func TestGenerateComponentTestDetailsReport(t *testing.T) {
 		}
 
 		t.Run(tc.name, func(t *testing.T) {
+			tc.generator.initializeMiddleware()
 			report := tc.generator.internalGenerateTestDetailsReport("", nil, nil, baseStats, sampleStats, tc.generator.ReqOptions.TestIDOptions[0])
 			assert.Equal(t, tc.expectedReport.RowIdentification, report.RowIdentification, "expected report row identification %+v, got %+v", tc.expectedReport.RowIdentification, report.RowIdentification)
 			assert.Equal(t, tc.expectedReport.ColumnIdentification, report.ColumnIdentification, "expected report column identification %+v, got %+v", tc.expectedReport.ColumnIdentification, report.ColumnIdentification)
@@ -1682,7 +1683,7 @@ func Test_componentReportGenerator_normalizeProwJobName(t *testing.T) {
 	}
 }
 
-func Test_componentReportGenerator_assessComponentStatus(t *testing.T) {
+func Test_componentReportGenerator_analyze(t *testing.T) {
 	tests := []struct {
 		name          string
 		sampleTotal   int
@@ -1824,6 +1825,7 @@ func Test_componentReportGenerator_assessComponentStatus(t *testing.T) {
 			c.ReqOptions.AdvancedOption.PassRateRequiredNewTests = tt.requiredPassRateForNewTests
 			c.ReqOptions.AdvancedOption.PassRateRequiredAllTests = tt.requiredPassRateForAllTests
 			c.ReqOptions.AdvancedOption.MinimumFailure = tt.minFail
+			c.initializeMiddleware()
 
 			testAnalysis := &testdetails.TestComparison{
 				SampleStats: testdetails.ReleaseStats{
@@ -1842,14 +1844,15 @@ func Test_componentReportGenerator_assessComponentStatus(t *testing.T) {
 				},
 			}
 
-			c.assessComponentStatus(testAnalysis, logrus.NewEntry(logrus.New()))
-			assert.Equalf(t, tt.expectedStatus, testAnalysis.ReportStatus, "assessComponentStatus expected status not equal")
+			testKey := crtest.Identification{}
+			_, err := c.middlewares.Analyze(testKey, testAnalysis)
+			assert.NoError(t, err)
+			assert.Equalf(t, tt.expectedStatus, testAnalysis.ReportStatus, "Analyze expected status not equal")
 			if tt.expectedFischers != nil {
-				// Mac and Linux do not matchup on floating point precision, so lets approximate the comparison:
 				assert.Equalf(t,
 					fmt.Sprintf("%.4f", *tt.expectedFischers),
 					fmt.Sprintf("%.4f", *testAnalysis.FisherExact),
-					"assessComponentStatus expected fischers value not equal")
+					"Analyze expected fischers value not equal")
 			} else {
 				assert.Nil(t, testAnalysis.FisherExact)
 			}