Use variant_combinations for query filtering and drop GIN index

pkg/api/test_analysis.go

@@ -56,11 +56,11 @@ func GetTestAnalysisOverallFromDB(dbc *db.DB, filters *filter.Filter, release, t
 	}
 
 	for _, bv := range blockedVariants {
-		jq = jq.Where("? != ANY(prow_jobs.variants)", bv)
+		jq = jq.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", bv)
 	}
 
 	for _, av := range allowedVariants {
-		jq = jq.Where("? = ANY(prow_jobs.variants)", av)
+		jq = jq.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", av)
 	}
 
 	r := jq.Scan(&rows)
@@ -122,11 +122,11 @@ func GetTestAnalysisByJobFromDB(dbc *db.DB, filters *filter.Filter, release, tes
 	}
 
 	for _, bv := range blockedVariants {
-		jq = jq.Where("? != ANY(variants)", bv)
+		jq = jq.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", bv)
 	}
 
 	for _, av := range allowedVariants {
-		jq = jq.Where("? = ANY(variants)", av)
+		jq = jq.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", av)
 	}
 
 	r := jq.Scan(&rows)

pkg/db/migrations/000004_drop_variants_gin_index.down.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS idx_prow_jobs_variants ON prow_jobs USING gin (variants);

pkg/db/migrations/000004_drop_variants_gin_index.up.sql

@@ -0,0 +1,4 @@
+-- Drop the GIN index on prow_jobs.variants. All variant filtering now
+-- goes through variant_combination_id lookups against the small
+-- variant_combinations table, so the GIN index is unused.
+DROP INDEX IF EXISTS idx_prow_jobs_variants;

pkg/db/models/prow.go

@@ -25,7 +25,7 @@ type ProwJob struct {
 	Kind     ProwKind
 	Name     string         `gorm:"unique"`
 	Release  string         `gorm:"index"`
-	Variants pq.StringArray `gorm:"type:text[];index:idx_prow_jobs_variants,type:gin"`
+	Variants pq.StringArray `gorm:"type:text[]"`
 	// VariantCombinationID references variant_combinations.id, maintained by a
 	// BEFORE INSERT/UPDATE trigger. NULL only when Variants is NULL.
 	VariantCombinationID *uint `gorm:"column:variant_combination_id"`

pkg/db/query/misc_queries.go

@@ -1,6 +1,7 @@
 package query
 
 import (
+	"database/sql"
 	"fmt"
 	"time"
 
@@ -28,20 +29,25 @@ func PlatformInfraSuccess(dbc *db.DB, platforms sets.String, period string) (map
 		return nil, fmt.Errorf("unknown period %s", period)
 	}
 
-	raw := dbc.DB.Table(table).
-		Select("*, unnest(variants) as variant").
-		Where("name = ?", testidentification.NewInfrastructureTestName)
-
 	var sqlResults []struct {
 		Variant        string
 		PassPercentage float64
 	}
-	q := dbc.DB.Table("(?) as results", raw).
-		Select(`
-			variant,
-			SUM(current_successes) * 100.0 / NULLIF(SUM(current_runs), 0) AS pass_percentage`).
-		Where("variant in ?", platforms.List()).
-		Group("variant").Scan(&sqlResults)
+	q := dbc.DB.Raw(fmt.Sprintf(`
+		WITH target_variants AS (
+			SELECT vc.id, v.variant
+			FROM variant_combinations vc, unnest(vc.variants) AS v(variant)
+			WHERE v.variant IN @platforms
+		)
+		SELECT tv.variant,
+			SUM(m.current_successes) * 100.0 / NULLIF(SUM(m.current_runs), 0) AS pass_percentage
+		FROM %s m
+		JOIN target_variants tv ON m.variant_combination_id = tv.id
+		WHERE m.name = @testname
+		GROUP BY tv.variant`, table),
+		sql.Named("platforms", platforms.List()),
+		sql.Named("testname", testidentification.NewInfrastructureTestName),
+	).Scan(&sqlResults)
 
 	for _, r := range sqlResults {
 		results[r.Variant] = r.PassPercentage

pkg/db/query/test_queries.go

@@ -113,7 +113,10 @@ func TestReportsByVariant(
 	// Query and group by variant:
 	var testReports []api.Test
 	q := `
-WITH results AS (
+WITH excluded_vc AS (
+    SELECT id FROM variant_combinations WHERE @excluded && variants
+),
+results AS (
     SELECT name,
            release,
            sum(current_runs)       AS current_runs,
@@ -127,7 +130,7 @@ WITH results AS (
            unnest(variants)        AS variant
     FROM prow_test_report_7d_matview
 	WHERE release = @release AND name ~* @testsubstrings
-          AND NOT(variants is not null and @excluded && variants)
+          AND variant_combination_id NOT IN (SELECT id FROM excluded_vc)
     GROUP BY name, release, variant
 )
 SELECT *,
@@ -169,7 +172,10 @@ func TestReportExcludeVariants(dbc *db.DB, release, testName string, excludeVari
 
 	// Query and group by variant:
 	var testReport api.Test
-	q := `WITH results AS (
+	q := `WITH excluded_vc AS (
+    SELECT id FROM variant_combinations WHERE @excluded && variants
+),
+results AS (
     SELECT name,
            release,
            sum(current_runs)       AS current_runs,
@@ -182,7 +188,7 @@ func TestReportExcludeVariants(dbc *db.DB, release, testName string, excludeVari
            sum(previous_flakes)    AS previous_flakes
     FROM prow_test_report_7d_matview
     WHERE release = @release AND name = @testname
-          AND NOT(variants is not null and @excluded && variants)
+          AND variant_combination_id NOT IN (SELECT id FROM excluded_vc)
     GROUP BY name, release
 ) SELECT *, %s FROM results;`
 
@@ -243,7 +249,7 @@ func TestsByNURPAndStandardDeviation(dbc *db.DB, release, table string) *gorm.DB
 			(current_pass_percentage - passing_average) AS delta_from_passing_average,
 			(current_flake_percentage - flake_average) AS delta_from_flake_average`).
 		Where(`release = ?`, release).
-		Where(fmt.Sprintf("NOT ('never-stable'=any(%s.variants))", table))
+		Where("variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE 'never-stable' = any(variants))")
 }
 
 func TestOutputs(dbc *db.DB, release, test string, includedVariants, excludedVariants []string, quantity int) ([]api.TestOutput, error) {
@@ -262,11 +268,11 @@ func TestOutputs(dbc *db.DB, release, test string, includedVariants, excludedVar
 		Where("prow_job_run_test_outputs.prow_job_run_test_release = ?", release)
 
 	for _, variant := range includedVariants {
-		q = q.Where("? = any(prow_jobs.variants)", variant)
+		q = q.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant)
 	}
 
 	for _, variant := range excludedVariants {
-		q = q.Where("NOT ? = any(prow_jobs.variants)", variant)
+		q = q.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant)
 	}
 
 	res := q.
@@ -295,11 +301,11 @@ func TestDurations(dbc *db.DB, release, test string, includedVariants, excludedV
 		Where("prow_job_run_tests.prow_job_run_release = ?", release)
 
 	for _, variant := range includedVariants {
-		q = q.Where("? = any(prow_jobs.variants)", variant)
+		q = q.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant)
 	}
 
 	for _, variant := range excludedVariants {
-		q = q.Where("NOT ? = any(prow_jobs.variants)", variant)
+		q = q.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant)
 	}
 
 	res := q.

pkg/db/views.go

@@ -372,10 +372,11 @@ var CollapsedVariantExclusions = []string{"never-stable", "aggregated"}
 var testReportCollapsedMatView = buildCollapsedMatViewSQL()
 
 func buildCollapsedMatViewSQL() string {
-	var clauses []string
-	for _, v := range CollapsedVariantExclusions {
-		clauses = append(clauses, fmt.Sprintf("NOT ('%s' = any(variants))", v))
+	quotedExclusions := make([]string, len(CollapsedVariantExclusions))
+	for i, v := range CollapsedVariantExclusions {
+		quotedExclusions[i] = fmt.Sprintf("'%s'", v)
 	}
+	excludedArray := "ARRAY[" + strings.Join(quotedExclusions, ",") + "]"
 	return `
 SELECT suite_name, name, id, jira_component, jira_component_id, release,
     SUM(current_runs)::bigint AS current_runs,
@@ -388,7 +389,9 @@ SELECT suite_name, name, id, jira_component, jira_component_id, release,
     SUM(previous_flakes)::bigint AS previous_flakes,
     (array_agg(open_bugs))[1] AS open_bugs
 FROM |||SOURCE|||
-WHERE ` + strings.Join(clauses, "\n  AND ") + `
+WHERE variant_combination_id NOT IN (
+    SELECT id FROM variant_combinations WHERE ` + excludedArray + ` && variants
+)
 GROUP BY suite_name, name, id, jira_component, jira_component_id, release
 `
 }