diff --git a/pkg/api/test_analysis.go b/pkg/api/test_analysis.go index b2043e5d9..94a0b610e 100644 --- a/pkg/api/test_analysis.go +++ b/pkg/api/test_analysis.go @@ -56,11 +56,11 @@ func GetTestAnalysisOverallFromDB(dbc *db.DB, filters *filter.Filter, release, t } for _, bv := range blockedVariants { - jq = jq.Where("? != ANY(prow_jobs.variants)", bv) + jq = jq.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", bv) } for _, av := range allowedVariants { - jq = jq.Where("? = ANY(prow_jobs.variants)", av) + jq = jq.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", av) } r := jq.Scan(&rows) @@ -122,11 +122,11 @@ func GetTestAnalysisByJobFromDB(dbc *db.DB, filters *filter.Filter, release, tes } for _, bv := range blockedVariants { - jq = jq.Where("? != ANY(variants)", bv) + jq = jq.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", bv) } for _, av := range allowedVariants { - jq = jq.Where("? = ANY(variants)", av) + jq = jq.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", av) } r := jq.Scan(&rows) diff --git a/pkg/db/migrations/000004_drop_variants_gin_index.down.sql b/pkg/db/migrations/000004_drop_variants_gin_index.down.sql new file mode 100644 index 000000000..884935116 --- /dev/null +++ b/pkg/db/migrations/000004_drop_variants_gin_index.down.sql @@ -0,0 +1 @@ +CREATE INDEX IF NOT EXISTS idx_prow_jobs_variants ON prow_jobs USING gin (variants); diff --git a/pkg/db/migrations/000004_drop_variants_gin_index.up.sql b/pkg/db/migrations/000004_drop_variants_gin_index.up.sql new file mode 100644 index 000000000..f7a1fe05b --- /dev/null +++ b/pkg/db/migrations/000004_drop_variants_gin_index.up.sql @@ -0,0 +1,4 @@ +-- Drop the GIN index on prow_jobs.variants. All variant filtering now +-- goes through variant_combination_id lookups against the small +-- variant_combinations table, so the GIN index is unused. +DROP INDEX IF EXISTS idx_prow_jobs_variants; diff --git a/pkg/db/models/prow.go b/pkg/db/models/prow.go index 1bcbf2fd1..6e24c143b 100644 --- a/pkg/db/models/prow.go +++ b/pkg/db/models/prow.go @@ -25,7 +25,7 @@ type ProwJob struct { Kind ProwKind Name string `gorm:"unique"` Release string `gorm:"index"` - Variants pq.StringArray `gorm:"type:text[];index:idx_prow_jobs_variants,type:gin"` + Variants pq.StringArray `gorm:"type:text[]"` // VariantCombinationID references variant_combinations.id, maintained by a // BEFORE INSERT/UPDATE trigger. NULL only when Variants is NULL. VariantCombinationID *uint `gorm:"column:variant_combination_id"` diff --git a/pkg/db/query/misc_queries.go b/pkg/db/query/misc_queries.go index cde09bd90..e48484974 100644 --- a/pkg/db/query/misc_queries.go +++ b/pkg/db/query/misc_queries.go @@ -1,6 +1,7 @@ package query import ( + "database/sql" "fmt" "time" @@ -28,20 +29,25 @@ func PlatformInfraSuccess(dbc *db.DB, platforms sets.String, period string) (map return nil, fmt.Errorf("unknown period %s", period) } - raw := dbc.DB.Table(table). - Select("*, unnest(variants) as variant"). - Where("name = ?", testidentification.NewInfrastructureTestName) - var sqlResults []struct { Variant string PassPercentage float64 } - q := dbc.DB.Table("(?) as results", raw). - Select(` - variant, - SUM(current_successes) * 100.0 / NULLIF(SUM(current_runs), 0) AS pass_percentage`). - Where("variant in ?", platforms.List()). - Group("variant").Scan(&sqlResults) + q := dbc.DB.Raw(fmt.Sprintf(` + WITH target_variants AS ( + SELECT vc.id, v.variant + FROM variant_combinations vc, unnest(vc.variants) AS v(variant) + WHERE v.variant IN @platforms + ) + SELECT tv.variant, + SUM(m.current_successes) * 100.0 / NULLIF(SUM(m.current_runs), 0) AS pass_percentage + FROM %s m + JOIN target_variants tv ON m.variant_combination_id = tv.id + WHERE m.name = @testname + GROUP BY tv.variant`, table), + sql.Named("platforms", platforms.List()), + sql.Named("testname", testidentification.NewInfrastructureTestName), + ).Scan(&sqlResults) for _, r := range sqlResults { results[r.Variant] = r.PassPercentage diff --git a/pkg/db/query/test_queries.go b/pkg/db/query/test_queries.go index ec5bbaaf8..9209ea322 100644 --- a/pkg/db/query/test_queries.go +++ b/pkg/db/query/test_queries.go @@ -113,7 +113,10 @@ func TestReportsByVariant( // Query and group by variant: var testReports []api.Test q := ` -WITH results AS ( +WITH excluded_vc AS ( + SELECT id FROM variant_combinations WHERE @excluded && variants +), +results AS ( SELECT name, release, sum(current_runs) AS current_runs, @@ -127,7 +130,7 @@ WITH results AS ( unnest(variants) AS variant FROM prow_test_report_7d_matview WHERE release = @release AND name ~* @testsubstrings - AND NOT(variants is not null and @excluded && variants) + AND variant_combination_id NOT IN (SELECT id FROM excluded_vc) GROUP BY name, release, variant ) SELECT *, @@ -169,7 +172,10 @@ func TestReportExcludeVariants(dbc *db.DB, release, testName string, excludeVari // Query and group by variant: var testReport api.Test - q := `WITH results AS ( + q := `WITH excluded_vc AS ( + SELECT id FROM variant_combinations WHERE @excluded && variants +), +results AS ( SELECT name, release, sum(current_runs) AS current_runs, @@ -182,7 +188,7 @@ func TestReportExcludeVariants(dbc *db.DB, release, testName string, excludeVari sum(previous_flakes) AS previous_flakes FROM prow_test_report_7d_matview WHERE release = @release AND name = @testname - AND NOT(variants is not null and @excluded && variants) + AND variant_combination_id NOT IN (SELECT id FROM excluded_vc) GROUP BY name, release ) SELECT *, %s FROM results;` @@ -243,7 +249,7 @@ func TestsByNURPAndStandardDeviation(dbc *db.DB, release, table string) *gorm.DB (current_pass_percentage - passing_average) AS delta_from_passing_average, (current_flake_percentage - flake_average) AS delta_from_flake_average`). Where(`release = ?`, release). - Where(fmt.Sprintf("NOT ('never-stable'=any(%s.variants))", table)) + Where("variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE 'never-stable' = any(variants))") } func TestOutputs(dbc *db.DB, release, test string, includedVariants, excludedVariants []string, quantity int) ([]api.TestOutput, error) { @@ -262,11 +268,11 @@ func TestOutputs(dbc *db.DB, release, test string, includedVariants, excludedVar Where("prow_job_run_test_outputs.prow_job_run_test_release = ?", release) for _, variant := range includedVariants { - q = q.Where("? = any(prow_jobs.variants)", variant) + q = q.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant) } for _, variant := range excludedVariants { - q = q.Where("NOT ? = any(prow_jobs.variants)", variant) + q = q.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant) } res := q. @@ -295,11 +301,11 @@ func TestDurations(dbc *db.DB, release, test string, includedVariants, excludedV Where("prow_job_run_tests.prow_job_run_release = ?", release) for _, variant := range includedVariants { - q = q.Where("? = any(prow_jobs.variants)", variant) + q = q.Where("prow_jobs.variant_combination_id IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant) } for _, variant := range excludedVariants { - q = q.Where("NOT ? = any(prow_jobs.variants)", variant) + q = q.Where("prow_jobs.variant_combination_id NOT IN (SELECT id FROM variant_combinations WHERE ? = any(variants))", variant) } res := q. diff --git a/pkg/db/views.go b/pkg/db/views.go index fc6d53c69..8795e29bd 100644 --- a/pkg/db/views.go +++ b/pkg/db/views.go @@ -372,10 +372,11 @@ var CollapsedVariantExclusions = []string{"never-stable", "aggregated"} var testReportCollapsedMatView = buildCollapsedMatViewSQL() func buildCollapsedMatViewSQL() string { - var clauses []string - for _, v := range CollapsedVariantExclusions { - clauses = append(clauses, fmt.Sprintf("NOT ('%s' = any(variants))", v)) + quotedExclusions := make([]string, len(CollapsedVariantExclusions)) + for i, v := range CollapsedVariantExclusions { + quotedExclusions[i] = fmt.Sprintf("'%s'", v) } + excludedArray := "ARRAY[" + strings.Join(quotedExclusions, ",") + "]" return ` SELECT suite_name, name, id, jira_component, jira_component_id, release, SUM(current_runs)::bigint AS current_runs, @@ -388,7 +389,9 @@ SELECT suite_name, name, id, jira_component, jira_component_id, release, SUM(previous_flakes)::bigint AS previous_flakes, (array_agg(open_bugs))[1] AS open_bugs FROM |||SOURCE||| -WHERE ` + strings.Join(clauses, "\n AND ") + ` +WHERE variant_combination_id NOT IN ( + SELECT id FROM variant_combinations WHERE ` + excludedArray + ` && variants +) GROUP BY suite_name, name, id, jira_component, jira_component_id, release ` }