TRT-2741: Expand variant keys and decouple daily summaries from variant_combination_id

Makefile

@@ -17,8 +17,8 @@ all: test build
 
 build: builddir clean npm frontend sippy sippy-daemon
 
-.PHONY: verify apm verify-apm
-verify: lint verify-apm
+.PHONY: verify apm verify-apm verify-migrations
+verify: lint verify-apm verify-migrations
 
 builddir:
 	mkdir -p sippy-ng/build
@@ -75,6 +75,9 @@ apm:
 	uvx --from apm-cli@0.13.0 apm install
 	uvx --from apm-cli@0.13.0 apm compile
 
+verify-migrations:
+	./hack/verify-migrations.sh
+
 verify-apm: apm
 	@if ! git diff --quiet HEAD -- .claude .cursor .gemini .opencode AGENTS.md CLAUDE.md GEMINI.md sippy-ng/AGENTS.md sippy-ng/CLAUDE.md mcp/AGENTS.md mcp/CLAUDE.md; then \
 		echo "ERROR: Generated APM files are out of date. Run 'make apm' and commit the results."; \

hack/verify-migrations.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+set -euo pipefail
+
+MIGRATIONS_DIR="pkg/db/migrations"
+MANIFEST="${MIGRATIONS_DIR}/MANIFEST"
+
+if [ ! -f "$MANIFEST" ]; then
+    echo "ERROR: ${MANIFEST} not found."
+    exit 1
+fi
+
+errors=0
+
+# Check each manifest entry has matching .up.sql and .down.sql files.
+prev_num=0
+while IFS= read -r entry; do
+    [ -z "$entry" ] && continue
+    [[ "$entry" == \#* ]] && continue
+
+    num=$(echo "$entry" | grep -oE '^[0-9]+' | sed 's/^0*//')
+    if [ -z "$num" ]; then
+        echo "ERROR: Invalid manifest entry '${entry}' (must start with a zero-padded number)."
+        errors=$((errors + 1))
+        continue
+    fi
+
+    # Check sequential numbering.
+    expected=$((prev_num + 1))
+    if [ "$num" -ne "$expected" ]; then
+        echo "ERROR: Expected migration $(printf '%06d' "$expected") but found $(printf '%06d' "$num") (gap or duplicate)."
+        errors=$((errors + 1))
+    fi
+    prev_num=$num
+
+    if [ ! -f "${MIGRATIONS_DIR}/${entry}.up.sql" ]; then
+        echo "ERROR: Manifest lists '${entry}' but ${MIGRATIONS_DIR}/${entry}.up.sql does not exist."
+        errors=$((errors + 1))
+    fi
+    if [ ! -f "${MIGRATIONS_DIR}/${entry}.down.sql" ]; then
+        echo "ERROR: Manifest lists '${entry}' but ${MIGRATIONS_DIR}/${entry}.down.sql does not exist."
+        errors=$((errors + 1))
+    fi
+done < "$MANIFEST"
+
+# Check no extra SQL files exist beyond what the manifest lists.
+expected_sql=$((prev_num * 2))
+actual_sql=$(find "$MIGRATIONS_DIR" -maxdepth 1 -name '*.sql' | wc -l | tr -d ' ')
+if [ "$actual_sql" -ne "$expected_sql" ]; then
+    echo "ERROR: Expected ${expected_sql} .sql files (${prev_num} up + ${prev_num} down) but found ${actual_sql}."
+    errors=$((errors + 1))
+fi
+
+if [ "$errors" -gt 0 ]; then
+    echo "FAILED: ${errors} migration verification error(s)."
+    exit 1
+fi
+
+echo "Migrations OK: ${prev_num} migrations verified."

pkg/db/dailysummary/dailysummary.go

@@ -17,7 +17,7 @@ const (
 	parallelWorkers     = 4
 )
 
-var valueColumns = []string{"variant_combination_id", "successes", "failures", "flakes", "runs"}
+var valueColumns = []string{"successes", "failures", "flakes", "runs"}
 
 var (
 	insertSQL        = buildInsertSQL()
@@ -33,17 +33,15 @@ func buildInsertSQL() string {
 			COALESCE(pjrt.suite_id, 0),
 			pjrt.prow_job_run_release,
 			date(pjrt.prow_job_run_timestamp),
-			pj.variant_combination_id,
 			COUNT(*) FILTER (WHERE pjrt.status = 1),
 			COUNT(*) FILTER (WHERE pjrt.status = 12),
 			COUNT(*) FILTER (WHERE pjrt.status = 13),
 			COUNT(*)
 		FROM prow_job_run_tests pjrt
-		JOIN prow_jobs pj ON pjrt.prow_job_id = pj.id
 		WHERE pjrt.prow_job_run_timestamp >= ?::date
 		  AND pjrt.prow_job_run_timestamp < (?::date + INTERVAL '1 day')
 		  AND pjrt.prow_job_run_release = ?
-		GROUP BY pjrt.test_id, pjrt.prow_job_id, COALESCE(pjrt.suite_id, 0), pjrt.prow_job_run_release, date(pjrt.prow_job_run_timestamp), pj.variant_combination_id`,
+		GROUP BY pjrt.test_id, pjrt.prow_job_id, COALESCE(pjrt.suite_id, 0), pjrt.prow_job_run_release, date(pjrt.prow_job_run_timestamp)`,
 		strings.Join(valueColumns, ", "))
 }
 

pkg/db/db.go

@@ -547,10 +547,9 @@ func ensureTriageSymptomCascade(db *gorm.DB) error {
 // function is created by migration 000003; the table is created by
 // AutoMigrate, so this must run after both.
 //
-// When the trigger is first attached, existing rows are backfilled
-// and test_daily_summaries is truncated so the next refresh
-// populates it with variant_combination_id set. In steady state
-// (trigger already exists) this is a single catalog lookup.
+// When the trigger is first attached, existing rows are backfilled.
+// In steady state (trigger already exists) this is a single catalog
+// lookup.
 func ensureVariantCombinationTrigger(db *gorm.DB) error {
 	return db.Exec(`
 		DO $$
@@ -576,8 +575,6 @@ func ensureVariantCombinationTrigger(db *gorm.DB) error {
 				WHERE prow_jobs.variants = vc.variants
 				  AND prow_jobs.variants IS NOT NULL
 				  AND prow_jobs.variant_combination_id IS NULL;
-
-				TRUNCATE test_daily_summaries;
 			END IF;
 		END $$`).Error
 }

pkg/db/migrations/000004_drop_daily_summary_variant_combination.down.sql

@@ -0,0 +1,2 @@
+TRUNCATE test_daily_summaries;
+ALTER TABLE test_daily_summaries ADD COLUMN IF NOT EXISTS variant_combination_id BIGINT;

pkg/db/migrations/000004_drop_daily_summary_variant_combination.up.sql

@@ -0,0 +1,5 @@
+DROP MATERIALIZED VIEW IF EXISTS prow_test_report_7d_collapsed_matview;
+DROP MATERIALIZED VIEW IF EXISTS prow_test_report_2d_collapsed_matview;
+DROP MATERIALIZED VIEW IF EXISTS prow_test_report_7d_matview;
+DROP MATERIALIZED VIEW IF EXISTS prow_test_report_2d_matview;
+ALTER TABLE test_daily_summaries DROP COLUMN IF EXISTS variant_combination_id;

pkg/db/migrations/MANIFEST

@@ -0,0 +1,13 @@
+# Migration manifest. Each line is a migration prefix (NNNNNN_name) that
+# must have matching .up.sql and .down.sql files in this directory.
+# Versions must be sequential with no gaps or duplicates.
+#
+# When adding a new migration, append it here. This file forces a git
+# merge conflict if two PRs independently add the same version number.
+#
+# Run `make verify-migrations` to validate.
+
+000001_create_partitioned_tables
+000002_create_test_daily_summaries
+000003_create_variant_combinations
+000004_drop_daily_summary_variant_combination

pkg/db/models/prow.go

@@ -172,16 +172,15 @@ type TestAnalysisByJobByDate struct {
 // TestDailySummary stores pre-aggregated daily test results used to
 // accelerate matview refreshes. Table managed by migration 000002.
 type TestDailySummary struct {
-	TestID               uint      `gorm:"column:test_id;not null"`
-	ProwJobID            uint      `gorm:"column:prow_job_id;not null"`
-	SuiteID              uint      `gorm:"column:suite_id;not null;default:0"`
-	Release              string    `gorm:"column:release;not null"`
-	SummaryDate          time.Time `gorm:"column:summary_date;type:date;not null"`
-	VariantCombinationID *uint     `gorm:"column:variant_combination_id"`
-	Successes            int32     `gorm:"column:successes;not null;default:0"`
-	Failures             int32     `gorm:"column:failures;not null;default:0"`
-	Flakes               int32     `gorm:"column:flakes;not null;default:0"`
-	Runs                 int32     `gorm:"column:runs;not null;default:0"`
+	TestID      uint      `gorm:"column:test_id;not null"`
+	ProwJobID   uint      `gorm:"column:prow_job_id;not null"`
+	SuiteID     uint      `gorm:"column:suite_id;not null;default:0"`
+	Release     string    `gorm:"column:release;not null"`
+	SummaryDate time.Time `gorm:"column:summary_date;type:date;not null"`
+	Successes   int32     `gorm:"column:successes;not null;default:0"`
+	Failures    int32     `gorm:"column:failures;not null;default:0"`
+	Flakes      int32     `gorm:"column:flakes;not null;default:0"`
+	Runs        int32     `gorm:"column:runs;not null;default:0"`
 }
 
 // Bug represents a Jira bug.

pkg/db/views.go

@@ -315,24 +315,25 @@ FROM (
     ),
     pre_agg AS (
       SELECT
-        variant_combination_id,
-        test_id,
-        suite_id,
-        release AS prow_job_run_release,
-        COALESCE(SUM(successes) FILTER (WHERE summary_date >= |||START||| AND summary_date < |||BOUNDARY|||), 0) AS previous_successes,
-        COALESCE(SUM(flakes)    FILTER (WHERE summary_date >= |||START||| AND summary_date < |||BOUNDARY|||), 0) AS previous_flakes,
-        COALESCE(SUM(failures)  FILTER (WHERE summary_date >= |||START||| AND summary_date < |||BOUNDARY|||), 0) AS previous_failures,
-        COALESCE(SUM(runs)      FILTER (WHERE summary_date >= |||START||| AND summary_date < |||BOUNDARY|||), 0) AS previous_runs,
-        COALESCE(SUM(successes) FILTER (WHERE summary_date >= |||BOUNDARY||| AND summary_date <= |||END|||), 0) AS current_successes,
-        COALESCE(SUM(flakes)    FILTER (WHERE summary_date >= |||BOUNDARY||| AND summary_date <= |||END|||), 0) AS current_flakes,
-        COALESCE(SUM(failures)  FILTER (WHERE summary_date >= |||BOUNDARY||| AND summary_date <= |||END|||), 0) AS current_failures,
-        COALESCE(SUM(runs)      FILTER (WHERE summary_date >= |||BOUNDARY||| AND summary_date <= |||END|||), 0) AS current_runs
+        pj.variant_combination_id,
+        tds.test_id,
+        tds.suite_id,
+        tds.release AS prow_job_run_release,
+        COALESCE(SUM(tds.successes) FILTER (WHERE tds.summary_date >= |||START||| AND tds.summary_date < |||BOUNDARY|||), 0) AS previous_successes,
+        COALESCE(SUM(tds.flakes)    FILTER (WHERE tds.summary_date >= |||START||| AND tds.summary_date < |||BOUNDARY|||), 0) AS previous_flakes,
+        COALESCE(SUM(tds.failures)  FILTER (WHERE tds.summary_date >= |||START||| AND tds.summary_date < |||BOUNDARY|||), 0) AS previous_failures,
+        COALESCE(SUM(tds.runs)      FILTER (WHERE tds.summary_date >= |||START||| AND tds.summary_date < |||BOUNDARY|||), 0) AS previous_runs,
+        COALESCE(SUM(tds.successes) FILTER (WHERE tds.summary_date >= |||BOUNDARY||| AND tds.summary_date <= |||END|||), 0) AS current_successes,
+        COALESCE(SUM(tds.flakes)    FILTER (WHERE tds.summary_date >= |||BOUNDARY||| AND tds.summary_date <= |||END|||), 0) AS current_flakes,
+        COALESCE(SUM(tds.failures)  FILTER (WHERE tds.summary_date >= |||BOUNDARY||| AND tds.summary_date <= |||END|||), 0) AS current_failures,
+        COALESCE(SUM(tds.runs)      FILTER (WHERE tds.summary_date >= |||BOUNDARY||| AND tds.summary_date <= |||END|||), 0) AS current_runs
       FROM
-        test_daily_summaries
+        test_daily_summaries tds
+        JOIN prow_jobs pj ON tds.prow_job_id = pj.id
       WHERE
-        summary_date >= |||START||| AND summary_date <= |||END|||
+        tds.summary_date >= |||START||| AND tds.summary_date <= |||END|||
       GROUP BY
-        variant_combination_id, test_id, suite_id, release
+        pj.variant_combination_id, tds.test_id, tds.suite_id, tds.release
     )
     SELECT
         tests.id,

pkg/testidentification/ocp_variants.go

@@ -24,6 +24,11 @@ import (
 var openshiftJobsNeverStableRaw string
 var openshiftJobsNeverStable = strings.Split(openshiftJobsNeverStableRaw, "\n")
 
+// importantVariants controls which BigQuery variant keys are stored in
+// prow_jobs.variants in PostgreSQL. Release-family keys (Release,
+// ReleaseMinor, ReleaseMajor, FromRelease, FromReleaseMinor,
+// FromReleaseMajor) are excluded because they are redundant with
+// prow_jobs.release and the Upgrade variant.
 var importantVariants = []string{
 	"Platform",
 	"Architecture",
@@ -39,6 +44,14 @@ var importantVariants = []string{
 	"Component",
 	"Capability",
 	"OS",
+	"Procedure",
+	"Aggregation",
+	"NetworkAccess",
+	"Scheduler",
+	"Suite",
+	"ContainerRuntime",
+	"CGroupMode",
+	"LayeredProduct",
 }
 
 const (
@@ -133,9 +146,6 @@ func (v *openshiftVariants) IdentifyVariants(jobName string) []string {
 		allVariants = append(allVariants, NeverStable)
 	}
 
-	// Ensure filtered by important variants; including them all
-	// significantly increases cardinality and slows matview refreshes
-	// to a crawl.
 	return filterVariants(allVariants, importantVariants)
 }