diff --git a/cmd/sippy/load.go b/cmd/sippy/load.go index 8a75b65223..a8afa8fc9c 100644 --- a/cmd/sippy/load.go +++ b/cmd/sippy/load.go @@ -37,6 +37,7 @@ import ( "github.com/openshift/sippy/pkg/dataloader/prowloader" "github.com/openshift/sippy/pkg/dataloader/prowloader/gcs" "github.com/openshift/sippy/pkg/dataloader/prowloader/github" + releasedefloader "github.com/openshift/sippy/pkg/dataloader/releasedefloader" "github.com/openshift/sippy/pkg/dataloader/releaseloader" "github.com/openshift/sippy/pkg/dataloader/testownershiploader" "github.com/openshift/sippy/pkg/db" @@ -100,7 +101,7 @@ func (f *LoadFlags) BindFlags(fs *pflag.FlagSet) { f.JiraFlags.BindFlags(fs) fs.BoolVar(&f.InitDatabase, "init-database", false, "Migrate the DB before loading") - fs.StringArrayVar(&f.Loaders, "loader", []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") + fs.StringArrayVar(&f.Loaders, "loader", []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates"}, "Which data sources to use for data loading") fs.StringArrayVar(&f.Releases, "release", f.Releases, "Which releases to load (one per arg instance)") fs.StringArrayVar(&f.Architectures, "arch", f.Architectures, "Which architectures to load (one per arg instance)") fs.StringVar(&f.JobVariantsInputFile, "job-variants-input-file", "expected-job-variants.json", "JSON input file for the job-variants loader") @@ -152,8 +153,6 @@ func NewLoadCommand() *cobra.Command { cacheClient = nil // error hygiene, since we pass this down to quite a few functions } - releaseConfigs := []sippyv1.Release{} - // initializing a bigquery client different from the normal one opCtx, ctx := bqcachedclient.OpCtxForCronEnv(ctx, "load") bqc, bigqueryErr := bqcachedclient.New( @@ -164,10 +163,14 @@ func NewLoadCommand() *cobra.Command { if f.CacheFlags.EnablePersistentCaching { bqc = f.CacheFlags.DecorateBiqQueryClientWithPersistentCache(bqc) } - releaseConfigs, err = api.GetReleasesFromBigQuery(context.Background(), bqc) - if err != nil { - return errors.Wrapf(err, "error querying releases from bq") - } + } + + // Read release definitions from PG for downstream loader construction. + // On the first run the table may be empty; the release-definitions + // loader will populate it for subsequent runs. + releaseConfigs := []sippyv1.Release{} + if dbErr == nil { + releaseConfigs, _ = api.GetReleasesFromDB(context.Background(), dbc) } // Ensure partitions exist for all releases (only when InitDatabase is true) @@ -201,6 +204,17 @@ func NewLoadCommand() *cobra.Command { var regressionCacheAdded bool for _, l := range f.Loaders { + if l == "release-definitions" { + if bigqueryErr != nil { + return errors.Wrap(bigqueryErr, "CRITICAL error getting BigQuery client which prevents release-definitions loading") + } + if dbErr != nil { + return errors.Wrap(dbErr, "CRITICAL error getting postgres client which prevents release-definitions loading") + } + rdl := releasedefloader.NewReleaseDefinitionLoader(ctx, dbc, bqc) + loaders = append(loaders, rdl) + } + // TODO: remove "component-readiness-cache" and "regression-tracker" once the cronjob // manifests are updated to use "regression-cache". if l == "component-readiness-cache" || l == "regression-tracker" || l == "regression-cache" { diff --git a/cmd/sippy/seed_data.go b/cmd/sippy/seed_data.go index adc6817edc..b8736bc5cf 100644 --- a/cmd/sippy/seed_data.go +++ b/cmd/sippy/seed_data.go @@ -411,6 +411,11 @@ func seedSyntheticData(dbc *db.DB) error { return nil } + if err := seedReleaseDefinitions(dbc); err != nil { + return errors.WithMessage(err, "failed to seed release definitions") + } + log.Info("Seeded release definitions") + if err := createTestSuite(dbc, "synthetic"); err != nil { return errors.WithMessage(err, "failed to create test suite") } @@ -447,6 +452,53 @@ func seedSyntheticData(dbc *db.DB) error { return nil } +func seedReleaseDefinitions(dbc *db.DB) error { + now := time.Now().UTC() + allCaps := pq.StringArray{models.CapComponentReadiness, models.CapFeatureGates, models.CapMetrics, models.CapPayloadTags, models.CapSippyClassic} + + type relMeta struct { + previous string + gaDays int // negative = days before now; 0 = no GA (in development) + } + meta := map[string]relMeta{ + "4.19": {previous: "4.18", gaDays: -289}, + "4.20": {previous: "4.19", gaDays: -163}, + "4.21": {previous: "4.20", gaDays: -58}, + "4.22": {previous: "4.21"}, + } + + for _, release := range syntheticReleases { + m := meta[release] + parts := strings.Split(release, ".") + major, minor := 0, 0 + if len(parts) >= 2 { + _, _ = fmt.Sscanf(parts[0], "%d", &major) + _, _ = fmt.Sscanf(parts[1], "%d", &minor) + } + + develStart := now.AddDate(0, 0, m.gaDays-180) + def := models.ReleaseDefinition{ + Release: release, + Major: major, + Minor: minor, + PreviousRelease: m.previous, + DevelopmentStartDate: &develStart, + Product: "OCP", + Status: "Full Support", + Capabilities: allCaps, + } + if m.gaDays != 0 { + ga := now.AddDate(0, 0, m.gaDays) + def.GADate = &ga + } + + if err := dbc.DB.Where("release = ?", release).FirstOrCreate(&def).Error; err != nil { + return fmt.Errorf("failed to create release definition %s: %w", release, err) + } + } + return nil +} + func seedProwJobs(dbc *db.DB) error { for _, release := range syntheticReleases { for _, job := range syntheticJobs { diff --git a/pkg/api/componentreadiness/dataprovider/postgres/provider.go b/pkg/api/componentreadiness/dataprovider/postgres/provider.go index 3ef5e2281a..cd07be787d 100644 --- a/pkg/api/componentreadiness/dataprovider/postgres/provider.go +++ b/pkg/api/componentreadiness/dataprovider/postgres/provider.go @@ -11,6 +11,7 @@ import ( "github.com/lib/pq" + "github.com/openshift/sippy/pkg/api" "github.com/openshift/sippy/pkg/api/componentreadiness/dataprovider" "github.com/openshift/sippy/pkg/api/componentreadiness/utils" "github.com/openshift/sippy/pkg/apis/api/componentreport/crstatus" @@ -132,60 +133,8 @@ func (p *PostgresProvider) QueryJobVariants(ctx context.Context) (crtest.JobVari return variants, nil } -// releaseMetadata holds hardcoded release info for known releases. -// This avoids needing a releases table — we derive release names from prow_jobs -// and fill in metadata from this map. -var releaseMetadata = map[string]struct { - previousRelease string - gaOffsetDays int // 0 = no GA date (in development) - product string // empty = defaults to "OCP" -}{ - "4.17": {previousRelease: "4.16", gaOffsetDays: -540}, - "4.18": {previousRelease: "4.17", gaOffsetDays: -395}, - "4.19": {previousRelease: "4.18", gaOffsetDays: -289}, - "4.20": {previousRelease: "4.19", gaOffsetDays: -163}, - "4.21": {previousRelease: "4.20", gaOffsetDays: -58}, - "4.22": {previousRelease: "4.21"}, - "5.0": {previousRelease: "4.22"}, -} - func (p *PostgresProvider) QueryReleases(ctx context.Context) ([]v1.Release, error) { - var releaseNames []string - err := p.dbc.DB.WithContext(ctx).Raw(`SELECT DISTINCT release FROM prow_jobs WHERE deleted_at IS NULL ORDER BY release DESC`). - Pluck("release", &releaseNames).Error - if err != nil { - return nil, fmt.Errorf("querying releases: %w", err) - } - - caps := map[v1.ReleaseCapability]bool{ - v1.ComponentReadinessCap: true, - v1.FeatureGatesCap: true, - v1.MetricsCap: true, - v1.PayloadTagsCap: true, - v1.SippyClassicCap: true, - } - - now := time.Now().UTC() - var releases []v1.Release - for _, name := range releaseNames { - rel := v1.Release{ - Release: name, - Capabilities: caps, - Product: "OCP", - } - if meta, ok := releaseMetadata[name]; ok { - rel.PreviousRelease = meta.previousRelease - if meta.gaOffsetDays != 0 { - ga := now.AddDate(0, 0, meta.gaOffsetDays) - rel.GADate = &ga - } - if meta.product != "" { - rel.Product = meta.product - } - } - releases = append(releases, rel) - } - return releases, nil + return api.GetReleasesFromDB(ctx, p.dbc) } func (p *PostgresProvider) QueryReleaseDates(ctx context.Context, _ reqopts.RequestOptions) ([]crtest.ReleaseTimeRange, []error) { diff --git a/pkg/api/releases.go b/pkg/api/releases.go index c543450568..39857600be 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -512,6 +512,66 @@ func transformRelease(r sippyv1.ReleaseRow) sippyv1.Release { return release } +// GetReleaseRowsFromBigQuery fetches raw release rows from BigQuery's Releases table. +func GetReleaseRowsFromBigQuery(ctx context.Context, client *bqcachedclient.Client) ([]sippyv1.ReleaseRow, error) { + var rows []sippyv1.ReleaseRow + + queryString := fmt.Sprintf("SELECT * FROM `%s` ORDER BY DevelStartDate DESC", client.ReleasesTable) + + q := client.Query(ctx, bqlabel.ReleaseAllReleases, queryString) + it, err := q.Read(ctx) + if err != nil { + log.WithError(err).Error("error querying releases data from bigquery") + return rows, err + } + + for { + r := sippyv1.ReleaseRow{} + err := it.Next(&r) + if err == iterator.Done { + break + } + if err != nil { + log.WithError(err).Error("error parsing release row from bigquery") + return rows, err + } + rows = append(rows, r) + } + return rows, nil +} + +// GetReleasesFromDB queries release metadata from the release_definitions table +// and converts to []sippyv1.Release for use by existing callers. +func GetReleasesFromDB(ctx context.Context, dbc *db.DB) ([]sippyv1.Release, error) { + var defs []models.ReleaseDefinition + err := dbc.DB.WithContext(ctx).Order("development_start_date DESC").Find(&defs).Error + if err != nil { + return nil, fmt.Errorf("querying release definitions: %w", err) + } + releases := make([]sippyv1.Release, 0, len(defs)) + for _, def := range defs { + releases = append(releases, DefinitionToRelease(def)) + } + return releases, nil +} + +// DefinitionToRelease converts a models.ReleaseDefinition to a sippyv1.Release. +func DefinitionToRelease(def models.ReleaseDefinition) sippyv1.Release { + caps := make(map[sippyv1.ReleaseCapability]bool, len(def.Capabilities)) + for _, cap := range def.Capabilities { + caps[sippyv1.ReleaseCapability(cap)] = true + } + return sippyv1.Release{ + Release: def.Release, + Status: def.Status, + GADate: def.GADate, + DevelopmentStartDate: def.DevelopmentStartDate, + PreviousRelease: def.PreviousRelease, + Capabilities: caps, + Product: def.Product, + } +} + // BuildReleasesResponse creates the API response structure for releases func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) apitype.Releases { gaDateMap := make(map[string]time.Time) diff --git a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go index fc2d5424c5..485ab53664 100644 --- a/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go +++ b/pkg/dataloader/loaderwithmetrics/loaderwithmetrics.go @@ -46,7 +46,7 @@ func New(wrappedLoaders []dataloader.DataLoader) *LoaderWithMetrics { return loader } -var loaderOrder = []string{"prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} +var loaderOrder = []string{"release-definitions", "prow", "releases", "jira", "github", "bugs", "test-mapping", "feature-gates", "regression-cache"} // sortLoaders guarantees that the loaders run in a predictable and proper order func (l *LoaderWithMetrics) sortLoaders() { diff --git a/pkg/dataloader/releasedefloader/releasedefloader.go b/pkg/dataloader/releasedefloader/releasedefloader.go new file mode 100644 index 0000000000..3defa97261 --- /dev/null +++ b/pkg/dataloader/releasedefloader/releasedefloader.go @@ -0,0 +1,103 @@ +package releasedefloader + +import ( + "context" + "fmt" + "sort" + "time" + + "github.com/lib/pq" + log "github.com/sirupsen/logrus" + "gorm.io/gorm/clause" + + "github.com/openshift/sippy/pkg/api" + v1 "github.com/openshift/sippy/pkg/apis/sippy/v1" + bqcachedclient "github.com/openshift/sippy/pkg/bigquery" + "github.com/openshift/sippy/pkg/db" + "github.com/openshift/sippy/pkg/db/models" +) + +// ReleaseDefinitionLoader fetches release metadata from BigQuery and syncs it to PostgreSQL. +type ReleaseDefinitionLoader struct { + ctx context.Context + dbc *db.DB + bqClient *bqcachedclient.Client + errs []error +} + +func NewReleaseDefinitionLoader(ctx context.Context, dbc *db.DB, bqClient *bqcachedclient.Client) *ReleaseDefinitionLoader { + return &ReleaseDefinitionLoader{ + ctx: ctx, + dbc: dbc, + bqClient: bqClient, + } +} + +func (l *ReleaseDefinitionLoader) Name() string { + return "release-definitions" +} + +func (l *ReleaseDefinitionLoader) Load() { + releaseRows, err := api.GetReleaseRowsFromBigQuery(l.ctx, l.bqClient) + if err != nil { + l.errs = append(l.errs, fmt.Errorf("fetching releases from bigquery: %w", err)) + return + } + defs := make([]models.ReleaseDefinition, 0, len(releaseRows)) + for _, row := range releaseRows { + defs = append(defs, ReleaseRowToDefinition(row)) + } + if err := syncReleaseDefinitions(l.dbc, defs); err != nil { + l.errs = append(l.errs, fmt.Errorf("syncing release definitions: %w", err)) + } +} + +func (l *ReleaseDefinitionLoader) Errors() []error { + return l.errs +} + +func syncReleaseDefinitions(dbc *db.DB, defs []models.ReleaseDefinition) error { + for _, def := range defs { + err := dbc.DB.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "release"}}, + DoUpdates: clause.AssignmentColumns([]string{"major", "minor", "patch", "previous_release", "ga_date", "development_start_date", "product", "status", "capabilities", "updated_at"}), + }).Create(&def).Error + if err != nil { + return fmt.Errorf("upserting release definition %s: %w", def.Release, err) + } + } + log.WithField("count", len(defs)).Info("synced release definitions to postgres") + return nil +} + +// ReleaseRowToDefinition converts a BigQuery ReleaseRow directly to a ReleaseDefinition DB model. +func ReleaseRowToDefinition(r v1.ReleaseRow) models.ReleaseDefinition { + caps := make(pq.StringArray, 0, len(r.Capabilities)) + for _, cap := range r.Capabilities { + caps = append(caps, string(cap)) + } + sort.Strings(caps) + + def := models.ReleaseDefinition{ + Release: r.Release, + Major: r.Major, + Minor: r.Minor, + PreviousRelease: r.PreviousRelease.StringVal, + Product: r.Product.StringVal, + Status: r.ReleaseStatus.StringVal, + Capabilities: caps, + } + if r.Patch.Valid { + p := int(r.Patch.Int64) + def.Patch = &p + } + if r.GADate.Valid { + ga := r.GADate.Date.In(time.UTC) + def.GADate = &ga + } + if r.DevelStartDate.IsValid() { + ds := r.DevelStartDate.In(time.UTC) + def.DevelopmentStartDate = &ds + } + return def +} diff --git a/pkg/db/db.go b/pkg/db/db.go index 3a76773d93..096d61fc10 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -137,6 +137,7 @@ func (d *DB) UpdateSchema(reportEnd *time.Time) error { // List of all models to migrate modelsToMigrate := []any{ + &models.ReleaseDefinition{}, &models.ReleaseTag{}, &models.ReleasePullRequest{}, &models.ReleaseRepository{}, diff --git a/pkg/db/models/releases.go b/pkg/db/models/releases.go index 01c578e628..a02f533f72 100644 --- a/pkg/db/models/releases.go +++ b/pkg/db/models/releases.go @@ -6,6 +6,41 @@ import ( "github.com/lib/pq" ) +// ReleaseDefinition stores release metadata synced from BigQuery's Releases table. +// This is distinct from ReleaseTag, which tracks individual payload tags. +type ReleaseDefinition struct { + Model + + Release string `json:"release" gorm:"uniqueIndex;column:release"` + Major int `json:"major" gorm:"column:major"` + Minor int `json:"minor" gorm:"column:minor"` + Patch *int `json:"patch,omitempty" gorm:"column:patch"` + PreviousRelease string `json:"previous_release" gorm:"column:previous_release"` + GADate *time.Time `json:"ga_date,omitempty" gorm:"column:ga_date;type:date"` + DevelopmentStartDate *time.Time `json:"development_start_date" gorm:"column:development_start_date;type:date"` + Product string `json:"product" gorm:"column:product"` + Status string `json:"status" gorm:"column:status"` + Capabilities pq.StringArray `json:"capabilities" gorm:"type:text[];column:capabilities"` +} + +const ( + CapComponentReadiness = "componentReadiness" + CapSippyClassic = "sippyClassic" + CapMetrics = "metrics" + CapPullRequests = "pullRequests" + CapFeatureGates = "featureGates" + CapPayloadTags = "payloadTags" +) + +func (rd *ReleaseDefinition) HasCapability(capability string) bool { + for _, c := range rd.Capabilities { + if c == capability { + return true + } + } + return false +} + type ReleaseTag struct { Model diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 266d756a51..70ca06ae36 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -189,15 +189,19 @@ type Server struct { rateLimiters map[string]*rateLimiter } -// getReleases returns release data, preferring the BigQuery client with caching -// when available, falling back to the data provider for mock mode. -func (s *Server) getReleases(ctx context.Context, forceRefresh ...bool) ([]sippyv1.Release, error) { - if s.bigQueryClient != nil { - refresh := len(forceRefresh) > 0 && forceRefresh[0] - return api.GetReleases(ctx, s.bigQueryClient, refresh) +// getReleases returns release data from PostgreSQL. +func (s *Server) getReleases(ctx context.Context) ([]sippyv1.Release, error) { + if s.db != nil { + releases, err := api.GetReleasesFromDB(ctx, s.db) + if err == nil && len(releases) > 0 { + return releases, nil + } + if err != nil { + log.WithError(err).Warn("error querying releases from database, trying bigquery fallback") + } } - if s.crDataProvider != nil { - return s.crDataProvider.QueryReleases(ctx) + if s.bigQueryClient != nil { + return api.GetReleasesFromBigQuery(ctx, s.bigQueryClient) } return nil, fmt.Errorf("no data source available for releases") } @@ -907,8 +911,8 @@ func (s *Server) jsonTestRunsAndOutputsFromBigQuery(w http.ResponseWriter, req * outputs, err := api.GetTestRunsAndOutputsFromBigQuery(req.Context(), s.bigQueryClient, testID, prowJobRunIDList, prowJobNames, includeSuccess, startDate, endDate) if err != nil { - log.WithError(err).Error("error querying test runs from bigquery") - failureResponse(w, http.StatusInternalServerError, "error querying test runs from bigquery") + log.WithError(err).Error("error querying test runs from database") + failureResponse(w, http.StatusInternalServerError, "error querying test runs from database") return } @@ -922,11 +926,11 @@ func (s *Server) jsonComponentTestVariantsFromBigQuery(w http.ResponseWriter, re } outputs, errs := componentreadiness.GetComponentTestVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying test variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying test variants from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying test variants from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -939,11 +943,11 @@ func (s *Server) jsonJobVariantsFromBigQuery(w http.ResponseWriter, req *http.Re } outputs, errs := componentreadiness.GetJobVariants(req.Context(), s.crDataProvider) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying job variants from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying job variants from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying job variants from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1034,7 +1038,7 @@ func (s *Server) getComponentReportFromRequest(req *http.Request) (componentrepo baseURL, ) if len(errs) > 0 { - return componentreport.ComponentReport{}, fmt.Errorf("error querying component from big query: %v", errs) + return componentreport.ComponentReport{}, fmt.Errorf("error querying component from database: %v", errs) } // Add any warnings from parsing to the report @@ -1080,11 +1084,11 @@ func (s *Server) jsonComponentReportTestDetailsFromBigQuery(w http.ResponseWrite baseURL := api.GetBaseFrontendURL(req) outputs, errs := componentreadiness.GetTestDetails(req.Context(), s.crDataProvider, s.db, reqOptions, allReleases, baseURL) if len(errs) > 0 { - log.Warningf("%d errors were encountered while querying component test details from big query:", len(errs)) + log.Warningf("%d errors were encountered while querying component test details from database:", len(errs)) for _, err := range errs { log.Error(err.Error()) } - failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details from big query: %v", errs)) + failureResponse(w, http.StatusInternalServerError, fmt.Sprintf("error querying component test details from database: %v", errs)) return } api.RespondWithJSON(http.StatusOK, w, outputs) @@ -1153,8 +1157,7 @@ func (s *Server) jsonTestDetailsReportFromDB(w http.ResponseWriter, req *http.Re } func (s *Server) jsonReleasesReportFromDB(w http.ResponseWriter, req *http.Request) { - forceRefresh := req.URL.Query().Get("forceRefresh") != "" - releases, err := s.getReleases(req.Context(), forceRefresh) + releases, err := s.getReleases(req.Context()) if err != nil { log.WithError(err).Error("error querying releases") failureResponse(w, http.StatusInternalServerError, "error querying releases")