From e8eb42c9a9339fa8abddf53c84240e4829d55daa Mon Sep 17 00:00:00 2001 From: Matthew Staebler Date: Wed, 24 Jun 2026 13:04:25 -0400 Subject: [PATCH] Move PayloadForJobRun from BigQuery to PostgreSQL Query release_job_runs JOIN release_tags instead of BigQuery's ci_analysis_us.jobs table. The release_job_runs table already has a unique index on prow_job_run_id, so lookups are fast and no schema migration is needed. Also removes the BigQuery dependency (bq.NullString) from the JobPayload API type in favor of *string, and changes the endpoint capability from ComponentReadinessCapability to LocalDBCapability. TRT-2735 Co-Authored-By: Claude Opus 4.6 --- pkg/api/releases.go | 50 ++++++++-------------------------- pkg/apis/api/types.go | 7 ++--- pkg/bigquery/bqlabel/labels.go | 1 - pkg/sippyserver/server.go | 9 ++---- 4 files changed, 16 insertions(+), 51 deletions(-) diff --git a/pkg/api/releases.go b/pkg/api/releases.go index c54345056..347c3bc79 100644 --- a/pkg/api/releases.go +++ b/pkg/api/releases.go @@ -548,46 +548,18 @@ func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) ap } // PayloadForJobRun returns the payload release tag that was used for a given job run. -func PayloadForJobRun(ctx context.Context, bigQueryClient *bqcachedclient.Client, jobRunID string) ([]apitype.JobPayload, error) { - // Calculate date range: 6 months ago through today - now := time.Now() - sixMonthsAgo := now.AddDate(0, -6, 0) - - queryStr := fmt.Sprintf(`SELECT prowjob_job_name, release_verify_tag, prowjob_build_id - FROM `+"`openshift-gce-devel.ci_analysis_us.jobs`"+` - WHERE prowjob_start BETWEEN DATETIME('%s') AND DATETIME_ADD('%s', INTERVAL 1 DAY) - AND prowjob_build_id = '%s' - LIMIT 10`, - sixMonthsAgo.Format("2006-01-02"), - now.Format("2006-01-02"), - jobRunID) - - q := bigQueryClient.Query(ctx, bqlabel.JobRunPayload, queryStr) - log.WithFields(log.Fields{ - "jobRunID": jobRunID, - "dateRange": fmt.Sprintf("%s to %s", sixMonthsAgo.Format("2006-01-02"), now.Format("2006-01-02")), - "query": queryStr, - }).Info("Executing BigQuery payload query") - - it, err := bqcachedclient.LoggedRead(ctx, q) - if err != nil { - log.WithError(err).Error("error querying job run payload from bigquery") - return nil, fmt.Errorf("error querying job run payload from bigquery: %w", err) - } - +func PayloadForJobRun(dbClient *db.DB, jobRunID string) ([]apitype.JobPayload, error) { var results []apitype.JobPayload - for { - var row apitype.JobPayload - err := it.Next(&row) - if err == iterator.Done { - break - } - if err != nil { - log.WithError(err).Error("error parsing job run payload from bigquery") - return nil, fmt.Errorf("error parsing job run payload from bigquery: %w", err) - } - - results = append(results, row) + res := dbClient.DB.Table("release_job_runs"). + Select(`release_job_runs.job_name AS prowjob_job_name, + release_tags.release_tag AS payload, + release_job_runs.prow_job_run_id AS prowjob_build_id`). + Joins("JOIN release_tags ON release_tags.id = release_job_runs.release_tag_id"). + Where("release_job_runs.prow_job_run_id = ?", jobRunID). + Find(&results) + if res.Error != nil { + log.WithError(res.Error).Error("error querying job run payload from database") + return nil, fmt.Errorf("error querying job run payload from database: %w", res.Error) } return results, nil diff --git a/pkg/apis/api/types.go b/pkg/apis/api/types.go index cae03adec..5774a1fee 100644 --- a/pkg/apis/api/types.go +++ b/pkg/apis/api/types.go @@ -6,7 +6,6 @@ import ( "math/big" "time" - bq "cloud.google.com/go/bigquery" "github.com/lib/pq" "github.com/openshift/sippy/pkg/apis/api/componentreport/crview" @@ -850,9 +849,9 @@ type FailedPayload struct { // JobPayload represents the payload release tag information for a job run. type JobPayload struct { - ProwjobJobName string `json:"prowjob_job_name" bigquery:"prowjob_job_name"` - Payload bq.NullString `json:"payload" bigquery:"release_verify_tag"` - ProwjobBuildID string `json:"prowjob_build_id" bigquery:"prowjob_build_id"` + ProwjobJobName string `json:"prowjob_job_name"` + Payload *string `json:"payload"` + ProwjobBuildID string `json:"prowjob_build_id"` } // CalendarEvent is an API type representing a FullCalendar.io event type, for use diff --git a/pkg/bigquery/bqlabel/labels.go b/pkg/bigquery/bqlabel/labels.go index 1ac1160ab..c1b593bd7 100644 --- a/pkg/bigquery/bqlabel/labels.go +++ b/pkg/bigquery/bqlabel/labels.go @@ -92,7 +92,6 @@ const ( TestOutputs QueryValue = "test-outputs" TestResults QueryValue = "test-results" TestResultsOverall QueryValue = "test-results-overall" - JobRunPayload QueryValue = "job-run-payload" JobRunLabels QueryValue = "job-run-labels" JobRunLabelsReEvaluate QueryValue = "job-run-labels-reevaluate" JobRunLabelsReEvaluateDelete QueryValue = "job-run-labels-reevaluate-delete" diff --git a/pkg/sippyserver/server.go b/pkg/sippyserver/server.go index 7856e874b..7a1b47f1d 100644 --- a/pkg/sippyserver/server.go +++ b/pkg/sippyserver/server.go @@ -1356,17 +1356,12 @@ func (s *Server) jsonJobRunSummary(w http.ResponseWriter, req *http.Request) { // jsonJobRunPayload returns the payload release tag that was used for a given job run. func (s *Server) jsonJobRunPayload(w http.ResponseWriter, req *http.Request) { - if s.bigQueryClient == nil { - failureResponse(w, http.StatusBadRequest, "job run payload API is only available when google-service-account-credential-file is configured") - return - } - jobRunIDStr := s.getParamOrFail(w, req, "prow_job_run_id") if jobRunIDStr == "" { return } - results, err := api.PayloadForJobRun(req.Context(), s.bigQueryClient, jobRunIDStr) + results, err := api.PayloadForJobRun(s.db, jobRunIDStr) if err != nil { failureResponse(w, http.StatusInternalServerError, err.Error()) return @@ -2318,7 +2313,7 @@ func (s *Server) Serve() { { EndpointPath: "/api/job/run/payload", Description: "Returns the payload a job run was using", - Capabilities: []string{ComponentReadinessCapability}, + Capabilities: []string{LocalDBCapability}, HandlerFunc: s.jsonJobRunPayload, CacheTime: 4 * time.Hour, },