Skip to content

TRT-2735: Move PayloadForJobRun from BigQuery to PostgreSQL #3690

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mstaeble wants to merge 1 commit into
base: main
Choose a base branch
from
Open

TRT-2735: Move PayloadForJobRun from BigQuery to PostgreSQL #3690

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 11 additions & 39 deletions pkg/api/releases.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -548,46 +548,18 @@ func BuildReleasesResponse(releases []sippyv1.Release, lastUpdated time.Time) ap
}

// PayloadForJobRun returns the payload release tag that was used for a given job run.
func PayloadForJobRun(ctx context.Context, bigQueryClient *bqcachedclient.Client, jobRunID string) ([]apitype.JobPayload, error) {
// Calculate date range: 6 months ago through today
now := time.Now()
sixMonthsAgo := now.AddDate(0, -6, 0)

queryStr := fmt.Sprintf(`SELECT prowjob_job_name, release_verify_tag, prowjob_build_id
FROM `+"`openshift-gce-devel.ci_analysis_us.jobs`"+`
WHERE prowjob_start BETWEEN DATETIME('%s') AND DATETIME_ADD('%s', INTERVAL 1 DAY)
AND prowjob_build_id = '%s'
LIMIT 10`,
sixMonthsAgo.Format("2006-01-02"),
now.Format("2006-01-02"),
jobRunID)

q := bigQueryClient.Query(ctx, bqlabel.JobRunPayload, queryStr)
log.WithFields(log.Fields{
"jobRunID": jobRunID,
"dateRange": fmt.Sprintf("%s to %s", sixMonthsAgo.Format("2006-01-02"), now.Format("2006-01-02")),
"query": queryStr,
}).Info("Executing BigQuery payload query")

it, err := bqcachedclient.LoggedRead(ctx, q)
if err != nil {
log.WithError(err).Error("error querying job run payload from bigquery")
return nil, fmt.Errorf("error querying job run payload from bigquery: %w", err)
}

func PayloadForJobRun(dbClient *db.DB, jobRunID string) ([]apitype.JobPayload, error) {
var results []apitype.JobPayload
for {
var row apitype.JobPayload
err := it.Next(&row)
if err == iterator.Done {
break
}
if err != nil {
log.WithError(err).Error("error parsing job run payload from bigquery")
return nil, fmt.Errorf("error parsing job run payload from bigquery: %w", err)
}

results = append(results, row)
res := dbClient.DB.Table("release_job_runs").
Select(`release_job_runs.job_name AS prowjob_job_name,
release_tags.release_tag AS payload,
release_job_runs.prow_job_run_id AS prowjob_build_id`).
Joins("JOIN release_tags ON release_tags.id = release_job_runs.release_tag_id").
Where("release_job_runs.prow_job_run_id = ?", jobRunID).
Find(&results)
if res.Error != nil {
log.WithError(res.Error).Error("error querying job run payload from database")
return nil, fmt.Errorf("error querying job run payload from database: %w", res.Error)
}

return results, nil
Expand Down
7 changes: 3 additions & 4 deletions pkg/apis/api/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"math/big"
"time"

bq "cloud.google.com/go/bigquery"
"github.com/lib/pq"

"github.com/openshift/sippy/pkg/apis/api/componentreport/crview"
Expand Down Expand Up @@ -850,9 +849,9 @@ type FailedPayload struct {

// JobPayload represents the payload release tag information for a job run.
type JobPayload struct {
ProwjobJobName string `json:"prowjob_job_name" bigquery:"prowjob_job_name"`
Payload bq.NullString `json:"payload" bigquery:"release_verify_tag"`
ProwjobBuildID string `json:"prowjob_build_id" bigquery:"prowjob_build_id"`
ProwjobJobName string `json:"prowjob_job_name"`
Payload *string `json:"payload"`
ProwjobBuildID string `json:"prowjob_build_id"`
}

// CalendarEvent is an API type representing a FullCalendar.io event type, for use
Expand Down
1 change: 0 additions & 1 deletion pkg/bigquery/bqlabel/labels.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ const (
TestOutputs QueryValue = "test-outputs"
TestResults QueryValue = "test-results"
TestResultsOverall QueryValue = "test-results-overall"
JobRunPayload QueryValue = "job-run-payload"
JobRunLabels QueryValue = "job-run-labels"
JobRunLabelsReEvaluate QueryValue = "job-run-labels-reevaluate"
JobRunLabelsReEvaluateDelete QueryValue = "job-run-labels-reevaluate-delete"
Expand Down
9 changes: 2 additions & 7 deletions pkg/sippyserver/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1356,17 +1356,12 @@ func (s *Server) jsonJobRunSummary(w http.ResponseWriter, req *http.Request) {

// jsonJobRunPayload returns the payload release tag that was used for a given job run.
func (s *Server) jsonJobRunPayload(w http.ResponseWriter, req *http.Request) {
if s.bigQueryClient == nil {
failureResponse(w, http.StatusBadRequest, "job run payload API is only available when google-service-account-credential-file is configured")
return
}

jobRunIDStr := s.getParamOrFail(w, req, "prow_job_run_id")
if jobRunIDStr == "" {
return
}

results, err := api.PayloadForJobRun(req.Context(), s.bigQueryClient, jobRunIDStr)
results, err := api.PayloadForJobRun(s.db, jobRunIDStr)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
if err != nil {
failureResponse(w, http.StatusInternalServerError, err.Error())
return
Expand Down Expand Up @@ -2318,7 +2313,7 @@ func (s *Server) Serve() {
{
EndpointPath: "/api/job/run/payload",
Description: "Returns the payload a job run was using",
Capabilities: []string{ComponentReadinessCapability},
Capabilities: []string{LocalDBCapability},
HandlerFunc: s.jsonJobRunPayload,
CacheTime: 4 * time.Hour,
},
Expand Down