diff --git a/CLAUDE.md b/CLAUDE.md index 047a7f4808..96ad12e7a2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -164,6 +164,7 @@ Defined in `.cargo/config.toml`: - `cargo gen-types` / `cargo gen-spec` / `cargo gen-ts` - Type generation - `cargo test-api` - API testing and DB seeding - `cargo test-api seed` needs the API server already running (`cargo run` in `services/api`) with a fresh database (`services/api/data` holds only a tracked `.gitignore`; delete `services/api/data/bencher.db`, not the whole directory) + - `cargo test-api seed` also needs the `bencher` CLI binary already built (`cargo build --bin bencher`); it shells out via `assert_cmd`, which panics with `` `CARGO_BIN_EXE_bencher` is unset `` if the binary is missing - Pass `--no-git` when running the seed test in this repo: there is no colocated `.git`, so `bencher run` cannot derive a git context and the on-the-fly project naming assertions (`bencher` vs `Project`) will fail without it - `cargo test-runner` - Runner integration tests (requires Linux + KVM) diff --git a/lib/api_projects/src/reports.rs b/lib/api_projects/src/reports.rs index 5063ecf8d0..beeec805f6 100644 --- a/lib/api_projects/src/reports.rs +++ b/lib/api_projects/src/reports.rs @@ -29,7 +29,10 @@ use bencher_schema::{ head::HeadId, version::{QueryVersion, VersionId}, }, - report::{NewRunReport, QueryReport, ReportId, report_benchmark::ReportBenchmarkId}, + report::{ + NewRunReport, QueryReport, ReportId, ReportMode, + report_benchmark::ReportBenchmarkId, + }, }, user::{ actor::{ApiActor, PubProjectBearerToken}, @@ -87,6 +90,8 @@ pub async fn proj_reports_options( /// If the project is private, then the user must be authenticated and have `view` permissions for the project, /// or provide a valid project key for the project. /// By default, the reports are sorted by date time in reverse chronological order. +/// By default, the results and alerts for each report are omitted and only their counts are included. +/// Set the `expand` query param to `true` to include the full results and alerts. /// The HTTP response header `X-Total-Count` contains the total number of reports. #[endpoint { method = GET, @@ -149,10 +154,15 @@ async fn get_ls_inner( (&query_project, &pagination_params, &query_params) ))?; + let mode = if query_params.expand.unwrap_or_default() { + ReportMode::Full + } else { + ReportMode::Collapsed + }; // Drop connection lock before iterating let json_reports = reports .into_iter() - .map(|report| async { report.into_json(log, actor_conn!(context, api_actor)) }) + .map(|report| async { report.into_json(log, actor_conn!(context, api_actor), mode) }) .collect::>() .collect::>() .await @@ -419,7 +429,7 @@ async fn get_one_inner( Report, (&query_project, path_params.report) ))? - .into_json(log, conn) + .into_json(log, conn, ReportMode::Full) }) } diff --git a/lib/api_projects/tests/project_key_auth.rs b/lib/api_projects/tests/project_key_auth.rs index dd07539ec5..3649652891 100644 --- a/lib/api_projects/tests/project_key_auth.rs +++ b/lib/api_projects/tests/project_key_auth.rs @@ -1114,12 +1114,13 @@ async fn project_key_can_dismiss_alert() { let report: JsonReport = resp.json().await.expect("Failed to parse report"); // Check that an alert was generated + let alerts = report.alerts.as_deref().expect("Report missing alerts"); assert!( - !report.alerts.is_empty(), + !alerts.is_empty(), "Expected at least one alert from the spike run" ); - let alert_uuid = report.alerts[0].uuid; + let alert_uuid = alerts[0].uuid; // Dismiss the alert with the project key let resp = server diff --git a/lib/api_projects/tests/reports.rs b/lib/api_projects/tests/reports.rs index f8d3379cd2..64dc37d223 100644 --- a/lib/api_projects/tests/reports.rs +++ b/lib/api_projects/tests/reports.rs @@ -1,5 +1,7 @@ #![expect( unused_crate_dependencies, + clippy::expect_used, + clippy::missing_assert_message, clippy::tests_outside_test_module, clippy::uninlined_format_args, reason = "integration test file" @@ -11,7 +13,7 @@ use bencher_api_tests::{ helpers::{base_timestamp, create_test_report, get_project_id}, }; use bencher_json::{ - BenchmarkUuid, BoundaryUuid, JsonReports, MeasureUuid, MetricUuid, ModelUuid, + BenchmarkUuid, BoundaryUuid, JsonReport, JsonReports, MeasureUuid, MetricUuid, ModelUuid, ReportBenchmarkUuid, ThresholdUuid, }; use bencher_schema::{ @@ -299,6 +301,43 @@ async fn reports_delete_chunked() { delete_report_and_assert_empty(count).await; } +/// Create a report with two iterations, each with two benchmarks and one measure. +async fn post_report(server: &TestServer, token: &str, project_slug: &str) -> JsonReport { + let resp = server + .client + .post(server.api_url(&format!("/v0/projects/{}/reports", project_slug))) + .header(bencher_json::AUTHORIZATION, bencher_json::bearer_header(token)) + .json(&serde_json::json!({ + "branch": "main", + "testbed": "localhost", + "start_time": "2024-01-01T00:00:00Z", + "end_time": "2024-01-01T00:01:00Z", + "results": [ + "{\"bench_one\": {\"latency\": {\"value\": 100.0}}, \"bench_two\": {\"latency\": {\"value\": 200.0}}}", + "{\"bench_one\": {\"latency\": {\"value\": 101.0}}, \"bench_two\": {\"latency\": {\"value\": 201.0}}}" + ] + })) + .send() + .await + .expect("Request failed"); + + assert_eq!(resp.status(), StatusCode::CREATED); + resp.json().await.expect("Failed to parse response") +} + +fn assert_counts(counts: &serde_json::Value) { + assert_eq!( + counts, + &serde_json::json!({ + "results": [ + { "benchmarks": 2, "measures": 1 }, + { "benchmarks": 2, "measures": 1 } + ], + "alerts": { "total": 0, "active": 0 } + }) + ); +} + // GET /v0/projects/{project}/reports - list reports (empty) #[tokio::test] async fn reports_list_empty() { @@ -383,6 +422,144 @@ async fn reports_get_not_found() { assert_eq!(resp.status(), StatusCode::NOT_FOUND); } +// GET /v0/projects/{project}/reports - results and alerts are collapsed by default +#[tokio::test] +async fn reports_list_collapses_by_default() { + let server = TestServer::new().await; + let user = server + .signup("Test User", "reportcollapse@example.com") + .await; + let org = server.create_org(&user, "Report Collapse Org").await; + let project = server + .create_project(&user, &org, "Report Collapse Project") + .await; + + let project_slug: &str = project.slug.as_ref(); + post_report(&server, &user.token, project_slug).await; + + let resp = server + .client + .get(server.api_url(&format!("/v0/projects/{}/reports", project_slug))) + .header( + bencher_json::AUTHORIZATION, + bencher_json::bearer_header(&user.token), + ) + .send() + .await + .expect("Request failed"); + + assert_eq!(resp.status(), StatusCode::OK); + let reports: serde_json::Value = resp.json().await.expect("Failed to parse response"); + let report = reports + .as_array() + .expect("Response is not an array") + .first() + .expect("Response is empty"); + // The results and alerts keys are omitted entirely, not just empty + assert!(report.get("results").is_none()); + assert!(report.get("alerts").is_none()); + assert_counts(report.get("counts").expect("Report missing counts")); + + // The collapsed response still parses as the typed JsonReports + let reports: JsonReports = + serde_json::from_value(reports).expect("Failed to parse typed response"); + let report = reports.0.first().expect("Reports are empty"); + assert!(report.results.is_none()); + assert!(report.alerts.is_none()); +} + +// GET /v0/projects/{project}/reports?expand=true - full results and alerts +#[tokio::test] +async fn reports_list_expand_true_includes_full_report() { + let server = TestServer::new().await; + let user = server.signup("Test User", "reportexpand@example.com").await; + let org = server.create_org(&user, "Report Expand Org").await; + let project = server + .create_project(&user, &org, "Report Expand Project") + .await; + + let project_slug: &str = project.slug.as_ref(); + post_report(&server, &user.token, project_slug).await; + + let resp = server + .client + .get(server.api_url(&format!( + "/v0/projects/{}/reports?expand=true", + project_slug + ))) + .header( + bencher_json::AUTHORIZATION, + bencher_json::bearer_header(&user.token), + ) + .send() + .await + .expect("Request failed"); + + assert_eq!(resp.status(), StatusCode::OK); + let reports: JsonReports = resp.json().await.expect("Failed to parse response"); + let report = reports.0.first().expect("Reports are empty"); + let results = report.results.as_ref().expect("Report missing results"); + let alerts = report.alerts.as_ref().expect("Report missing alerts"); + // The counts are consistent with the full results and alerts + assert_eq!(results.len(), report.counts.results.len()); + for (iteration, counts) in results.iter().zip(&report.counts.results) { + assert_eq!(iteration.len(), counts.benchmarks as usize); + } + assert_eq!(alerts.len(), report.counts.alerts.total as usize); +} + +// GET /v0/projects/{project}/reports/{report} - always returns the full report +#[tokio::test] +async fn reports_get_one_always_full() { + let server = TestServer::new().await; + let user = server.signup("Test User", "reportgetone@example.com").await; + let org = server.create_org(&user, "Report GetOne Org").await; + let project = server + .create_project(&user, &org, "Report GetOne Project") + .await; + + let project_slug: &str = project.slug.as_ref(); + let report = post_report(&server, &user.token, project_slug).await; + + let resp = server + .client + .get(server.api_url(&format!( + "/v0/projects/{}/reports/{}", + project_slug, report.uuid + ))) + .header( + bencher_json::AUTHORIZATION, + bencher_json::bearer_header(&user.token), + ) + .send() + .await + .expect("Request failed"); + + assert_eq!(resp.status(), StatusCode::OK); + let report: JsonReport = resp.json().await.expect("Failed to parse response"); + assert!(report.results.is_some()); + assert!(report.alerts.is_some()); +} + +// POST /v0/projects/{project}/reports - response includes full results, alerts, and counts +#[tokio::test] +async fn reports_post_full_with_counts() { + let server = TestServer::new().await; + let user = server.signup("Test User", "reportpost@example.com").await; + let org = server.create_org(&user, "Report Post Org").await; + let project = server + .create_project(&user, &org, "Report Post Project") + .await; + + let project_slug: &str = project.slug.as_ref(); + let report = post_report(&server, &user.token, project_slug).await; + + assert!(report.results.is_some()); + assert!(report.alerts.is_some()); + let counts = serde_json::to_value(&report.counts).expect("Failed to serialize counts"); + assert_counts(&counts); +} + // DELETE /v0/projects/{project}/reports/{report} - not found #[tokio::test] async fn reports_delete_not_found() { diff --git a/lib/bencher_comment/src/lib.rs b/lib/bencher_comment/src/lib.rs index 8e4b4dc4df..4ebbe55b54 100644 --- a/lib/bencher_comment/src/lib.rs +++ b/lib/bencher_comment/src/lib.rs @@ -56,12 +56,13 @@ impl ReportComment { sub_adapter: SubAdapter, source: String, ) -> Self { + let results = json_report.results.as_deref().unwrap_or_default(); Self { console_url, project_slug: json_report.project.slug.clone(), public_links: json_report.project.visibility.is_public(), - multiple_iterations: json_report.results.len() > 1, - benchmark_count: json_report.results.iter().map(Vec::len).sum(), + multiple_iterations: results.len() > 1, + benchmark_count: results.iter().map(Vec::len).sum(), missing_threshold: Measure::missing_threshold(&json_report), json_report, sub_adapter, @@ -69,6 +70,14 @@ impl ReportComment { } } + fn results(&self) -> &[JsonReportIteration] { + self.json_report.results.as_deref().unwrap_or_default() + } + + fn alerts(&self) -> &[JsonAlert] { + self.json_report.alerts.as_deref().unwrap_or_default() + } + pub fn human(&self) -> String { let mut text = String::new(); self.human_report_link(&mut text); @@ -95,7 +104,7 @@ impl ReportComment { return; } text.push_str("\n\nView results:"); - for (i, iteration) in self.json_report.results.iter().enumerate() { + for (i, iteration) in self.results().iter().enumerate() { if self.multiple_iterations { if i != 0 { text.push('\n'); @@ -121,12 +130,12 @@ impl ReportComment { } fn human_alerts_list(&self, text: &mut String) { - if self.json_report.alerts.is_empty() { + if self.alerts().is_empty() { return; } text.push_str("\n\nView alerts:"); - for alert in &self.json_report.alerts { + for alert in self.alerts() { text.push_str(&format!( "\n- {benchmark_name} ({measure_name}){iter}: {console_url}", benchmark_name = alert.benchmark.name, @@ -269,10 +278,10 @@ impl ReportComment { } fn html_alerts(&self, html: &mut String, truncated: bool) { - if self.json_report.alerts.is_empty() { + if self.alerts().is_empty() { return; } - let alerts_len = self.json_report.alerts.len(); + let alerts_len = self.alerts().len(); html.push_str(&format!( "

🚨 {alerts_len} {alert}

", alert = if alerts_len == 1 { "Alert" } else { "Alerts" }, @@ -313,7 +322,7 @@ impl ReportComment { fn html_alerts_table_body(&self, html: &mut String) { html.push_str(""); - for alert in &self.json_report.alerts { + for alert in self.alerts() { let (factor, units, units_symbol) = { let mut min = alert.metric.value; if let Some(lower_limit) = alert.boundary.lower_limit { @@ -408,7 +417,7 @@ impl ReportComment { html.push_str("
Click to view all benchmark results"); html.push_str("
"); - for iteration in &self.json_report.results { + for iteration in self.results() { self.html_iteration_table(html, iteration, require_threshold); } html.push_str("
"); @@ -423,8 +432,7 @@ impl ReportComment { } fn has_boundary_alert(&self, boundary_limit: BoundaryLimit) -> bool { - self.json_report - .alerts + self.alerts() .iter() .any(|alert| alert.limit == boundary_limit) } @@ -651,7 +659,7 @@ impl ReportComment { } pub fn has_threshold(&self) -> bool { - for iteration in &self.json_report.results { + for iteration in self.results() { for result in iteration { for report_measure in &result.measures { if report_measure.threshold.is_some() { @@ -664,11 +672,11 @@ impl ReportComment { } pub fn has_alert(&self) -> bool { - !self.json_report.alerts.is_empty() + !self.alerts().is_empty() } pub fn find_alert(&self, result: &JsonReportResult, measure: &Measure) -> Option<&JsonAlert> { - self.json_report.alerts.iter().find(|alert| { + self.alerts().iter().find(|alert| { alert.benchmark.slug == result.benchmark.slug && alert.threshold.measure.slug == measure.slug }) @@ -869,6 +877,8 @@ impl Measure { fn missing_threshold(json_report: &JsonReport) -> HashSet { json_report .results + .as_deref() + .unwrap_or_default() .iter() .flat_map(|iteration| { iteration.iter().flat_map(|result| { @@ -1125,7 +1135,7 @@ fn boundary_limits_map( #[cfg(test)] mod tests { use bencher_json::{ - DateTime, JsonBranch, JsonHead, JsonProject, JsonReport, JsonTestbed, + DateTime, JsonBranch, JsonHead, JsonProject, JsonReport, JsonReportCounts, JsonTestbed, project::{Visibility, report::Adapter}, }; @@ -1179,8 +1189,9 @@ mod tests { start_time: DateTime::TEST, end_time: DateTime::TEST, adapter: Adapter::Magic, - results: Vec::new(), - alerts: Vec::new(), + results: Some(Vec::new()), + alerts: Some(Vec::new()), + counts: JsonReportCounts::default(), #[cfg(feature = "plus")] job: None, created: DateTime::TEST, diff --git a/lib/bencher_json/src/lib.rs b/lib/bencher_json/src/lib.rs index fb80ee2e78..cd566cd6c7 100644 --- a/lib/bencher_json/src/lib.rs +++ b/lib/bencher_json/src/lib.rs @@ -88,7 +88,10 @@ pub use project::{ model::{JsonModel, ModelUuid}, perf::{JsonPerf, JsonPerfQuery, ReportBenchmarkUuid}, plot::{JsonNewPlot, JsonPlot, JsonPlots, PlotUuid}, - report::{Iteration, JsonNewReport, JsonReport, JsonReports, ReportUuid}, + report::{ + Iteration, JsonNewReport, JsonReport, JsonReportAlertsCounts, JsonReportCounts, + JsonReportIterationCounts, JsonReports, ReportUuid, + }, testbed::{ JsonNewTestbed, JsonTestbed, JsonTestbeds, TestbedNameId, TestbedResourceId, TestbedSlug, TestbedUuid, diff --git a/lib/bencher_json/src/project/report.rs b/lib/bencher_json/src/project/report.rs index 869e244dc6..3f33d6267b 100644 --- a/lib/bencher_json/src/project/report.rs +++ b/lib/bencher_json/src/project/report.rs @@ -371,8 +371,17 @@ pub struct JsonReport { pub start_time: DateTime, pub end_time: DateTime, pub adapter: Adapter, - pub results: JsonReportResults, - pub alerts: JsonReportAlerts, + /// The report results, one list of results per iteration. + /// Omitted by default from the reports list endpoint; use the `expand` query param to include them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub results: Option, + /// The report alerts. + /// Omitted by default from the reports list endpoint; use the `expand` query param to include them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub alerts: Option, + /// The report counts. + #[serde(default)] + pub counts: JsonReportCounts, #[cfg(feature = "plus")] #[serde(skip_serializing_if = "Option::is_none")] pub job: Option, @@ -407,6 +416,39 @@ pub struct JsonReportMeasure { #[typeshare::typeshare] pub type JsonReportAlerts = Vec; +/// Counts for a report. +#[typeshare::typeshare] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(JsonSchema))] +pub struct JsonReportCounts { + /// Counts for the report results, one entry per iteration. + pub results: Vec, + /// Counts for the report alerts. + pub alerts: JsonReportAlertsCounts, +} + +/// Counts for a single iteration of a report. +#[typeshare::typeshare] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(JsonSchema))] +pub struct JsonReportIterationCounts { + /// The number of benchmarks in this iteration. + pub benchmarks: u32, + /// The number of distinct measures in this iteration. + pub measures: u32, +} + +/// Counts for the alerts of a report. +#[typeshare::typeshare] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(JsonSchema))] +pub struct JsonReportAlertsCounts { + /// The total number of alerts. + pub total: u32, + /// The number of active alerts. + pub active: u32, +} + #[derive(Debug, Clone, Deserialize)] #[cfg_attr(feature = "schema", derive(JsonSchema))] pub struct JsonReportQueryParams { @@ -421,6 +463,9 @@ pub struct JsonReportQueryParams { /// If set to `true`, only return reports with an archived branch or testbed. /// If not set or set to `false`, only returns reports with non-archived branches and testbeds. pub archived: Option, + /// If set to `true`, include the full results and alerts for each report. + /// If not set or set to `false`, the results and alerts are omitted and only the counts are included. + pub expand: Option, } #[derive(Debug, Clone)] @@ -430,6 +475,7 @@ pub struct JsonReportQuery { pub start_time: Option, pub end_time: Option, pub archived: Option, + pub expand: Option, } impl TryFrom for JsonReportQuery { @@ -442,6 +488,7 @@ impl TryFrom for JsonReportQuery { start_time, end_time, archived, + expand, } = query_params; let branch = if let Some(branch) = branch { @@ -461,6 +508,7 @@ impl TryFrom for JsonReportQuery { start_time: start_time.map(Into::into), end_time: end_time.map(Into::into), archived, + expand, }) } } diff --git a/lib/bencher_schema/src/model/project/report/mod.rs b/lib/bencher_schema/src/model/project/report/mod.rs index a4dbd1682e..f1c96acbf2 100644 --- a/lib/bencher_schema/src/model/project/report/mod.rs +++ b/lib/bencher_schema/src/model/project/report/mod.rs @@ -1,16 +1,22 @@ #[cfg(feature = "plus")] use bencher_json::runner::job::{JobUuid, JsonNewRunJob}; +use std::collections::HashSet; + use bencher_json::{ - DateTime, JsonNewReport, JsonReport, ReportUuid, - project::report::{ - Adapter, Iteration, JsonReportAlerts, JsonReportMeasure, JsonReportResult, - JsonReportResults, JsonReportSettings, ReportIdempotencyKey, + DateTime, JsonNewReport, JsonReport, JsonReportAlertsCounts, JsonReportCounts, + JsonReportIterationCounts, ReportUuid, + project::{ + alert::AlertStatus, + report::{ + Adapter, Iteration, JsonReportAlerts, JsonReportMeasure, JsonReportResult, + JsonReportResults, JsonReportSettings, ReportIdempotencyKey, + }, }, }; use diesel::OptionalExtension as _; use diesel::{ - ExpressionMethods as _, NullableExpressionMethods as _, QueryDsl as _, RunQueryDsl as _, - SelectableHelper as _, + AggregateExpressionMethods as _, ExpressionMethods as _, NullableExpressionMethods as _, + QueryDsl as _, RunQueryDsl as _, SelectableHelper as _, }; use dropshot::HttpError; @@ -164,7 +170,7 @@ impl QueryReport { if let Some(existing) = Self::check_idempotency(actor_conn!(context, api_actor), project_id, idempotency_key)? { - return existing.into_json(log, actor_conn!(context, api_actor)); + return existing.into_json(log, actor_conn!(context, api_actor), ReportMode::Full); } #[cfg(all(feature = "plus", not(feature = "otel")))] @@ -404,10 +410,15 @@ impl QueryReport { ); } - self.into_json(log, actor_conn!(context, api_actor)) + self.into_json(log, actor_conn!(context, api_actor), ReportMode::Full) } - pub fn into_json(self, log: &Logger, conn: &mut DbConnection) -> Result { + pub fn into_json( + self, + log: &Logger, + conn: &mut DbConnection, + mode: ReportMode, + ) -> Result { let Self { id, uuid, @@ -432,8 +443,16 @@ impl QueryReport { }; let branch = QueryBranch::get_json_for_report(conn, &query_project, head_id, version_id)?; let testbed = QueryTestbed::get_json_for_report(conn, &query_project, testbed_id, spec_id)?; - let results = get_report_results(log, conn, &query_project, id)?; - let alerts = get_report_alerts(conn, &query_project, id, head_id, version_id, spec_id)?; + let (results, alerts, counts) = match mode { + ReportMode::Full => { + let results = get_report_results(log, conn, &query_project, id)?; + let alerts = + get_report_alerts(conn, &query_project, id, head_id, version_id, spec_id)?; + let counts = report_counts(&results, &alerts); + (Some(results), Some(alerts), counts) + }, + ReportMode::Collapsed => (None, None, get_report_counts(conn, id)?), + }; #[cfg(feature = "plus")] let job = get_report_job(conn, id)?; @@ -450,6 +469,7 @@ impl QueryReport { adapter, results, alerts, + counts, #[cfg(feature = "plus")] job, created, @@ -612,6 +632,15 @@ async fn post_series_usage( }); } +/// Whether to materialize the full results and alerts when converting a report to JSON. +#[derive(Debug, Clone, Copy)] +pub enum ReportMode { + /// Include the full report results and alerts and compute the counts from them. + Full, + /// Omit the report results and alerts and compute the counts with aggregate queries. + Collapsed, +} + type ResultsQuery = ( Iteration, QueryBenchmark, @@ -753,6 +782,93 @@ fn into_report_results_json( report_results } +/// Compute the report counts with aggregate queries, without loading the results or alerts. +fn get_report_counts( + conn: &mut DbConnection, + report_id: ReportId, +) -> Result { + // Only count benchmarks that have at least one metric, + // matching the inner join used to build the full results JSON. + let results = schema::report_benchmark::table + .filter(schema::report_benchmark::report_id.eq(report_id)) + .inner_join(schema::metric::table) + .group_by(schema::report_benchmark::iteration) + .order(schema::report_benchmark::iteration.asc()) + .select(( + schema::report_benchmark::iteration, + diesel::dsl::count(schema::report_benchmark::benchmark_id).aggregate_distinct(), + diesel::dsl::count(schema::metric::measure_id).aggregate_distinct(), + )) + .load::<(Iteration, i64, i64)>(conn) + .map_err(resource_not_found_err!(ReportBenchmark, report_id))? + .into_iter() + .map( + |(_iteration, benchmarks, measures)| JsonReportIterationCounts { + benchmarks: u32::try_from(benchmarks).unwrap_or(u32::MAX), + measures: u32::try_from(measures).unwrap_or(u32::MAX), + }, + ) + .collect(); + + let mut alerts = JsonReportAlertsCounts::default(); + schema::alert::table + .inner_join( + schema::boundary::table + .inner_join(schema::metric::table.inner_join(schema::report_benchmark::table)), + ) + .filter(schema::report_benchmark::report_id.eq(report_id)) + .group_by(schema::alert::status) + .select((schema::alert::status, diesel::dsl::count_star())) + .load::<(AlertStatus, i64)>(conn) + .map_err(resource_not_found_err!(Alert, report_id))? + .into_iter() + .for_each(|(status, count)| { + // The GROUP BY yields at most one row per status, + // so a direct assignment suffices for the active count. + let count = u32::try_from(count).unwrap_or(u32::MAX); + alerts.total = alerts.total.saturating_add(count); + if matches!(status, AlertStatus::Active) { + alerts.active = count; + } + }); + + Ok(JsonReportCounts { results, alerts }) +} + +/// Compute the report counts from already loaded results and alerts. +fn report_counts(results: &JsonReportResults, alerts: &JsonReportAlerts) -> JsonReportCounts { + let results = results + .iter() + .map(|iteration| { + let measures = iteration + .iter() + .flat_map(|result| { + result + .measures + .iter() + .map(|report_measure| report_measure.measure.uuid) + }) + .collect::>(); + JsonReportIterationCounts { + benchmarks: u32::try_from(iteration.len()).unwrap_or(u32::MAX), + measures: u32::try_from(measures.len()).unwrap_or(u32::MAX), + } + }) + .collect(); + + let active = alerts + .iter() + .filter(|alert| matches!(alert.status, AlertStatus::Active)) + .count(); + JsonReportCounts { + results, + alerts: JsonReportAlertsCounts { + total: u32::try_from(alerts.len()).unwrap_or(u32::MAX), + active: u32::try_from(active).unwrap_or(u32::MAX), + }, + } +} + #[cfg(feature = "plus")] fn get_report_job( conn: &mut DbConnection, @@ -924,20 +1040,468 @@ pub fn upsert_metric_count( #[cfg(test)] mod tests { - use diesel::{ExpressionMethods as _, QueryDsl as _, RunQueryDsl as _}; + use diesel::{ExpressionMethods as _, QueryDsl as _, RunQueryDsl as _, SelectableHelper as _}; - use bencher_json::DateTime; + use bencher_json::{ + DateTime, JsonReportAlertsCounts, JsonReportIterationCounts, + project::{alert::AlertStatus, boundary::BoundaryLimit}, + }; use crate::{ + context::DbConnection, schema, test_util::{ - create_base_entities, create_branch_with_head, create_head_version, create_testbed, + BranchIds, create_alert, create_base_entities, create_benchmark, create_boundary, + create_branch_with_head, create_head_version, create_measure, create_metric, + create_model, create_report, create_report_benchmark, create_testbed, create_threshold, create_version, setup_test_db, }, }; - use super::ReportId; + use super::{QueryReport, ReportId, ReportMode, get_report_counts, report_counts}; use crate::macros::sql::last_insert_rowid; + use crate::model::project::{ProjectId, testbed::TestbedId}; + + fn test_logger() -> slog::Logger { + slog::Logger::root(slog::Discard, slog::o!()) + } + + struct ReportFixture { + project_id: ProjectId, + branch: BranchIds, + testbed_id: TestbedId, + report_id: ReportId, + } + + fn create_report_fixture(conn: &mut DbConnection) -> ReportFixture { + let base = create_base_entities(conn); + let branch = create_branch_with_head( + conn, + base.project_id, + "00000000-0000-0000-0000-000000000010", + "main", + "main", + "00000000-0000-0000-0000-000000000020", + ); + let testbed_id = create_testbed( + conn, + base.project_id, + "00000000-0000-0000-0000-000000000030", + "localhost", + "localhost", + ); + let version_id = create_version( + conn, + base.project_id, + "00000000-0000-0000-0000-000000000040", + 0, + None, + ); + create_head_version(conn, branch.head_id, version_id); + let report_id = create_report( + conn, + "00000000-0000-0000-0000-000000000050", + base.project_id, + branch.head_id, + version_id, + testbed_id, + ); + ReportFixture { + project_id: base.project_id, + branch, + testbed_id, + report_id, + } + } + + #[test] + #[expect(clippy::too_many_lines, reason = "test data setup")] + fn get_report_counts_per_iteration() { + let mut conn = setup_test_db(); + let fixture = create_report_fixture(&mut conn); + + let measure_one = create_measure( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000060", + "Latency", + "latency", + ); + let measure_two = create_measure( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000061", + "Throughput", + "throughput", + ); + let benchmark_one = create_benchmark( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000070", + "bench_one", + "bench-one", + ); + let benchmark_two = create_benchmark( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000071", + "bench_two", + "bench-two", + ); + + // Iteration 0: two benchmarks, each with both measures + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000080", + fixture.report_id, + 0, + benchmark_one, + ); + create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000090", + report_benchmark, + measure_one, + 1.0, + ); + create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000091", + report_benchmark, + measure_two, + 2.0, + ); + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000081", + fixture.report_id, + 0, + benchmark_two, + ); + create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000092", + report_benchmark, + measure_one, + 3.0, + ); + create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000093", + report_benchmark, + measure_two, + 4.0, + ); + + // Iteration 1: two benchmarks, only the first measure + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000082", + fixture.report_id, + 1, + benchmark_one, + ); + create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000094", + report_benchmark, + measure_one, + 5.0, + ); + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000083", + fixture.report_id, + 1, + benchmark_two, + ); + create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000095", + report_benchmark, + measure_one, + 6.0, + ); + + // A benchmark with no metrics is excluded, matching the full results JSON + let benchmark_three = create_benchmark( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000072", + "bench_three", + "bench-three", + ); + create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000084", + fixture.report_id, + 0, + benchmark_three, + ); + + let counts = + get_report_counts(&mut conn, fixture.report_id).expect("Failed to get report counts"); + assert_eq!( + counts.results, + vec![ + JsonReportIterationCounts { + benchmarks: 2, + measures: 2, + }, + JsonReportIterationCounts { + benchmarks: 2, + measures: 1, + }, + ] + ); + assert_eq!(counts.alerts, JsonReportAlertsCounts::default()); + } + + #[test] + fn get_report_counts_empty_report() { + let mut conn = setup_test_db(); + let fixture = create_report_fixture(&mut conn); + + let counts = + get_report_counts(&mut conn, fixture.report_id).expect("Failed to get report counts"); + assert!(counts.results.is_empty()); + assert_eq!(counts.alerts, JsonReportAlertsCounts::default()); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "test data setup")] + fn get_report_counts_alerts() { + let mut conn = setup_test_db(); + let fixture = create_report_fixture(&mut conn); + + let measure_id = create_measure( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000060", + "Latency", + "latency", + ); + let benchmark_one = create_benchmark( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000070", + "bench_one", + "bench-one", + ); + let benchmark_two = create_benchmark( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000071", + "bench_two", + "bench-two", + ); + let threshold_id = create_threshold( + &mut conn, + fixture.project_id, + fixture.branch.branch_id, + fixture.testbed_id, + measure_id, + "00000000-0000-0000-0000-0000000000a0", + ); + let model_id = create_model( + &mut conn, + threshold_id, + "00000000-0000-0000-0000-0000000000b0", + 0, + ); + + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000080", + fixture.report_id, + 0, + benchmark_one, + ); + let metric_id = create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000090", + report_benchmark, + measure_id, + 1.0, + ); + let boundary_id = create_boundary( + &mut conn, + "00000000-0000-0000-0000-0000000000c0", + metric_id, + threshold_id, + model_id, + ); + create_alert( + &mut conn, + "00000000-0000-0000-0000-0000000000d0", + boundary_id, + BoundaryLimit::Upper, + AlertStatus::Active, + ); + + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000081", + fixture.report_id, + 0, + benchmark_two, + ); + let metric_id = create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000091", + report_benchmark, + measure_id, + 2.0, + ); + let boundary_id = create_boundary( + &mut conn, + "00000000-0000-0000-0000-0000000000c1", + metric_id, + threshold_id, + model_id, + ); + create_alert( + &mut conn, + "00000000-0000-0000-0000-0000000000d1", + boundary_id, + BoundaryLimit::Upper, + AlertStatus::Dismissed, + ); + + let counts = + get_report_counts(&mut conn, fixture.report_id).expect("Failed to get report counts"); + assert_eq!( + counts.alerts, + JsonReportAlertsCounts { + total: 2, + active: 1, + } + ); + } + + #[test] + #[expect(clippy::too_many_lines, reason = "test data setup")] + fn into_json_full_and_collapsed_agree() { + let mut conn = setup_test_db(); + let fixture = create_report_fixture(&mut conn); + + let measure_id = create_measure( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000060", + "Latency", + "latency", + ); + let benchmark_one = create_benchmark( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000070", + "bench_one", + "bench-one", + ); + let benchmark_two = create_benchmark( + &mut conn, + fixture.project_id, + "00000000-0000-0000-0000-000000000071", + "bench_two", + "bench-two", + ); + let threshold_id = create_threshold( + &mut conn, + fixture.project_id, + fixture.branch.branch_id, + fixture.testbed_id, + measure_id, + "00000000-0000-0000-0000-0000000000a0", + ); + let model_id = create_model( + &mut conn, + threshold_id, + "00000000-0000-0000-0000-0000000000b0", + 0, + ); + + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000080", + fixture.report_id, + 0, + benchmark_one, + ); + let metric_id = create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000090", + report_benchmark, + measure_id, + 1.0, + ); + let boundary_id = create_boundary( + &mut conn, + "00000000-0000-0000-0000-0000000000c0", + metric_id, + threshold_id, + model_id, + ); + create_alert( + &mut conn, + "00000000-0000-0000-0000-0000000000d0", + boundary_id, + BoundaryLimit::Upper, + AlertStatus::Active, + ); + let report_benchmark = create_report_benchmark( + &mut conn, + "00000000-0000-0000-0000-000000000081", + fixture.report_id, + 0, + benchmark_two, + ); + create_metric( + &mut conn, + "00000000-0000-0000-0000-000000000091", + report_benchmark, + measure_id, + 2.0, + ); + + let log = test_logger(); + let load_report = |conn: &mut DbConnection| -> QueryReport { + schema::report::table + .filter(schema::report::id.eq(fixture.report_id)) + .select(QueryReport::as_select()) + .first(conn) + .expect("Failed to load report") + }; + + let full = load_report(&mut conn) + .into_json(&log, &mut conn, ReportMode::Full) + .expect("Failed to convert full report"); + let results = full.results.as_ref().expect("Full report missing results"); + let alerts = full.alerts.as_ref().expect("Full report missing alerts"); + assert_eq!(full.counts, report_counts(results, alerts)); + + let collapsed = load_report(&mut conn) + .into_json(&log, &mut conn, ReportMode::Collapsed) + .expect("Failed to convert collapsed report"); + assert!(collapsed.results.is_none()); + assert!(collapsed.alerts.is_none()); + + assert_eq!(full.counts, collapsed.counts); + assert_eq!( + full.counts.results, + vec![JsonReportIterationCounts { + benchmarks: 2, + measures: 1, + }] + ); + assert_eq!( + full.counts.alerts, + JsonReportAlertsCounts { + total: 1, + active: 1, + } + ); + } #[test] fn last_insert_rowid_returns_report_id() { diff --git a/services/api/openapi.json b/services/api/openapi.json index 761d2a6ee2..2bbdba8511 100644 --- a/services/api/openapi.json +++ b/services/api/openapi.json @@ -6668,7 +6668,7 @@ "reports" ], "summary": "List reports for a project", - "description": "List all reports for a project. If the project is public, then the user does not need to be authenticated. If the project is private, then the user must be authenticated and have `view` permissions for the project, or provide a valid project key for the project. By default, the reports are sorted by date time in reverse chronological order. The HTTP response header `X-Total-Count` contains the total number of reports.", + "description": "List all reports for a project. If the project is public, then the user does not need to be authenticated. If the project is private, then the user must be authenticated and have `view` permissions for the project, or provide a valid project key for the project. By default, the reports are sorted by date time in reverse chronological order. By default, the results and alerts for each report are omitted and only their counts are included. Set the `expand` query param to `true` to include the full results and alerts. The HTTP response header `X-Total-Count` contains the total number of reports.", "operationId": "proj_reports_get", "parameters": [ { @@ -6744,6 +6744,15 @@ "$ref": "#/components/schemas/DateTimeMillis" } }, + { + "in": "query", + "name": "expand", + "description": "If set to `true`, include the full results and alerts for each report. If not set or set to `false`, the results and alerts are omitted and only the counts are included.", + "schema": { + "nullable": true, + "type": "boolean" + } + }, { "in": "query", "name": "start_time", @@ -15973,6 +15982,8 @@ "$ref": "#/components/schemas/Adapter" }, "alerts": { + "nullable": true, + "description": "The report alerts. Omitted by default from the reports list endpoint; use the `expand` query param to include them.", "type": "array", "items": { "$ref": "#/components/schemas/JsonAlert" @@ -15981,6 +15992,21 @@ "branch": { "$ref": "#/components/schemas/JsonBranch" }, + "counts": { + "description": "The report counts.", + "default": { + "alerts": { + "active": 0, + "total": 0 + }, + "results": [] + }, + "allOf": [ + { + "$ref": "#/components/schemas/JsonReportCounts" + } + ] + }, "created": { "$ref": "#/components/schemas/DateTime" }, @@ -15999,6 +16025,8 @@ "$ref": "#/components/schemas/JsonProject" }, "results": { + "nullable": true, + "description": "The report results, one list of results per iteration. Omitted by default from the reports list endpoint; use the `expand` query param to include them.", "type": "array", "items": { "type": "array", @@ -16027,17 +16055,84 @@ }, "required": [ "adapter", - "alerts", "branch", "created", "end_time", "project", - "results", "start_time", "testbed", "uuid" ] }, + "JsonReportAlertsCounts": { + "description": "Counts for the alerts of a report.", + "type": "object", + "properties": { + "active": { + "description": "The number of active alerts.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "total": { + "description": "The total number of alerts.", + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "active", + "total" + ] + }, + "JsonReportCounts": { + "description": "Counts for a report.", + "type": "object", + "properties": { + "alerts": { + "description": "Counts for the report alerts.", + "allOf": [ + { + "$ref": "#/components/schemas/JsonReportAlertsCounts" + } + ] + }, + "results": { + "description": "Counts for the report results, one entry per iteration.", + "type": "array", + "items": { + "$ref": "#/components/schemas/JsonReportIterationCounts" + } + } + }, + "required": [ + "alerts", + "results" + ] + }, + "JsonReportIterationCounts": { + "description": "Counts for a single iteration of a report.", + "type": "object", + "properties": { + "benchmarks": { + "description": "The number of benchmarks in this iteration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + }, + "measures": { + "description": "The number of distinct measures in this iteration.", + "type": "integer", + "format": "uint32", + "minimum": 0 + } + }, + "required": [ + "benchmarks", + "measures" + ] + }, "JsonReportMeasure": { "type": "object", "properties": { diff --git a/services/cli/src/bencher/sub/project/report/list.rs b/services/cli/src/bencher/sub/project/report/list.rs index cb334ef8eb..bad9146ac0 100644 --- a/services/cli/src/bencher/sub/project/report/list.rs +++ b/services/cli/src/bencher/sub/project/report/list.rs @@ -21,6 +21,7 @@ pub struct List { pub end_time: Option, pub pagination: Pagination, pub archived: bool, + pub expand: bool, pub backend: PubBackend, } @@ -44,6 +45,7 @@ impl TryFrom for List { end_time, pagination, archived, + expand, backend, } = list; Ok(Self { @@ -54,6 +56,7 @@ impl TryFrom for List { end_time, pagination: pagination.into(), archived, + expand, backend: backend.try_into()?, }) } @@ -86,6 +89,7 @@ impl From for JsonReportQuery { start_time, end_time, archived, + expand, .. } = list; Self { @@ -94,6 +98,7 @@ impl From for JsonReportQuery { start_time, end_time, archived: archived.then_some(archived), + expand: expand.then_some(expand), } } } @@ -123,6 +128,9 @@ impl SubCmd for List { if let Some(archived) = json_report_query.archived { client = client.archived(archived); } + if let Some(expand) = json_report_query.expand { + client = client.expand(expand); + } if let Some(sort) = self.pagination.sort { client = client.sort(sort); diff --git a/services/cli/src/bencher/sub/run/mod.rs b/services/cli/src/bencher/sub/run/mod.rs index e9d82b9cc2..df4fed12e2 100644 --- a/services/cli/src/bencher/sub/run/mod.rs +++ b/services/cli/src/bencher/sub/run/mod.rs @@ -532,7 +532,7 @@ impl Run { json_report: JsonReport, ci_check: &mut Option, ) -> Result<(), RunError> { - let alerts_count = json_report.alerts.len(); + let alerts_count = usize::try_from(json_report.counts.alerts.total).unwrap_or(usize::MAX); self.display_results(json_report, ci_check).await?; if self.error_on_alert && alerts_count > 0 { Err(RunError::Alerts(alerts_count)) diff --git a/services/cli/src/parser/project/report.rs b/services/cli/src/parser/project/report.rs index 17f92ed526..fd959ca760 100644 --- a/services/cli/src/parser/project/report.rs +++ b/services/cli/src/parser/project/report.rs @@ -51,6 +51,10 @@ pub struct CliReportList { #[clap(long)] pub archived: bool, + /// Include the full results and alerts for each report + #[clap(long)] + pub expand: bool, + #[clap(flatten)] pub backend: CliBackend, } diff --git a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx index 8bb670416b..4c62ac97e2 100644 --- a/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx +++ b/services/console/src/chunks/docs-reference/changelog/en/changelog.mdx @@ -7,6 +7,7 @@ - Create the GitHub Check with an `in_progress` status before benchmarks run and complete it with the results once they finish, so reruns no longer show a stale conclusion; if `bencher run` exits with an error before the results are posted, the check is marked as failed (Thank you [@OmarTawfik](https://github.com/OmarTawfik)) - Add runner update channels via `runner up --update-channel ` (default: `stable`); the `canary` channel tracks a rolling prerelease that is rebuilt whenever the runner changes on a Bencher Cloud deploy, so runner changes are field-tested on Bencher Cloud before a versioned release; `canary` runners report their binary checksum and converge on the published build by content rather than by version - Runner self-update checksum fetch failures no longer drop the WebSocket channel into a reconnect loop; the server now logs the failure, skips the update offer, and retries at the runner's next `ready` poll +- **BREAKING CHANGE** The reports list endpoint (`GET /v0/projects/{project}/reports`) now omits the `results` and `alerts` for each report by default, drastically reducing response sizes for projects with large reports; set the new `expand` query param to `true` (`bencher report list --expand`) to restore the previous behavior. A new `counts` field summarizes each report with the number of benchmarks and distinct measures per iteration and the total and active alerts. Note that older CLI versions cannot parse the new default list response, so update the CLI before using `bencher report list`; `bencher run` is unaffected ## `v0.6.8` - **BREAKING CHANGE** Change the default self-hosted API server port from `61016` to the newly [IANA-registered](https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xhtml?search=6610) `6610`. Deployments relying on the default now serve the API on `6610`. Update clients, reverse proxies, and `--host`/`BENCHER_HOST` references (Docker Compose, devcontainer, and docs are updated to match). To stay on `61016`, set `server.bind_address` to `0.0.0.0:61016` (or run `bencher up --api-port 61016`). diff --git a/services/console/src/components/console/perf/PerfPanel.tsx b/services/console/src/components/console/perf/PerfPanel.tsx index 73d5813424..fde8128c74 100644 --- a/services/console/src/components/console/perf/PerfPanel.tsx +++ b/services/console/src/components/console/perf/PerfPanel.tsx @@ -842,13 +842,41 @@ const PerfPanel = (props: Props) => { setReportsTotalCount(headers?.[X_TOTAL_COUNT]), ), ); + // The reports list endpoint does not include the results, + // so fetch the full first report to bootstrap the default perf plot. + const first_report_fetcher = createMemo(() => { + if ( + !clear() && + branchesIsEmpty() && + testbedsIsEmpty() && + benchmarksIsEmpty() && + measuresIsEmpty() && + tab() === DEFAULT_PERF_TAB + ) { + return reports_data()?.[0]?.uuid; + } + return undefined; + }); + const [first_report_data] = createResource( + first_report_fetcher, + async (uuid) => { + const path = `/v0/projects/${project_slug()}/reports/${uuid}`; + return await httpGet(props.apiUrl, path, user?.token) + .then((resp) => resp?.data) + .catch((error) => { + console.error(error); + Sentry.captureException(error); + return undefined; + }); + }, + ); createEffect(() => { const data = reports_data(); if (data) { setReportsTab(resourcesToCheckable(data, [report()])); } const first = 0; - const first_report = data?.[first]; + const first_report = first_report_data(); if ( !clear() && first_report && @@ -1345,6 +1373,7 @@ const PerfPanel = (props: Props) => { > { } export interface Props { + apiUrl: string; project_slug: Accessor; theme: Accessor; isConsole: boolean; @@ -341,6 +342,7 @@ const PlotTab = (props: Props) => { ; theme: Accessor; isConsole: boolean; @@ -80,6 +92,7 @@ const ReportsTab = (props: { {(report, index) => { return ( ; theme: Accessor; isConsole: boolean; @@ -114,12 +128,29 @@ const ReportRow = (props: { }) => { const report = props.report?.resource as JsonReport; const hasBenchmarks = - report?.results - ?.map((iteration) => iteration?.length) - ?.reduce((acc, n) => acc + n, 0) > 0; + (report?.counts?.results?.reduce( + (acc, counts) => acc + (counts?.benchmarks ?? 0), + 0, + ) ?? 0) > 0; const viewReport = createMemo(() => props.isChecked && hasBenchmarks); + // The reports list endpoint does not include the results, + // so fetch the full report when it is expanded. + const [fullReport] = createResource( + () => (viewReport() ? report?.uuid : undefined), + async (uuid) => { + const path = `/v0/projects/${props.project_slug()}/reports/${uuid}`; + return await httpGet(props.apiUrl, path, authUser()?.token) + .then((resp) => resp?.data) + .catch((error) => { + console.error(error); + Sentry.captureException(error); + return undefined; + }); + }, + ); + return (
@@ -168,14 +199,25 @@ const ReportRow = (props: {
- report} - width={props.width} - /> + + } + > + + ); diff --git a/services/console/src/components/console/perf/plot/tab/Tab.tsx b/services/console/src/components/console/perf/plot/tab/Tab.tsx index 39d31bb4a7..aadbdf9e3f 100644 --- a/services/console/src/components/console/perf/plot/tab/Tab.tsx +++ b/services/console/src/components/console/perf/plot/tab/Tab.tsx @@ -23,6 +23,7 @@ import PlotsTab from "./PlotsTab"; import ReportsTab from "./ReportsTab"; const Tab = (props: { + apiUrl: string; project_slug: Accessor; theme: Accessor; isConsole: boolean; @@ -157,6 +158,7 @@ const Tab = (props: { } > { - const benchmarkCount = props.report?.results?.map( - (iteration) => iteration?.length, - ); + const benchmarkCount = + props.report?.counts?.results?.map((counts) => counts?.benchmarks) ?? []; - const totalAlerts = props.report?.alerts?.length; - const activeAlerts = props.report?.alerts?.filter( - (alert) => alert.status === AlertStatus.Active, - ).length; + const totalAlerts = props.report?.counts?.alerts?.total ?? 0; + const activeAlerts = props.report?.counts?.alerts?.active ?? 0; return (
@@ -54,9 +50,9 @@ export const ReportRow = (props: { report: JsonReport }) => { { - const measureCount = props.report?.results?.map( - (iteration) => boundaryLimitsMap(iteration).size, - ); + const measureCount = + props.report?.counts?.results?.map((counts) => counts?.measures) ?? + []; if (measureCount.length === 0) { return "0 measures"; } diff --git a/services/console/src/content/docs-reference/de/changelog.mdx b/services/console/src/content/docs-reference/de/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/de/changelog.mdx +++ b/services/console/src/content/docs-reference/de/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/content/docs-reference/en/changelog.mdx b/services/console/src/content/docs-reference/en/changelog.mdx index 28f1e15e1d..37394ef3a7 100644 --- a/services/console/src/content/docs-reference/en/changelog.mdx +++ b/services/console/src/content/docs-reference/en/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher release changelog" heading: "Bencher Changelog" published: "2023-08-12T16:07:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 --- diff --git a/services/console/src/content/docs-reference/es/changelog.mdx b/services/console/src/content/docs-reference/es/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/es/changelog.mdx +++ b/services/console/src/content/docs-reference/es/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/content/docs-reference/fr/changelog.mdx b/services/console/src/content/docs-reference/fr/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/fr/changelog.mdx +++ b/services/console/src/content/docs-reference/fr/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/content/docs-reference/ja/changelog.mdx b/services/console/src/content/docs-reference/ja/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/ja/changelog.mdx +++ b/services/console/src/content/docs-reference/ja/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/content/docs-reference/ko/changelog.mdx b/services/console/src/content/docs-reference/ko/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/ko/changelog.mdx +++ b/services/console/src/content/docs-reference/ko/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/content/docs-reference/pt/changelog.mdx b/services/console/src/content/docs-reference/pt/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/pt/changelog.mdx +++ b/services/console/src/content/docs-reference/pt/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/content/docs-reference/ru/changelog.mdx b/services/console/src/content/docs-reference/ru/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/ru/changelog.mdx +++ b/services/console/src/content/docs-reference/ru/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/content/docs-reference/zh/changelog.mdx b/services/console/src/content/docs-reference/zh/changelog.mdx index 373cce97ac..ac34987fbd 100644 --- a/services/console/src/content/docs-reference/zh/changelog.mdx +++ b/services/console/src/content/docs-reference/zh/changelog.mdx @@ -3,7 +3,7 @@ title: "Changelog" description: "Bencher changelog" heading: "Bencher Changelog" published: "2023-10-27T08:40:00Z" -modified: "2026-07-17T00:00:00Z" +modified: "2026-07-18T00:00:00Z" sortOrder: 10 canonicalize: true --- diff --git a/services/console/src/types/bencher.ts b/services/console/src/types/bencher.ts index 915436bf82..354ada5412 100644 --- a/services/console/src/types/bencher.ts +++ b/services/console/src/types/bencher.ts @@ -1056,6 +1056,30 @@ export enum Adapter { DartBenchmarkHarness = "dart_benchmark_harness", } +/** Counts for a single iteration of a report. */ +export interface JsonReportIterationCounts { + /** The number of benchmarks in this iteration. */ + benchmarks: number; + /** The number of distinct measures in this iteration. */ + measures: number; +} + +/** Counts for the alerts of a report. */ +export interface JsonReportAlertsCounts { + /** The total number of alerts. */ + total: number; + /** The number of active alerts. */ + active: number; +} + +/** Counts for a report. */ +export interface JsonReportCounts { + /** Counts for the report results, one entry per iteration. */ + results: JsonReportIterationCounts[]; + /** Counts for the report alerts. */ + alerts: JsonReportAlertsCounts; +} + export interface JsonReport { uuid: Uuid; user?: JsonPubUser; @@ -1065,8 +1089,18 @@ export interface JsonReport { start_time: string; end_time: string; adapter: Adapter; - results: JsonReportResults; - alerts: JsonReportAlerts; + /** + * The report results, one list of results per iteration. + * Omitted by default from the reports list endpoint; use the `expand` query param to include them. + */ + results?: JsonReportResults; + /** + * The report alerts. + * Omitted by default from the reports list endpoint; use the `expand` query param to include them. + */ + alerts?: JsonReportAlerts; + /** The report counts. */ + counts?: JsonReportCounts; job?: Uuid; created: string; } diff --git a/tasks/test_api/src/task/test/seed_test.rs b/tasks/test_api/src/task/test/seed_test.rs index 7c6835dfa3..66090520b7 100644 --- a/tasks/test_api/src/task/test/seed_test.rs +++ b/tasks/test_api/src/task/test/seed_test.rs @@ -1139,6 +1139,55 @@ impl SeedTest { serde_json::from_slice(&assert.get_output().stdout).unwrap(); assert_eq!(alerts.0.len(), 5); + // cargo run -- report ls --host http://localhost:6610 the-computer + let mut cmd = Command::cargo_bin(BENCHER_CMD)?; + cmd.args(["report", "ls", HOST_ARG, host, PROJECT_SLUG]) + .current_dir(CLI_DIR); + let assert = cmd.assert().success(); + let reports: bencher_json::JsonReports = + serde_json::from_slice(&assert.get_output().stdout).unwrap(); + // The reports list collapses the results and alerts by default + for report in &reports.0 { + assert!( + report.results.is_none(), + "Report results should be collapsed by default" + ); + assert!( + report.alerts.is_none(), + "Report alerts should be collapsed by default" + ); + assert!( + !report.counts.results.is_empty(), + "Report counts should always be included" + ); + } + // The reports are in reverse chronological order, + // so the first report is the one that generated the alerts + let first_report = reports.0.first().unwrap(); + assert_eq!(first_report.counts.alerts.total, 5); + assert_eq!(first_report.counts.alerts.active, 5); + + // cargo run -- report ls --host http://localhost:6610 --expand the-computer + let mut cmd = Command::cargo_bin(BENCHER_CMD)?; + cmd.args(["report", "ls", HOST_ARG, host, "--expand", PROJECT_SLUG]) + .current_dir(CLI_DIR); + let assert = cmd.assert().success(); + let reports: bencher_json::JsonReports = + serde_json::from_slice(&assert.get_output().stdout).unwrap(); + // Expanded reports include the full results and alerts, + // and the counts are consistent with them + for report in &reports.0 { + let results = report.results.as_ref().unwrap(); + assert_eq!(results.len(), report.counts.results.len()); + let alerts = report.alerts.as_ref().unwrap(); + assert_eq!( + u32::try_from(alerts.len()).unwrap(), + report.counts.alerts.total + ); + } + let first_report = reports.0.first().unwrap(); + assert_eq!(first_report.counts.alerts.total, 5); + std::thread::sleep(std::time::Duration::from_secs(1)); // Reset the feature branch