From f6376add6bd6e5f9b6814ad541d318705c550abb Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 14:25:29 +0300 Subject: [PATCH 01/15] feat(scheduler): Redirect '/' to nighlies.a.o WebTUI Opening http://localhost:50050 in a browser will now redirect to the convenience deployment of the WebTUI app at https://nightlies.apache.org/datafusion/ballista/tui/... --- .github/workflows/web-tui.yml | 2 +- ballista/scheduler/src/api/handlers.rs | 53 ++++++++++++++++++++------ ballista/scheduler/src/api/routes.rs | 1 + 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index e292f1fa71..52d41830d0 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -90,7 +90,7 @@ jobs: --no-default-features --features web - name: Deploy at nightlies.a.o - if: ${{ github.event == 'push' && github.ref == 'refs/heads/main' }} + #if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} run: | TUI_DIR="datafusion/ballista/tui/${{ steps.cargo_metadata.outputs.ballista_version }}" REMOTE_TARGET_DIR="${{ secrets.NIGHTLIES_RSYNC_PATH }}/${TUI_DIR}/" diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index a0e1a60d57..5fa60d5000 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -17,6 +17,7 @@ use crate::state::execution_graph_dot::ExecutionGraphDot; use crate::state::execution_stage::TaskInfo; use crate::{api::SchedulerErrorResponse, scheduler_server::SchedulerServer}; use axum::extract::Query; +use axum::response::Redirect; use axum::{ Json, extract::{Path, State}, @@ -46,7 +47,7 @@ use graphviz_rust::{ exec, printer::PrinterContext, }; -use http::{StatusCode, header::CONTENT_TYPE}; +use http::{HeaderMap, StatusCode, header::CONTENT_TYPE}; use serde::Serialize; use std::sync::Arc; use std::time::Duration; @@ -230,6 +231,36 @@ pub enum PlanFormat { Metrics, } +/// A handler for GET requests to the root (`/`). +/// It redirects to https://nightlies.apache.org/datafusion/ballista/tui// +/// forwarding any query parameters +pub async fn get_root( + header_map: HeaderMap, + Query(mut params): Query>, +) -> Result { + const NIGHTLIES_URL: &str = "https://nightlies.apache.org/datafusion/ballista/tui"; + + let ballista_scheduler_url = + params.remove("ballista_scheduler_url").unwrap_or_else(|| { + let default_scheduler_url = "localhost:50050"; + let scheduler_url = header_map + .get("host") + .map(|hv| hv.to_str().unwrap_or(default_scheduler_url)) + .unwrap_or(default_scheduler_url); + format!("http://{scheduler_url}") + }); + + let mut target = format!( + "{NIGHTLIES_URL}/{BALLISTA_VERSION}/?ballista_scheduler_url={ballista_scheduler_url}", + ); + + for (k, v) in params.iter() { + target.push_str(format!("&{}={}", k, v).as_str()); + } + + Ok(Redirect::temporary(&target)) +} + pub async fn get_scheduler_state< T: AsLogicalPlan + Clone + Send + Sync + 'static, U: AsExecutionPlan + Send + Sync + 'static, @@ -549,7 +580,7 @@ pub async fn get_query_stages< let metrics = running_stage.stage_metrics.as_deref().unwrap_or(&[]); summary.stage_plan = Some(match plan_format { PlanFormat::Default => displayable(running_stage.plan.as_ref()).indent(false).to_string(), - PlanFormat::Tree => displayable(running_stage.plan.as_ref()).tree_render().to_string(), + PlanFormat::Tree => displayable(running_stage.plan.as_ref()).tree_render().to_string(), PlanFormat::Metrics => format_stage_metrics(running_stage.plan.as_ref(), metrics), }); summary.input_rows = running_stage @@ -599,7 +630,7 @@ pub async fn get_query_stages< finish_time: info.finish_time as u64, input_rows, output_rows, - status: task_status + status: task_status, }) }) .collect(); @@ -607,7 +638,7 @@ pub async fn get_query_stages< ExecutionStage::Successful(completed_stage) => { summary.stage_plan = Some(match plan_format { PlanFormat::Default => displayable(completed_stage.plan.as_ref()).indent(false).to_string(), - PlanFormat::Tree => displayable(completed_stage.plan.as_ref()).tree_render().to_string(), + PlanFormat::Tree => displayable(completed_stage.plan.as_ref()).tree_render().to_string(), PlanFormat::Metrics => format_stage_metrics(completed_stage.plan.as_ref(), &completed_stage.stage_metrics), }); summary.input_rows = get_combined_count( @@ -648,7 +679,7 @@ pub async fn get_query_stages< finish_time: task_info.finish_time as u64, input_rows, output_rows, - status: task_status + status: task_status, }) }) .collect(); @@ -829,7 +860,7 @@ fn failed_reason(failed: &FailedTask) -> String { Some(TaskKilled(_)) => "TaskKilled", None => "Failed", } - .to_string() + .to_string() } fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option { @@ -923,7 +954,7 @@ pub async fn get_job_dot_graph< })? { ExecutionGraphDot::generate(graph.as_ref()) - .map_err(|e| { + .map_err(|e| { tracing::error!("Error occurred while getting the dot graph for job '{job_id}' reason: {e:?}"); SchedulerErrorResponse::new(StatusCode::INTERNAL_SERVER_ERROR) }) @@ -968,10 +999,10 @@ pub async fn get_job_svg_graph< &mut PrinterContext::default(), vec![CommandArg::Format(Format::Svg)], ) - .map_err(|e| { - tracing::error!("Error occurred while getting job svg graph for job '{job_id}' reason: {e:?}"); - SchedulerErrorResponse::new(StatusCode::INTERNAL_SERVER_ERROR) - })?; + .map_err(|e| { + tracing::error!("Error occurred while getting job svg graph for job '{job_id}' reason: {e:?}"); + SchedulerErrorResponse::new(StatusCode::INTERNAL_SERVER_ERROR) + })?; let svg = String::from_utf8_lossy(&result).to_string(); Ok(Response::builder() diff --git a/ballista/scheduler/src/api/routes.rs b/ballista/scheduler/src/api/routes.rs index 76f2b78518..b3950a48b1 100644 --- a/ballista/scheduler/src/api/routes.rs +++ b/ballista/scheduler/src/api/routes.rs @@ -28,6 +28,7 @@ pub fn get_routes< scheduler_server: Arc>, ) -> Router { let router = Router::new() + .route("/", get(handlers::get_root)) .route("/api/state", get(handlers::get_scheduler_state::)) .route("/api/version", get(handlers::get_scheduler_version)) .route("/api/executors", get(handlers::get_executors::)) From ad9525ded031d555cc6fd80a2b1eafa4b3457b4c Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 14:34:24 +0300 Subject: [PATCH 02/15] Fix rsync delete command: `--delete does not work without --recursive or --dirs` --- .github/workflows/web-tui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 52d41830d0..ceb1b65b13 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -101,7 +101,7 @@ jobs: ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }} ${{ secrets.NIGHTLIES_RSYNC_HOST }} "mkdir -p ${REMOTE_TARGET_DIR}" - rsync --times --compress --delete --verbose -e "ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }}" ./target/web-tui/* ${{ secrets.NIGHTLIES_RSYNC_HOST }}:${REMOTE_TARGET_DIR} + rsync --times --compress --delete --recursive --verbose -e "ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }}" ./target/web-tui/* ${{ secrets.NIGHTLIES_RSYNC_HOST }}:${REMOTE_TARGET_DIR} - name: Upload WASM32 application uses: actions/upload-artifact@v7 From 1cdb88a26c3768d520135196e6077f92a09399a2 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 14:38:16 +0300 Subject: [PATCH 03/15] Re-enable the CI job condition to deploy WebTUI only on push to main --- .github/workflows/web-tui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index ceb1b65b13..de13941ab9 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -90,7 +90,7 @@ jobs: --no-default-features --features web - name: Deploy at nightlies.a.o - #if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} + if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} run: | TUI_DIR="datafusion/ballista/tui/${{ steps.cargo_metadata.outputs.ballista_version }}" REMOTE_TARGET_DIR="${{ secrets.NIGHTLIES_RSYNC_PATH }}/${TUI_DIR}/" From cde3f596185fd3675a778774954255eec16f1fe5 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 15:11:18 +0300 Subject: [PATCH 04/15] URL encode the query parameters Use x-forwarded-proto request header to read the url protocol for Scheduler server --- .github/workflows/web-tui.yml | 2 +- Cargo.lock | 12 +++++++++- Cargo.toml | 1 + ballista-cli/Cargo.toml | 6 ++--- ballista-cli/src/tui/http_client.rs | 3 +-- ballista-cli/src/tui/infrastructure/config.rs | 8 ++++--- ballista/scheduler/Cargo.toml | 1 + ballista/scheduler/src/api/handlers.rs | 24 ++++++++++++++----- 8 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index de13941ab9..77cfb1e2ed 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -90,7 +90,7 @@ jobs: --no-default-features --features web - name: Deploy at nightlies.a.o - if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} + # if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} run: | TUI_DIR="datafusion/ballista/tui/${{ steps.cargo_metadata.outputs.ballista_version }}" REMOTE_TARGET_DIR="${{ secrets.NIGHTLIES_RSYNC_PATH }}/${TUI_DIR}/" diff --git a/Cargo.lock b/Cargo.lock index 74ae6d104f..332ea76b76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1061,7 +1061,6 @@ dependencies = [ "gloo-timers", "js-sys", "mimalloc", - "percent-encoding", "prometheus-parse", "ratatui", "ratzilla", @@ -1075,6 +1074,7 @@ dependencies = [ "tracing-subscriber", "tracing-web", "tui-shimmer", + "url-escape", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -1214,6 +1214,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "url-escape", "uuid", ] @@ -7648,6 +7649,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + [[package]] name = "urlencoding" version = "2.1.3" diff --git a/Cargo.toml b/Cargo.toml index 5719b9b16c..5ab5e99fcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ async-trait = { version = "0.1" } serde = { version = "1.0" } tokio-stream = { version = "0.1" } url = { version = "2.5" } +url-escape = { version = "0.1.2" } # cargo build --profile release-lto [profile.release-lto] diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml index 0344a3b5c9..dba61ca682 100644 --- a/ballista-cli/Cargo.toml +++ b/ballista-cli/Cargo.toml @@ -46,7 +46,7 @@ config = { version = "0.15.23", default-features = false, features = ["yaml"], o dirs = "6.0" dotparser = { version = "0.3.0", optional = true } futures = { version = "0.3.31", optional = true } -percent-encoding = { version = "2.3.2", optional = true } +url-escape = { workspace = true, optional = true } prometheus-parse = { version = "0.2", optional = true } ratatui = { version = "0.30.1", default-features = false, optional = true } reqwest = { version = "0.13.4", features = ["json"], optional = true } @@ -87,14 +87,14 @@ cli = [ tui = [ "dep:chrono", "dep:config", "dep:crossterm", "dep:dotparser", "dep:futures", - "dep:percent-encoding", "dep:prometheus-parse", "dep:ratatui", "dep:reqwest", + "dep:url-escape", "dep:prometheus-parse", "dep:ratatui", "dep:reqwest", "dep:serde", "dep:serde_json", "dep:tracing-appender", "dep:tui-shimmer", "ratatui/crossterm", "ratatui/all-widgets", "ratatui/macros", "ratatui/layout-cache", "ratatui/underline-color", "ratatui/serde" ] web = [ - "dep:chrono", "dep:config", "dep:critical-section", "dep:dotparser", "dep:futures", "dep:percent-encoding", + "dep:chrono", "dep:config", "dep:critical-section", "dep:dotparser", "dep:futures", "dep:url-escape", "dep:prometheus-parse", "dep:ratatui", "ratatui/serde", "dep:reqwest", "dep:serde", "dep:serde_json", "dep:tui-shimmer", "dep:ratzilla", "dep:wasm-bindgen", "dep:wasm-bindgen-futures", diff --git a/ballista-cli/src/tui/http_client.rs b/ballista-cli/src/tui/http_client.rs index bccabec647..a315c734a3 100644 --- a/ballista-cli/src/tui/http_client.rs +++ b/ballista-cli/src/tui/http_client.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use percent_encoding::{NON_ALPHANUMERIC, percent_encode}; use reqwest::{Client, Response}; use serde::de::DeserializeOwned; #[cfg(not(target_arch = "wasm32"))] @@ -213,6 +212,6 @@ impl HttpClient { } fn url_encode(&self, job_id: &str) -> String { - percent_encode(job_id.as_bytes(), NON_ALPHANUMERIC).to_string() + url_escape::encode_path(job_id).to_string() } } diff --git a/ballista-cli/src/tui/infrastructure/config.rs b/ballista-cli/src/tui/infrastructure/config.rs index 2d64f0710e..ab2d4b8eef 100644 --- a/ballista-cli/src/tui/infrastructure/config.rs +++ b/ballista-cli/src/tui/infrastructure/config.rs @@ -131,13 +131,15 @@ mod web { "ballista_repaint_interval_ms" => { repaint_interval_ms = value.parse::().unwrap_or(50) } - "ballista_scheduler_url" => scheduler_url = value, + "ballista_scheduler_url" => { + scheduler_url = url_escape::decode(value) + } "ballista_http_timeout" => { http_timeout_ms = value.parse::().unwrap_or(2000) } - "ballista_theme_name" => theme_name = value, + "ballista_theme_name" => theme_name = url_escape::decode(value), "ballista_theme_overrides_app_background_bg" => { - theme_app_background_bg = value.replace("%23", "#"); + theme_app_background_bg = url_escape::decode(value); } _ => {} } diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index d2012b3469..9d4a1fc23e 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -77,6 +77,7 @@ tonic-prost = { workspace = true, optional = true } tracing = { workspace = true, optional = true } tracing-appender = { workspace = true, optional = true } tracing-subscriber = { workspace = true, optional = true } +url-escape = { workspace = true } uuid = { workspace = true } [[test]] diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 5fa60d5000..72de4947e0 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -232,7 +232,7 @@ pub enum PlanFormat { } /// A handler for GET requests to the root (`/`). -/// It redirects to https://nightlies.apache.org/datafusion/ballista/tui// +/// It redirects to `https://nightlies.apache.org/datafusion/ballista/tui//` /// forwarding any query parameters pub async fn get_root( header_map: HeaderMap, @@ -247,17 +247,29 @@ pub async fn get_root( .get("host") .map(|hv| hv.to_str().unwrap_or(default_scheduler_url)) .unwrap_or(default_scheduler_url); - format!("http://{scheduler_url}") + let proto = header_map + .get("x-forwarded-proto") + .and_then(|v| v.to_str().ok()) + .unwrap_or("http"); + format!("{proto}://{scheduler_url}") }); - let mut target = format!( - "{NIGHTLIES_URL}/{BALLISTA_VERSION}/?ballista_scheduler_url={ballista_scheduler_url}", - ); + let mut query_string = String::new(); + query_string.push_str(&format!( + "ballista_scheduler_url={}", + url_escape::encode_query(&ballista_scheduler_url) + )); for (k, v) in params.iter() { - target.push_str(format!("&{}={}", k, v).as_str()); + query_string.push_str(&format!( + "&{}={}", + url_escape::encode_query(k), + url_escape::encode_query(v) + )); } + let target = format!("{NIGHTLIES_URL}/{BALLISTA_VERSION}/?{query_string}"); + Ok(Redirect::temporary(&target)) } From c6b255f4ff3707f21b96d29c2a657f52308e01f9 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 15:17:28 +0300 Subject: [PATCH 05/15] Use "See Other" redirect (code 303) --- ballista/scheduler/src/api/handlers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 72de4947e0..99072c18cb 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -270,7 +270,7 @@ pub async fn get_root( let target = format!("{NIGHTLIES_URL}/{BALLISTA_VERSION}/?{query_string}"); - Ok(Redirect::temporary(&target)) + Ok(Redirect::to(&target)) } pub async fn get_scheduler_state< From f3cabe251ff476707c968922835014cee978fea9 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 15:18:26 +0300 Subject: [PATCH 06/15] Import std::collections::HashMap --- ballista/scheduler/src/api/handlers.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 99072c18cb..09940a57ee 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -49,6 +49,7 @@ use graphviz_rust::{ }; use http::{HeaderMap, StatusCode, header::CONTENT_TYPE}; use serde::Serialize; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -236,7 +237,7 @@ pub enum PlanFormat { /// forwarding any query parameters pub async fn get_root( header_map: HeaderMap, - Query(mut params): Query>, + Query(mut params): Query>, ) -> Result { const NIGHTLIES_URL: &str = "https://nightlies.apache.org/datafusion/ballista/tui"; From 54a34e6d68075adc2489a961c4b872adcaf18b5c Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 15:24:58 +0300 Subject: [PATCH 07/15] Own the parsed query parameter values --- ballista-cli/src/tui/infrastructure/config.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ballista-cli/src/tui/infrastructure/config.rs b/ballista-cli/src/tui/infrastructure/config.rs index ab2d4b8eef..ffba3dbb46 100644 --- a/ballista-cli/src/tui/infrastructure/config.rs +++ b/ballista-cli/src/tui/infrastructure/config.rs @@ -114,9 +114,9 @@ mod web { pub(super) fn parse() -> File { let mut data_reload_interval_ms = 2000; let mut repaint_interval_ms = 50; - let mut scheduler_url = "http://localhost:50050"; + let mut scheduler_url = String::from("http://localhost:50050"); let mut http_timeout_ms = 2000; - let mut theme_name = "dark"; + let mut theme_name = String::from("dark"); let mut theme_app_background_bg = String::from("black"); let query_string = Self::decode_request(); @@ -132,14 +132,17 @@ mod web { repaint_interval_ms = value.parse::().unwrap_or(50) } "ballista_scheduler_url" => { - scheduler_url = url_escape::decode(value) + scheduler_url = url_escape::decode(value).to_string() } "ballista_http_timeout" => { http_timeout_ms = value.parse::().unwrap_or(2000) } - "ballista_theme_name" => theme_name = url_escape::decode(value), + "ballista_theme_name" => { + theme_name = url_escape::decode(value).to_string() + } "ballista_theme_overrides_app_background_bg" => { - theme_app_background_bg = url_escape::decode(value); + theme_app_background_bg = + url_escape::decode(value).to_string(); } _ => {} } From 824699fc7b51e857416159817b4052e13cb8a1a4 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 15:29:48 +0300 Subject: [PATCH 08/15] Update python/Cargo.lock --- python/Cargo.lock | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/Cargo.lock b/python/Cargo.lock index d5d6b2b1a6..6877c33a3f 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -587,6 +587,7 @@ dependencies = [ "tokio-stream", "tonic", "tower-http 0.7.0", + "url-escape", "uuid", ] @@ -4343,6 +4344,15 @@ dependencies = [ "serde", ] +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + [[package]] name = "utf8_iter" version = "1.0.4" From 97ea732d8af8c2634c1063a6fbe0392e4a808dac Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 16 Jul 2026 15:30:13 +0300 Subject: [PATCH 09/15] Re-enable the CI condition for deployment at nightlies.a.o --- .github/workflows/web-tui.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index 77cfb1e2ed..de13941ab9 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -90,7 +90,7 @@ jobs: --no-default-features --features web - name: Deploy at nightlies.a.o - # if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} + if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} run: | TUI_DIR="datafusion/ballista/tui/${{ steps.cargo_metadata.outputs.ballista_version }}" REMOTE_TARGET_DIR="${{ secrets.NIGHTLIES_RSYNC_PATH }}/${TUI_DIR}/" From 8d9175e44549854cbb181221e7d9e0ec8f4f126f Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 17 Jul 2026 21:03:09 +0300 Subject: [PATCH 10/15] Make it possible to use custom Axum route for the path redirecting to the WebTUI at https://nightlies.apache.org --- ballista/scheduler/src/api/routes.rs | 4 +++- ballista/scheduler/src/config.rs | 33 +++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/ballista/scheduler/src/api/routes.rs b/ballista/scheduler/src/api/routes.rs index b3950a48b1..e951e3644f 100644 --- a/ballista/scheduler/src/api/routes.rs +++ b/ballista/scheduler/src/api/routes.rs @@ -27,8 +27,10 @@ pub fn get_routes< >( scheduler_server: Arc>, ) -> Router { + let web_tui_route = scheduler_server.state.config.web_tui_route.clone(); + let router = Router::new() - .route("/", get(handlers::get_root)) + .route(&web_tui_route, get(handlers::get_root)) .route("/api/state", get(handlers::get_scheduler_state::)) .route("/api/version", get(handlers::get_scheduler_version)) .route("/api/executors", get(handlers::get_executors::)) diff --git a/ballista/scheduler/src/config.rs b/ballista/scheduler/src/config.rs index 940888714b..8e8f45d648 100644 --- a/ballista/scheduler/src/config.rs +++ b/ballista/scheduler/src/config.rs @@ -48,13 +48,26 @@ pub struct Config { )] pub advertise_flight_sql_endpoint: Option, /// Namespace for the ballista cluster. - #[arg(short = 'n', long, default_value_t = String::from("ballista"), help = "Namespace for the ballista cluster that this executor will join.")] + #[arg( + short = 'n', + long, + default_value_t = String::from("ballista"), + help = "Namespace for the ballista cluster that this executor will join." + )] pub namespace: String, /// Local host name or IP address to bind to. - #[arg(long, default_value_t = String::from("0.0.0.0"), help = "Local host name or IP address to bind to.")] + #[arg( + long, + default_value_t = String::from("0.0.0.0"), + help = "Local host name or IP address to bind to." + )] pub bind_host: String, /// External host name for executors to connect to. - #[arg(long, default_value_t = String::from("localhost"), help = "Host name or IP address so that executors can connect to this scheduler.")] + #[arg( + long, + default_value_t = String::from("localhost"), + help = "Host name or IP address so that executors can connect to this scheduler." + )] pub external_host: String, /// Port to bind the scheduler gRPC service. #[arg( @@ -218,6 +231,14 @@ pub struct Config { help = "Comma-separated list of allowed methods for CORS. By default, GET, PATCH, and OPTIONS are allowed." )] pub cors_allowed_methods: String, + #[cfg(feature = "rest-api")] + /// The HTTP path that will redirect to the WebTUI app at https://nightlies.apache.org + #[arg( + long, + default_value_t = String::from("/"), + help = "The HTTP path that will redirect to the WebTUI app at https://nightlies.apache.org." + )] + pub web_tui_route: String, } /// Configurations for the ballista scheduler of scheduling jobs and tasks @@ -287,6 +308,8 @@ pub struct SchedulerConfig { #[cfg(feature = "rest-api")] /// Comma-separated list of allowed methods for CORS pub cors_allowed_methods: String, + /// The HTTP path that will redirect to the WebTUI app at https://nightlies.apache.org + pub web_tui_route: String, } impl Default for SchedulerConfig { @@ -324,6 +347,8 @@ impl Default for SchedulerConfig { cors_allowed_origins: String::default(), #[cfg(feature = "rest-api")] cors_allowed_methods: String::default(), + #[cfg(feature = "rest-api")] + web_tui_route: String::from("/"), } } } @@ -557,6 +582,8 @@ impl TryFrom for SchedulerConfig { cors_allowed_origins: opt.cors_allowed_origins, #[cfg(feature = "rest-api")] cors_allowed_methods: opt.cors_allowed_methods, + #[cfg(feature = "rest-api")] + web_tui_route: opt.web_tui_route, }; Ok(config) From 3371f635217fd137febb6eda87b54e1866a3c1a7 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 17 Jul 2026 21:20:02 +0300 Subject: [PATCH 11/15] Use config's external_host & bind_port as a fallback for the redirect url to WebTUI --- ballista/scheduler/src/api/handlers.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 09940a57ee..7aa1a70091 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -235,15 +235,21 @@ pub enum PlanFormat { /// A handler for GET requests to the root (`/`). /// It redirects to `https://nightlies.apache.org/datafusion/ballista/tui//` /// forwarding any query parameters -pub async fn get_root( +pub async fn get_root< + T: AsLogicalPlan + Clone + Send + Sync + 'static, + U: AsExecutionPlan + Send + Sync + 'static, +>( header_map: HeaderMap, Query(mut params): Query>, + State(data_server): State>>, ) -> Result { const NIGHTLIES_URL: &str = "https://nightlies.apache.org/datafusion/ballista/tui"; + let external_host = data_server.state.config.external_host.clone(); + let bind_port = data_server.state.config.bind_port.clone(); let ballista_scheduler_url = params.remove("ballista_scheduler_url").unwrap_or_else(|| { - let default_scheduler_url = "localhost:50050"; + let default_scheduler_url = &format!("{external_host}:{bind_port}"); let scheduler_url = header_map .get("host") .map(|hv| hv.to_str().unwrap_or(default_scheduler_url)) From b9077055b024c227d2db65c944459bd6ec908b7b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 17 Jul 2026 21:26:29 +0300 Subject: [PATCH 12/15] clippy --- ballista/scheduler/src/api/handlers.rs | 2 +- ballista/scheduler/src/config.rs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 7aa1a70091..43f6513e38 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -245,7 +245,7 @@ pub async fn get_root< ) -> Result { const NIGHTLIES_URL: &str = "https://nightlies.apache.org/datafusion/ballista/tui"; let external_host = data_server.state.config.external_host.clone(); - let bind_port = data_server.state.config.bind_port.clone(); + let bind_port = data_server.state.config.bind_port; let ballista_scheduler_url = params.remove("ballista_scheduler_url").unwrap_or_else(|| { diff --git a/ballista/scheduler/src/config.rs b/ballista/scheduler/src/config.rs index 8e8f45d648..1cb40c26bd 100644 --- a/ballista/scheduler/src/config.rs +++ b/ballista/scheduler/src/config.rs @@ -232,7 +232,7 @@ pub struct Config { )] pub cors_allowed_methods: String, #[cfg(feature = "rest-api")] - /// The HTTP path that will redirect to the WebTUI app at https://nightlies.apache.org + /// The HTTP path that will redirect to the WebTUI app at `https://nightlies.apache.org` #[arg( long, default_value_t = String::from("/"), @@ -308,7 +308,8 @@ pub struct SchedulerConfig { #[cfg(feature = "rest-api")] /// Comma-separated list of allowed methods for CORS pub cors_allowed_methods: String, - /// The HTTP path that will redirect to the WebTUI app at https://nightlies.apache.org + #[cfg(feature = "rest-api")] + /// The HTTP path that will redirect to the WebTUI app at `https://nightlies.apache.org` pub web_tui_route: String, } From 429cac677f4a6d48a2d0282ad567886c61fed027 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 17 Jul 2026 21:53:21 +0300 Subject: [PATCH 13/15] Use X-Forwarded-XYZ headers to read the protocol+host+port Fallback to http://external_host:bind_port --- ballista/scheduler/src/api/handlers.rs | 19 ++++++++++++------- ballista/scheduler/src/api/routes.rs | 2 +- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 43f6513e38..0d7b585fd4 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -235,7 +235,7 @@ pub enum PlanFormat { /// A handler for GET requests to the root (`/`). /// It redirects to `https://nightlies.apache.org/datafusion/ballista/tui//` /// forwarding any query parameters -pub async fn get_root< +pub async fn get_webtui< T: AsLogicalPlan + Clone + Send + Sync + 'static, U: AsExecutionPlan + Send + Sync + 'static, >( @@ -244,21 +244,26 @@ pub async fn get_root< State(data_server): State>>, ) -> Result { const NIGHTLIES_URL: &str = "https://nightlies.apache.org/datafusion/ballista/tui"; - let external_host = data_server.state.config.external_host.clone(); + let external_host = &data_server.state.config.external_host; let bind_port = data_server.state.config.bind_port; let ballista_scheduler_url = params.remove("ballista_scheduler_url").unwrap_or_else(|| { let default_scheduler_url = &format!("{external_host}:{bind_port}"); - let scheduler_url = header_map - .get("host") - .map(|hv| hv.to_str().unwrap_or(default_scheduler_url)) - .unwrap_or(default_scheduler_url); let proto = header_map .get("x-forwarded-proto") .and_then(|v| v.to_str().ok()) .unwrap_or("http"); - format!("{proto}://{scheduler_url}") + let host = header_map + .get("x-forwarded-host") + .and_then(|hv| hv.to_str().ok()) + .unwrap_or(external_host); + let port = header_map + .get("x-forwarded-port") + .and_then(|hv| hv.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(bind_port); + format!("{proto}://{host}:{port}") }); let mut query_string = String::new(); diff --git a/ballista/scheduler/src/api/routes.rs b/ballista/scheduler/src/api/routes.rs index e951e3644f..6a3341934c 100644 --- a/ballista/scheduler/src/api/routes.rs +++ b/ballista/scheduler/src/api/routes.rs @@ -30,7 +30,7 @@ pub fn get_routes< let web_tui_route = scheduler_server.state.config.web_tui_route.clone(); let router = Router::new() - .route(&web_tui_route, get(handlers::get_root)) + .route(&web_tui_route, get(handlers::get_webtui)) .route("/api/state", get(handlers::get_scheduler_state::)) .route("/api/version", get(handlers::get_scheduler_version)) .route("/api/executors", get(handlers::get_executors::)) From 314b08f94ae8e37b6d454cf4289d37a3d857ac95 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 21 Jul 2026 11:20:13 +0300 Subject: [PATCH 14/15] fmt --- ballista/scheduler/src/api/handlers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index 0d7b585fd4..a427794f5e 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -884,7 +884,7 @@ fn failed_reason(failed: &FailedTask) -> String { Some(TaskKilled(_)) => "TaskKilled", None => "Failed", } - .to_string() + .to_string() } fn get_finished_stage_time(task_infos: &[TaskInfo]) -> Option { From 47332e23d282d26d4ca93729ece53b5a810c47d0 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 21 Jul 2026 12:00:51 +0300 Subject: [PATCH 15/15] clippy --- ballista/scheduler/src/api/handlers.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index a427794f5e..9261137ea4 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -249,7 +249,6 @@ pub async fn get_webtui< let ballista_scheduler_url = params.remove("ballista_scheduler_url").unwrap_or_else(|| { - let default_scheduler_url = &format!("{external_host}:{bind_port}"); let proto = header_map .get("x-forwarded-proto") .and_then(|v| v.to_str().ok())