diff --git a/.github/workflows/web-tui.yml b/.github/workflows/web-tui.yml index e292f1fa71..de13941ab9 100644 --- a/.github/workflows/web-tui.yml +++ b/.github/workflows/web-tui.yml @@ -90,7 +90,7 @@ jobs: --no-default-features --features web - name: Deploy at nightlies.a.o - if: ${{ github.event == 'push' && github.ref == 'refs/heads/main' }} + if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} run: | TUI_DIR="datafusion/ballista/tui/${{ steps.cargo_metadata.outputs.ballista_version }}" REMOTE_TARGET_DIR="${{ secrets.NIGHTLIES_RSYNC_PATH }}/${TUI_DIR}/" @@ -101,7 +101,7 @@ jobs: ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }} ${{ secrets.NIGHTLIES_RSYNC_HOST }} "mkdir -p ${REMOTE_TARGET_DIR}" - rsync --times --compress --delete --verbose -e "ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }}" ./target/web-tui/* ${{ secrets.NIGHTLIES_RSYNC_HOST }}:${REMOTE_TARGET_DIR} + rsync --times --compress --delete --recursive --verbose -e "ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} -l ${{ secrets.NIGHTLIES_RSYNC_USER }}" ./target/web-tui/* ${{ secrets.NIGHTLIES_RSYNC_HOST }}:${REMOTE_TARGET_DIR} - name: Upload WASM32 application uses: actions/upload-artifact@v7 diff --git a/Cargo.lock b/Cargo.lock index 74ae6d104f..332ea76b76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1061,7 +1061,6 @@ dependencies = [ "gloo-timers", "js-sys", "mimalloc", - "percent-encoding", "prometheus-parse", "ratatui", "ratzilla", @@ -1075,6 +1074,7 @@ dependencies = [ "tracing-subscriber", "tracing-web", "tui-shimmer", + "url-escape", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -1214,6 +1214,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", + "url-escape", "uuid", ] @@ -7648,6 +7649,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + [[package]] name = "urlencoding" version = "2.1.3" diff --git a/Cargo.toml b/Cargo.toml index 5719b9b16c..5ab5e99fcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ async-trait = { version = "0.1" } serde = { version = "1.0" } tokio-stream = { version = "0.1" } url = { version = "2.5" } +url-escape = { version = "0.1.2" } # cargo build --profile release-lto [profile.release-lto] diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml index 0344a3b5c9..dba61ca682 100644 --- a/ballista-cli/Cargo.toml +++ b/ballista-cli/Cargo.toml @@ -46,7 +46,7 @@ config = { version = "0.15.23", default-features = false, features = ["yaml"], o dirs = "6.0" dotparser = { version = "0.3.0", optional = true } futures = { version = "0.3.31", optional = true } -percent-encoding = { version = "2.3.2", optional = true } +url-escape = { workspace = true, optional = true } prometheus-parse = { version = "0.2", optional = true } ratatui = { version = "0.30.1", default-features = false, optional = true } reqwest = { version = "0.13.4", features = ["json"], optional = true } @@ -87,14 +87,14 @@ cli = [ tui = [ "dep:chrono", "dep:config", "dep:crossterm", "dep:dotparser", "dep:futures", - "dep:percent-encoding", "dep:prometheus-parse", "dep:ratatui", "dep:reqwest", + "dep:url-escape", "dep:prometheus-parse", "dep:ratatui", "dep:reqwest", "dep:serde", "dep:serde_json", "dep:tracing-appender", "dep:tui-shimmer", "ratatui/crossterm", "ratatui/all-widgets", "ratatui/macros", "ratatui/layout-cache", "ratatui/underline-color", "ratatui/serde" ] web = [ - "dep:chrono", "dep:config", "dep:critical-section", "dep:dotparser", "dep:futures", "dep:percent-encoding", + "dep:chrono", "dep:config", "dep:critical-section", "dep:dotparser", "dep:futures", "dep:url-escape", "dep:prometheus-parse", "dep:ratatui", "ratatui/serde", "dep:reqwest", "dep:serde", "dep:serde_json", "dep:tui-shimmer", "dep:ratzilla", "dep:wasm-bindgen", "dep:wasm-bindgen-futures", diff --git a/ballista-cli/src/tui/http_client.rs b/ballista-cli/src/tui/http_client.rs index bccabec647..a315c734a3 100644 --- a/ballista-cli/src/tui/http_client.rs +++ b/ballista-cli/src/tui/http_client.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use percent_encoding::{NON_ALPHANUMERIC, percent_encode}; use reqwest::{Client, Response}; use serde::de::DeserializeOwned; #[cfg(not(target_arch = "wasm32"))] @@ -213,6 +212,6 @@ impl HttpClient { } fn url_encode(&self, job_id: &str) -> String { - percent_encode(job_id.as_bytes(), NON_ALPHANUMERIC).to_string() + url_escape::encode_path(job_id).to_string() } } diff --git a/ballista-cli/src/tui/infrastructure/config.rs b/ballista-cli/src/tui/infrastructure/config.rs index 2d64f0710e..ffba3dbb46 100644 --- a/ballista-cli/src/tui/infrastructure/config.rs +++ b/ballista-cli/src/tui/infrastructure/config.rs @@ -114,9 +114,9 @@ mod web { pub(super) fn parse() -> File { let mut data_reload_interval_ms = 2000; let mut repaint_interval_ms = 50; - let mut scheduler_url = "http://localhost:50050"; + let mut scheduler_url = String::from("http://localhost:50050"); let mut http_timeout_ms = 2000; - let mut theme_name = "dark"; + let mut theme_name = String::from("dark"); let mut theme_app_background_bg = String::from("black"); let query_string = Self::decode_request(); @@ -131,13 +131,18 @@ mod web { "ballista_repaint_interval_ms" => { repaint_interval_ms = value.parse::().unwrap_or(50) } - "ballista_scheduler_url" => scheduler_url = value, + "ballista_scheduler_url" => { + scheduler_url = url_escape::decode(value).to_string() + } "ballista_http_timeout" => { http_timeout_ms = value.parse::().unwrap_or(2000) } - "ballista_theme_name" => theme_name = value, + "ballista_theme_name" => { + theme_name = url_escape::decode(value).to_string() + } "ballista_theme_overrides_app_background_bg" => { - theme_app_background_bg = value.replace("%23", "#"); + theme_app_background_bg = + url_escape::decode(value).to_string(); } _ => {} } diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml index d2012b3469..9d4a1fc23e 100644 --- a/ballista/scheduler/Cargo.toml +++ b/ballista/scheduler/Cargo.toml @@ -77,6 +77,7 @@ tonic-prost = { workspace = true, optional = true } tracing = { workspace = true, optional = true } tracing-appender = { workspace = true, optional = true } tracing-subscriber = { workspace = true, optional = true } +url-escape = { workspace = true } uuid = { workspace = true } [[test]] diff --git a/ballista/scheduler/src/api/handlers.rs b/ballista/scheduler/src/api/handlers.rs index a0e1a60d57..9261137ea4 100644 --- a/ballista/scheduler/src/api/handlers.rs +++ b/ballista/scheduler/src/api/handlers.rs @@ -17,6 +17,7 @@ use crate::state::execution_graph_dot::ExecutionGraphDot; use crate::state::execution_stage::TaskInfo; use crate::{api::SchedulerErrorResponse, scheduler_server::SchedulerServer}; use axum::extract::Query; +use axum::response::Redirect; use axum::{ Json, extract::{Path, State}, @@ -46,8 +47,9 @@ use graphviz_rust::{ exec, printer::PrinterContext, }; -use http::{StatusCode, header::CONTENT_TYPE}; +use http::{HeaderMap, StatusCode, header::CONTENT_TYPE}; use serde::Serialize; +use std::collections::HashMap; use std::sync::Arc; use std::time::Duration; @@ -230,6 +232,58 @@ pub enum PlanFormat { Metrics, } +/// A handler for GET requests to the root (`/`). +/// It redirects to `https://nightlies.apache.org/datafusion/ballista/tui//` +/// forwarding any query parameters +pub async fn get_webtui< + T: AsLogicalPlan + Clone + Send + Sync + 'static, + U: AsExecutionPlan + Send + Sync + 'static, +>( + header_map: HeaderMap, + Query(mut params): Query>, + State(data_server): State>>, +) -> Result { + const NIGHTLIES_URL: &str = "https://nightlies.apache.org/datafusion/ballista/tui"; + let external_host = &data_server.state.config.external_host; + let bind_port = data_server.state.config.bind_port; + + let ballista_scheduler_url = + params.remove("ballista_scheduler_url").unwrap_or_else(|| { + let proto = header_map + .get("x-forwarded-proto") + .and_then(|v| v.to_str().ok()) + .unwrap_or("http"); + let host = header_map + .get("x-forwarded-host") + .and_then(|hv| hv.to_str().ok()) + .unwrap_or(external_host); + let port = header_map + .get("x-forwarded-port") + .and_then(|hv| hv.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .unwrap_or(bind_port); + format!("{proto}://{host}:{port}") + }); + + let mut query_string = String::new(); + query_string.push_str(&format!( + "ballista_scheduler_url={}", + url_escape::encode_query(&ballista_scheduler_url) + )); + + for (k, v) in params.iter() { + query_string.push_str(&format!( + "&{}={}", + url_escape::encode_query(k), + url_escape::encode_query(v) + )); + } + + let target = format!("{NIGHTLIES_URL}/{BALLISTA_VERSION}/?{query_string}"); + + Ok(Redirect::to(&target)) +} + pub async fn get_scheduler_state< T: AsLogicalPlan + Clone + Send + Sync + 'static, U: AsExecutionPlan + Send + Sync + 'static, @@ -549,7 +603,7 @@ pub async fn get_query_stages< let metrics = running_stage.stage_metrics.as_deref().unwrap_or(&[]); summary.stage_plan = Some(match plan_format { PlanFormat::Default => displayable(running_stage.plan.as_ref()).indent(false).to_string(), - PlanFormat::Tree => displayable(running_stage.plan.as_ref()).tree_render().to_string(), + PlanFormat::Tree => displayable(running_stage.plan.as_ref()).tree_render().to_string(), PlanFormat::Metrics => format_stage_metrics(running_stage.plan.as_ref(), metrics), }); summary.input_rows = running_stage @@ -599,7 +653,7 @@ pub async fn get_query_stages< finish_time: info.finish_time as u64, input_rows, output_rows, - status: task_status + status: task_status, }) }) .collect(); @@ -607,7 +661,7 @@ pub async fn get_query_stages< ExecutionStage::Successful(completed_stage) => { summary.stage_plan = Some(match plan_format { PlanFormat::Default => displayable(completed_stage.plan.as_ref()).indent(false).to_string(), - PlanFormat::Tree => displayable(completed_stage.plan.as_ref()).tree_render().to_string(), + PlanFormat::Tree => displayable(completed_stage.plan.as_ref()).tree_render().to_string(), PlanFormat::Metrics => format_stage_metrics(completed_stage.plan.as_ref(), &completed_stage.stage_metrics), }); summary.input_rows = get_combined_count( @@ -648,7 +702,7 @@ pub async fn get_query_stages< finish_time: task_info.finish_time as u64, input_rows, output_rows, - status: task_status + status: task_status, }) }) .collect(); @@ -923,7 +977,7 @@ pub async fn get_job_dot_graph< })? { ExecutionGraphDot::generate(graph.as_ref()) - .map_err(|e| { + .map_err(|e| { tracing::error!("Error occurred while getting the dot graph for job '{job_id}' reason: {e:?}"); SchedulerErrorResponse::new(StatusCode::INTERNAL_SERVER_ERROR) }) @@ -968,10 +1022,10 @@ pub async fn get_job_svg_graph< &mut PrinterContext::default(), vec![CommandArg::Format(Format::Svg)], ) - .map_err(|e| { - tracing::error!("Error occurred while getting job svg graph for job '{job_id}' reason: {e:?}"); - SchedulerErrorResponse::new(StatusCode::INTERNAL_SERVER_ERROR) - })?; + .map_err(|e| { + tracing::error!("Error occurred while getting job svg graph for job '{job_id}' reason: {e:?}"); + SchedulerErrorResponse::new(StatusCode::INTERNAL_SERVER_ERROR) + })?; let svg = String::from_utf8_lossy(&result).to_string(); Ok(Response::builder() diff --git a/ballista/scheduler/src/api/routes.rs b/ballista/scheduler/src/api/routes.rs index 76f2b78518..6a3341934c 100644 --- a/ballista/scheduler/src/api/routes.rs +++ b/ballista/scheduler/src/api/routes.rs @@ -27,7 +27,10 @@ pub fn get_routes< >( scheduler_server: Arc>, ) -> Router { + let web_tui_route = scheduler_server.state.config.web_tui_route.clone(); + let router = Router::new() + .route(&web_tui_route, get(handlers::get_webtui)) .route("/api/state", get(handlers::get_scheduler_state::)) .route("/api/version", get(handlers::get_scheduler_version)) .route("/api/executors", get(handlers::get_executors::)) diff --git a/ballista/scheduler/src/config.rs b/ballista/scheduler/src/config.rs index 940888714b..1cb40c26bd 100644 --- a/ballista/scheduler/src/config.rs +++ b/ballista/scheduler/src/config.rs @@ -48,13 +48,26 @@ pub struct Config { )] pub advertise_flight_sql_endpoint: Option, /// Namespace for the ballista cluster. - #[arg(short = 'n', long, default_value_t = String::from("ballista"), help = "Namespace for the ballista cluster that this executor will join.")] + #[arg( + short = 'n', + long, + default_value_t = String::from("ballista"), + help = "Namespace for the ballista cluster that this executor will join." + )] pub namespace: String, /// Local host name or IP address to bind to. - #[arg(long, default_value_t = String::from("0.0.0.0"), help = "Local host name or IP address to bind to.")] + #[arg( + long, + default_value_t = String::from("0.0.0.0"), + help = "Local host name or IP address to bind to." + )] pub bind_host: String, /// External host name for executors to connect to. - #[arg(long, default_value_t = String::from("localhost"), help = "Host name or IP address so that executors can connect to this scheduler.")] + #[arg( + long, + default_value_t = String::from("localhost"), + help = "Host name or IP address so that executors can connect to this scheduler." + )] pub external_host: String, /// Port to bind the scheduler gRPC service. #[arg( @@ -218,6 +231,14 @@ pub struct Config { help = "Comma-separated list of allowed methods for CORS. By default, GET, PATCH, and OPTIONS are allowed." )] pub cors_allowed_methods: String, + #[cfg(feature = "rest-api")] + /// The HTTP path that will redirect to the WebTUI app at `https://nightlies.apache.org` + #[arg( + long, + default_value_t = String::from("/"), + help = "The HTTP path that will redirect to the WebTUI app at https://nightlies.apache.org." + )] + pub web_tui_route: String, } /// Configurations for the ballista scheduler of scheduling jobs and tasks @@ -287,6 +308,9 @@ pub struct SchedulerConfig { #[cfg(feature = "rest-api")] /// Comma-separated list of allowed methods for CORS pub cors_allowed_methods: String, + #[cfg(feature = "rest-api")] + /// The HTTP path that will redirect to the WebTUI app at `https://nightlies.apache.org` + pub web_tui_route: String, } impl Default for SchedulerConfig { @@ -324,6 +348,8 @@ impl Default for SchedulerConfig { cors_allowed_origins: String::default(), #[cfg(feature = "rest-api")] cors_allowed_methods: String::default(), + #[cfg(feature = "rest-api")] + web_tui_route: String::from("/"), } } } @@ -557,6 +583,8 @@ impl TryFrom for SchedulerConfig { cors_allowed_origins: opt.cors_allowed_origins, #[cfg(feature = "rest-api")] cors_allowed_methods: opt.cors_allowed_methods, + #[cfg(feature = "rest-api")] + web_tui_route: opt.web_tui_route, }; Ok(config) diff --git a/python/Cargo.lock b/python/Cargo.lock index d5d6b2b1a6..6877c33a3f 100644 --- a/python/Cargo.lock +++ b/python/Cargo.lock @@ -587,6 +587,7 @@ dependencies = [ "tokio-stream", "tonic", "tower-http 0.7.0", + "url-escape", "uuid", ] @@ -4343,6 +4344,15 @@ dependencies = [ "serde", ] +[[package]] +name = "url-escape" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd4105a4fc511f6b1c9616a372a1313ae7c15afcaac839617c1c7cf25ce0a965" +dependencies = [ + "percent-encoding", +] + [[package]] name = "utf8_iter" version = "1.0.4"