From a448fe9a75d48739707f177fb54bd14efaed24ee Mon Sep 17 00:00:00 2001 From: pmqueiroz Date: Mon, 18 May 2026 01:38:31 -0300 Subject: [PATCH] feat: implement kitty graphics protocol --- src/core/grid.rs | 29 +++++ src/sys/kitty_graphics.rs | 190 +++++++++++++++++++++++++++++++ src/sys/mod.rs | 1 + src/ui/app_state/update/input.rs | 147 +++++++++++++++++++++++- src/ui/app_state/update/tabs.rs | 2 + src/ui/components/term.rs | 62 ++++++++-- src/ui/tab.rs | 7 ++ 7 files changed, 425 insertions(+), 13 deletions(-) create mode 100644 src/sys/kitty_graphics.rs diff --git a/src/core/grid.rs b/src/core/grid.rs index d4dba41..b6c104b 100644 --- a/src/core/grid.rs +++ b/src/core/grid.rs @@ -6,6 +6,15 @@ use std::sync::Arc; pub const DEFAULT_SCROLLBACK_LIMIT: usize = 10_000; +pub struct PlacedImage { + pub id: u32, + pub row: usize, + pub col: usize, + pub pixel_width: u32, + pub pixel_height: u32, + pub rgba: Vec, +} + bitflags! { #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct CellAttrs: u8 { @@ -86,6 +95,7 @@ pub struct Grid { pub scrollback_limit: usize, pub cursor_visible: bool, pub kitty_keyboard_flags_stack: Vec, + pub images: Vec, alt_cells: Option>, alt_cursor: Option<(usize, usize)>, alt_scrollback: Option, bool)>>, @@ -134,6 +144,7 @@ impl Grid { scrollback_limit: DEFAULT_SCROLLBACK_LIMIT, cursor_visible: true, kitty_keyboard_flags_stack: Vec::new(), + images: Vec::new(), alt_cells: None, alt_cursor: None, alt_scrollback: None, @@ -157,6 +168,7 @@ impl Grid { pub fn enter_alt_screen(&mut self) { if self.alt_cells.is_none() { + self.images.clear(); self.alt_cells = Some(self.cells.clone()); self.alt_cursor = Some((self.cursor_x, self.cursor_y)); self.alt_scrollback = Some(std::mem::take(&mut self.scrollback)); @@ -177,6 +189,7 @@ impl Grid { } pub fn leave_alt_screen(&mut self) { + self.images.clear(); if let Some(cells) = self.alt_cells.take() { self.cells = cells; } @@ -226,6 +239,15 @@ impl Grid { } self.row_continuation[bottom] = false; } + + self.images.retain_mut(|img| { + if img.row < count { + false + } else { + img.row -= count; + true + } + }); } pub fn scroll_down(&mut self, n: usize) { @@ -245,6 +267,12 @@ impl Grid { } self.row_continuation[top] = false; } + + let rows = self.rows; + self.images.retain_mut(|img| { + img.row += count; + img.row < rows + }); } pub fn newline(&mut self) { @@ -300,6 +328,7 @@ impl Grid { } self.cells = vec![Cell::default(); new_cols * new_rows]; + self.images.clear(); self.cols = new_cols; self.rows = new_rows; self.scroll_top = 0; diff --git a/src/sys/kitty_graphics.rs b/src/sys/kitty_graphics.rs new file mode 100644 index 0000000..7f79236 --- /dev/null +++ b/src/sys/kitty_graphics.rs @@ -0,0 +1,190 @@ +use base64::Engine; + +#[derive(Default)] +pub enum ApcState { + #[default] + None, + SawEsc, + Collecting(Vec), + CollectingSawEsc(Vec), +} + +impl ApcState { + pub fn advance(&mut self, byte: u8) -> (Vec, Option>) { + match self { + ApcState::None => { + if byte == 0x1b { + *self = ApcState::SawEsc; + (vec![], None) + } else { + (vec![byte], None) + } + } + ApcState::SawEsc => { + if byte == b'_' { + *self = ApcState::Collecting(Vec::new()); + (vec![], None) + } else { + let pass = vec![0x1b, byte]; + *self = ApcState::None; + (pass, None) + } + } + ApcState::Collecting(buf) => { + if byte == 0x1b { + *self = ApcState::CollectingSawEsc(std::mem::take(buf)); + (vec![], None) + } else if byte == 0x9c { + let content = std::mem::take(buf); + *self = ApcState::None; + (vec![], Some(content)) + } else { + buf.push(byte); + (vec![], None) + } + } + ApcState::CollectingSawEsc(buf) => { + if byte == b'\\' { + let content = std::mem::take(buf); + *self = ApcState::None; + (vec![], Some(content)) + } else if byte == 0x1b { + buf.push(0x1b); + (vec![], None) + } else { + let mut new_buf = std::mem::take(buf); + new_buf.push(0x1b); + new_buf.push(byte); + *self = ApcState::Collecting(new_buf); + (vec![], None) + } + } + } + } +} + +pub struct PendingKittyImage { + pub format: u32, + pub width: u32, + pub height: u32, + pub id: u32, + pub chunks: Vec>, +} + +pub struct KittyApcCommand { + pub action: u8, + pub format: u32, + pub width: u32, + pub height: u32, + pub id: u32, + pub more: bool, + pub quiet: u8, + pub data: Vec, +} + +pub fn parse_kitty_apc(content: &[u8]) -> Option { + if content.first() != Some(&b'G') { + return None; + } + let rest = &content[1..]; + + let (keys_bytes, data_bytes) = match rest.iter().position(|&b| b == b';') { + Some(sep) => (&rest[..sep], &rest[sep + 1..]), + None => (rest, &[] as &[u8]), + }; + + let keys_str = std::str::from_utf8(keys_bytes).ok()?; + + let mut action = b'T'; + let mut format = 32u32; + let mut width = 0u32; + let mut height = 0u32; + let mut id = 0u32; + let mut more = false; + let mut quiet = 0u8; + let mut medium = b'd'; + + for kv in keys_str.split(',') { + let mut parts = kv.splitn(2, '='); + let key = match parts.next() { + Some(k) => k.trim(), + None => continue, + }; + let val = parts.next().unwrap_or("").trim(); + match key { + "a" => action = val.bytes().next().unwrap_or(b'T'), + "f" => format = val.parse().unwrap_or(32), + "s" => width = val.parse().unwrap_or(0), + "v" => height = val.parse().unwrap_or(0), + "i" => id = val.parse().unwrap_or(0), + "m" => more = val == "1", + "q" => quiet = val.parse().unwrap_or(0), + "t" => medium = val.bytes().next().unwrap_or(b'd'), + _ => {} + } + } + + if medium != b'd' { + return None; + } + + Some(KittyApcCommand { + action, + format, + width, + height, + id, + more, + quiet, + data: data_bytes.to_vec(), + }) +} + +pub fn decode_kitty_image( + prior_chunks: &[Vec], + final_data: &[u8], + format: u32, + width: u32, + height: u32, +) -> Option<(Vec, u32, u32)> { + let mut combined = Vec::new(); + for chunk in prior_chunks { + combined.extend_from_slice(chunk); + } + combined.extend_from_slice(final_data); + + let raw = base64::engine::general_purpose::STANDARD + .decode(&combined) + .ok() + .or_else(|| { + base64::engine::general_purpose::STANDARD_NO_PAD + .decode(&combined) + .ok() + })?; + + match format { + 32 => { + if width == 0 || height == 0 || raw.len() != (width * height * 4) as usize { + return None; + } + Some((raw, width, height)) + } + 24 => { + if width == 0 || height == 0 || raw.len() != (width * height * 3) as usize { + return None; + } + let rgba: Vec = raw + .chunks_exact(3) + .flat_map(|p| [p[0], p[1], p[2], 255u8]) + .collect(); + Some((rgba, width, height)) + } + 100 => { + let img = image::load_from_memory(&raw).ok()?.to_rgba8(); + let w = img.width(); + let h = img.height(); + Some((img.into_raw(), w, h)) + } + _ => None, + } +} diff --git a/src/sys/mod.rs b/src/sys/mod.rs index f3dc061..422acc2 100644 --- a/src/sys/mod.rs +++ b/src/sys/mod.rs @@ -1,4 +1,5 @@ pub mod bell; +pub mod kitty_graphics; pub mod notification; pub mod parser; pub mod pty; diff --git a/src/ui/app_state/update/input.rs b/src/ui/app_state/update/input.rs index 91ba3c3..19d1794 100644 --- a/src/ui/app_state/update/input.rs +++ b/src/ui/app_state/update/input.rs @@ -2,7 +2,8 @@ use std::io::Write; use std::sync::atomic::Ordering; -use crate::core::grid::ControlCommand; +use crate::core::grid::{ControlCommand, PlacedImage}; +use crate::sys::kitty_graphics::{self, PendingKittyImage}; use crate::sys::parser::AnsiExecutor; use crate::sys::pty::PtyCommand; use crate::ui::components; @@ -11,6 +12,107 @@ use super::super::helpers::{command_history_path, os_name}; use super::super::message::Message; use super::super::nova::{AI_OPEN, Nova}; +fn process_kitty_apc( + content: &[u8], + grid: &mut crate::core::grid::Grid, + pending: &mut Option, + font_size: f32, +) { + let Some(cmd) = kitty_graphics::parse_kitty_apc(content) else { + return; + }; + + match cmd.action { + b'T' | b't' => { + if cmd.more { + match pending { + Some(p) => p.chunks.push(cmd.data.clone()), + None => { + *pending = Some(PendingKittyImage { + format: cmd.format, + width: cmd.width, + height: cmd.height, + id: cmd.id, + chunks: vec![cmd.data.clone()], + }); + } + } + } else { + let prior = pending.take(); + let format = if cmd.format != 0 { + cmd.format + } else { + prior.as_ref().map_or(32, |p| p.format) + }; + let width = if cmd.width != 0 { + cmd.width + } else { + prior.as_ref().map_or(0, |p| p.width) + }; + let height = if cmd.height != 0 { + cmd.height + } else { + prior.as_ref().map_or(0, |p| p.height) + }; + let image_id = if cmd.id != 0 { + cmd.id + } else { + prior.as_ref().map_or(0, |p| p.id) + }; + let prior_chunks: Vec> = prior.map(|p| p.chunks).unwrap_or_default(); + + if let Some((rgba, pw, ph)) = + kitty_graphics::decode_kitty_image(&prior_chunks, &cmd.data, format, width, height) + { + let line_height = (font_size * 1.29).max(1.0); + let term_rows = ((ph as f32 / line_height).ceil() as usize).max(1); + + if image_id != 0 { + grid.images.retain(|i| i.id != image_id); + } + + grid.images.push(PlacedImage { + id: image_id, + row: grid.cursor_y, + col: grid.cursor_x, + pixel_width: pw, + pixel_height: ph, + rgba, + }); + + let new_y = grid.cursor_y + term_rows; + if new_y >= grid.rows { + let scroll_n = new_y - grid.rows + 1; + grid.scroll_up(scroll_n); + grid.cursor_y = grid.rows.saturating_sub(1); + } else { + grid.cursor_y = new_y; + } + grid.cursor_x = 0; + + if cmd.quiet < 2 { + let id_part = if image_id != 0 { + format!(",i={}", image_id) + } else { + String::new() + }; + let resp = format!("\x1b_Ga=T{};OK\x1b\\", id_part).into_bytes(); + grid.output_queue.push(resp); + } + } + } + } + b'd' => { + if cmd.id != 0 { + grid.images.retain(|i| i.id != cmd.id); + } else { + grid.images.clear(); + } + } + _ => {} + } +} + impl Nova { pub(super) fn handle_type_input(&mut self, bytes: Vec) -> iced::Task { if self.settings_open || self.command_palette_open || self.ai_overlay_open || self.ai_loading { @@ -179,6 +281,7 @@ impl Nova { } pub(super) fn handle_pty_output(&mut self, tab_id: usize, bytes: Vec) -> iced::Task { + let font_size = self.settings.theme.font.size; if let Some(tab_idx) = self .tabs .iter() @@ -195,11 +298,30 @@ impl Nova { { let _ = f.write_all(&bytes); } + + let mut vte_bytes = Vec::with_capacity(bytes.len()); + let mut completed_apcs: Vec> = Vec::new(); + for byte in &bytes { + let (pass, apc) = split.apc_state.advance(*byte); + vte_bytes.extend_from_slice(&pass); + if let Some(content) = apc { + completed_apcs.push(content); + } + } + for apc_content in &completed_apcs { + process_kitty_apc( + apc_content, + &mut split.grid, + &mut split.pending_kitty, + font_size, + ); + } + let mut executor = AnsiExecutor { grid: &mut split.grid, bell_pending: false, }; - for byte in bytes { + for byte in vte_bytes { split.ansi_parser.advance(&mut executor, &[byte]); } while !split.grid.output_queue.is_empty() { @@ -267,11 +389,30 @@ impl Nova { { let _ = f.write_all(&bytes); } + + let mut vte_bytes = Vec::with_capacity(bytes.len()); + let mut completed_apcs: Vec> = Vec::new(); + for byte in &bytes { + let (pass, apc) = tab.apc_state.advance(*byte); + vte_bytes.extend_from_slice(&pass); + if let Some(content) = apc { + completed_apcs.push(content); + } + } + for apc_content in &completed_apcs { + process_kitty_apc( + apc_content, + &mut tab.grid, + &mut tab.pending_kitty, + font_size, + ); + } + let mut executor = AnsiExecutor { grid: &mut tab.grid, bell_pending: false, }; - for byte in bytes { + for byte in vte_bytes { tab.ansi_parser.advance(&mut executor, &[byte]); } let bell_fired = executor.bell_pending; diff --git a/src/ui/app_state/update/tabs.rs b/src/ui/app_state/update/tabs.rs index c65eb19..f812473 100644 --- a/src/ui/app_state/update/tabs.rs +++ b/src/ui/app_state/update/tabs.rs @@ -144,6 +144,8 @@ impl Nova { scroll_offset: 0, initial_cwd, waiting_after_exit: false, + apc_state: crate::sys::kitty_graphics::ApcState::default(), + pending_kitty: None, }); tab.active_pane_is_split = true; if let Some(split) = &mut tab.split { diff --git a/src/ui/components/term.rs b/src/ui/components/term.rs index 2aeb3b9..b94e86c 100644 --- a/src/ui/components/term.rs +++ b/src/ui/components/term.rs @@ -1,7 +1,7 @@ use iced::{ - Background, Color, Element, Length, Padding, + Background, Color, ContentFit, Element, Length, Padding, widget::{ - column, container, rich_text, row, + column, container, image as iced_image, rich_text, row, space::Space, stack, text::{self, Span}, @@ -357,21 +357,63 @@ pub fn term<'a>( let term_bg = theme::color::runtime().background; - container(grid_ui) + let padding = Padding { + top: 12.0, + right: 20.0, + bottom: 8.0, + left: 20.0, + }; + + let text_layer: Element<'_, Message> = container(grid_ui) .style(move |_| container::Style { background: Some(term_bg.into()), ..Default::default() }) - .padding(Padding { - top: 12.0, - right: 20.0, - bottom: 8.0, - left: 20.0, - }) + .padding(padding) .height(Length::Fill) .width(Length::Fill) .clip(true) - .into() + .into(); + + let visible_images: Vec<_> = grid + .images + .iter() + .filter(|img| img.row < grid.rows.saturating_sub(clamped_offset)) + .collect(); + + if visible_images.is_empty() { + return text_layer; + } + + let mut s = iced::widget::Stack::new(); + s = s.push(text_layer); + + for img in visible_images { + let display_row = clamped_offset + img.row; + let offset_top = padding.top + display_row as f32 * line_height; + let offset_left = padding.left + img.col as f32 * char_width; + + let handle = iced_image::Handle::from_rgba(img.pixel_width, img.pixel_height, img.rgba.clone()); + + let positioned: Element<'_, Message> = column![ + Space::new().height(offset_top), + row![ + Space::new().width(offset_left), + iced_image::Image::new(handle) + .width(img.pixel_width as f32) + .height(img.pixel_height as f32) + .content_fit(ContentFit::Fill), + ] + .height(Length::Shrink), + ] + .width(Length::Fill) + .height(Length::Fill) + .into(); + + s = s.push(positioned); + } + + s.width(Length::Fill).height(Length::Fill).into() } fn cell_span( diff --git a/src/ui/tab.rs b/src/ui/tab.rs index 888ebb2..25a141a 100644 --- a/src/ui/tab.rs +++ b/src/ui/tab.rs @@ -9,6 +9,7 @@ use vte::Parser; use crate::core::config; use crate::core::grid::{DEFAULT_SCROLLBACK_LIMIT, Grid}; +use crate::sys::kitty_graphics::{ApcState, PendingKittyImage}; use crate::sys::pty::PtyCommand; pub struct SplitPane { @@ -28,6 +29,8 @@ pub struct SplitPane { pub scroll_offset: usize, pub initial_cwd: String, pub waiting_after_exit: bool, + pub apc_state: ApcState, + pub pending_kitty: Option, } impl SplitPane { @@ -60,6 +63,8 @@ pub struct Tab { pub last_pty_output: Option, pub waiting_after_exit: bool, pub initial_command: Option, + pub apc_state: ApcState, + pub pending_kitty: Option, } impl Tab { @@ -102,6 +107,8 @@ impl Tab { last_pty_output: None, waiting_after_exit: false, initial_command: config::get().general.initial_command.clone(), + apc_state: ApcState::default(), + pending_kitty: None, } } }