From bc08562d86afa84fa7ff614ea20dce419d6779d3 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 16:09:26 +0800 Subject: [PATCH 01/15] feat(layout): unify tab header actions by width and centralize constants Use ResizeObserver on the tab row instead of tab-group count so panel headers show icon buttons when wide enough and collapse to a more menu when narrow. Consolidate tab and sidebar tuning in layoutConstants.ts. --- src/features/layout/PanelTabActions.tsx | 163 ++++++++++++++++++++++ src/features/layout/PanelTabHeader.tsx | 158 +++------------------ src/features/layout/layoutConstants.ts | 68 +++++++++ src/features/layout/sidebarPanelSize.ts | 30 ++++ src/features/panels/PANEL_CONTRACT.md | 5 +- src/features/viewer/RosViewContent.tsx | 44 ++---- src/index.css | 1 + src/shared/hooks/useElementWidth.test.tsx | 83 +++++++++++ src/shared/hooks/useElementWidth.ts | 27 ++++ 9 files changed, 404 insertions(+), 175 deletions(-) create mode 100644 src/features/layout/PanelTabActions.tsx create mode 100644 src/features/layout/layoutConstants.ts create mode 100644 src/features/layout/sidebarPanelSize.ts create mode 100644 src/shared/hooks/useElementWidth.test.tsx create mode 100644 src/shared/hooks/useElementWidth.ts diff --git a/src/features/layout/PanelTabActions.tsx b/src/features/layout/PanelTabActions.tsx new file mode 100644 index 0000000..0bb596b --- /dev/null +++ b/src/features/layout/PanelTabActions.tsx @@ -0,0 +1,163 @@ +import React from 'react'; +import type { IntlShape } from 'react-intl'; +import { MoreHorizontal, Plus, Settings2, X } from 'lucide-react'; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSub, + DropdownMenuSubContent, + DropdownMenuSubTrigger, + DropdownMenuTrigger, +} from '@/shared/ui/dropdown-menu'; +import { Button, buttonVariants } from '@/shared/ui/button'; +import { cn } from '@/shared/lib/utils'; +import { panelTabDropdownIconRowClass } from './PanelTabAddPanelDefinitionsSubmenus'; + +const addPanelMenuContentClassName = 'max-h-[min(24rem,70vh)] overflow-y-auto'; + +const tabIconButtonClassName = cn( + buttonVariants({ variant: 'ghost', size: 'icon' }), + 'ros-dockview-tab-action h-8 w-8 shrink-0 font-normal focus-visible:ring-2 focus-visible:ring-ring/40 focus-visible:ring-offset-2 ring-offset-background [&_svg]:size-3.5', +); + +export interface PanelTabActionsProps { + compact: boolean; + hasSettings: boolean; + onOpenSettings: () => void; + onClose: () => void; + addPanelSubmenus: React.ReactNode; + formatMessage: IntlShape['formatMessage']; +} + +export const PanelTabActions: React.FC = ({ + compact, + hasSettings, + onOpenSettings, + onClose, + addPanelSubmenus, + formatMessage, +}) => { + if (compact) { + return ( + + + + + e.stopPropagation()} + > + {hasSettings && ( + { + onOpenSettings(); + }} + > + + {formatMessage({ id: 'layout.panelTab.openSettings' })} + + )} + + + + {formatMessage({ id: 'layout.panelTab.addPanelSubmenu' })} + + + {addPanelSubmenus} + + + { + onClose(); + }} + > + + {formatMessage({ id: 'layout.panelTab.closePanel' })} + + + + ); + } + + return ( + <> + {hasSettings && ( + + )} + + + + + + e.stopPropagation()} + > + {addPanelSubmenus} + + + + + + ); +}; diff --git a/src/features/layout/PanelTabHeader.tsx b/src/features/layout/PanelTabHeader.tsx index ce9a3d2..0ca1349 100644 --- a/src/features/layout/PanelTabHeader.tsx +++ b/src/features/layout/PanelTabHeader.tsx @@ -1,19 +1,8 @@ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useRef, useState } from 'react'; import type { IDockviewPanelHeaderProps } from 'dockview'; -import { MoreHorizontal, Plus, Settings2, X } from 'lucide-react'; import { useIntl } from 'react-intl'; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuSub, - DropdownMenuSubContent, - DropdownMenuSubTrigger, - DropdownMenuTrigger, -} from '@/shared/ui/dropdown-menu'; -import { Button, buttonVariants } from '@/shared/ui/button'; import { Separator } from '@/shared/ui/separator'; -import { cn } from '@/shared/lib/utils'; +import { useElementWidth } from '@/shared/hooks/useElementWidth'; import type { PanelType } from '../panels/framework'; import { getPanelTypeFromId, listPanelStates, usePanelActions } from '../panels/framework'; import { PANEL_TYPE_MESSAGE_SLUG } from '../panels/framework/panelMessageSlug'; @@ -25,10 +14,9 @@ import { import { WELCOME_PANEL_ID } from './dockviewIds'; import { openDockviewPanel } from './dockviewController'; import { getFoxgloveAdapter, getPanelDefinition, getPanelDefinitions, hasFoxgloveAdapter, hasPanelDefinition } from '../panels/registry'; -import { - PanelTabAddPanelDefinitionsSubmenus, - panelTabDropdownIconRowClass, -} from './PanelTabAddPanelDefinitionsSubmenus'; +import { PanelTabAddPanelDefinitionsSubmenus } from './PanelTabAddPanelDefinitionsSubmenus'; +import { PanelTabActions } from './PanelTabActions'; +import { PANEL_TAB_EXPANDED_MIN_WIDTH_PX } from './layoutConstants'; /** Tab label uses localized panel titles (`panels..defaultTitle`). */ function resolveTabDefaultTitle(panelId: string, dockviewTitle: string): string { @@ -62,13 +50,6 @@ function resolveTabPanelType(panelId: string): PanelType | null { return null; } -const addPanelMenuContentClassName = 'max-h-[min(24rem,70vh)] overflow-y-auto'; - -const tabIconButtonClassName = cn( - buttonVariants({ variant: 'ghost', size: 'icon' }), - 'ros-dockview-tab-action h-8 w-8 shrink-0 font-normal focus-visible:ring-2 focus-visible:ring-ring/40 focus-visible:ring-offset-2 ring-offset-background [&_svg]:size-3.5', -); - export const PanelTabHeader: React.FC = ({ api, containerApi }) => { const { formatMessage } = useIntl(); const panel = containerApi.getPanel(api.id); @@ -84,7 +65,10 @@ export const PanelTabHeader: React.FC = ({ api, conta }, [formatMessage, panelType, defaultTitleFallback]); const actions = usePanelActions(api.id); const isWelcome = api.id === WELCOME_PANEL_ID; - const isCompactTabActions = (panel?.group.panels.length ?? 0) > 1; + const tabRowRef = useRef(null); + const tabWidth = useElementWidth(tabRowRef); + const useCompactTabActions = + tabWidth === undefined ? true : tabWidth < PANEL_TAB_EXPANDED_MIN_WIDTH_PX; const definitions = useMemo( () => getPanelDefinitions().filter((d) => d.type !== 'Unavailable'), [], @@ -147,6 +131,7 @@ export const PanelTabHeader: React.FC = ({ api, conta return (
@@ -169,121 +154,14 @@ export const PanelTabHeader: React.FC = ({ api, conta event.stopPropagation(); }} > - {isCompactTabActions ? ( - - - - - e.stopPropagation()} - > - {actions?.hasSettings && ( - { - openSettings(); - }} - > - - {formatMessage({ id: 'layout.panelTab.openSettings' })} - - )} - - - - {formatMessage({ id: 'layout.panelTab.addPanelSubmenu' })} - - - {addPanelSubmenus} - - - { - closePanel(); - }} - > - - {formatMessage({ id: 'layout.panelTab.closePanel' })} - - - - ) : ( - <> - {actions?.hasSettings && ( - - )} - - - - - - e.stopPropagation()} - > - {addPanelSubmenus} - - - - - - )} +
)} diff --git a/src/features/layout/layoutConstants.ts b/src/features/layout/layoutConstants.ts new file mode 100644 index 0000000..fbf14bb --- /dev/null +++ b/src/features/layout/layoutConstants.ts @@ -0,0 +1,68 @@ +/** + * Layout tuning constants + * ======================= + * + * Single source of truth for viewer shell dimensions: Dockview tab headers, + * resizable sidebar bounds, and related responsive breakpoints. + * + * **How to change values** + * - Tab actions: adjust {@link PANEL_TAB_EXPANDED_MIN_WIDTH_PX}. + * - Sidebar drag limits: adjust `SIDEBAR_*_WIDTH_PX`; panel min/max/default + * percentages are derived from {@link LAYOUT_REFERENCE_VIEWPORT_WIDTH_PX}. + * - Sidebar compact UI: adjust {@link SIDEBAR_COMPACT_MAX_WIDTH_PX} and keep + * `src/index.css` `@container sidebar (max-width: …)` in sync. + * + * @see PanelTabHeader — tab action compact / expanded toggle + * @see RosViewContent — resizable sidebar panel + * @see Sidebar — `[container-name:sidebar]` container queries + */ + +// --------------------------------------------------------------------------- +// Dockview panel tab header +// --------------------------------------------------------------------------- + +/** + * Minimum tab row width (px) to show settings, add-panel, and close as icon + * buttons. Below this width, actions collapse into a single “more” menu. + */ +export const PANEL_TAB_EXPANDED_MIN_WIDTH_PX = 180; + +// --------------------------------------------------------------------------- +// Resizable sidebar (RosViewContent ↔ Sidebar) +// --------------------------------------------------------------------------- + +/** Default sidebar width on a {@link LAYOUT_REFERENCE_VIEWPORT_WIDTH_PX} viewport. */ +export const SIDEBAR_DEFAULT_WIDTH_PX = 288; + +/** Minimum draggable sidebar width (px). */ +export const SIDEBAR_MIN_WIDTH_PX = 240; + +/** Maximum draggable sidebar width (px). */ +export const SIDEBAR_MAX_WIDTH_PX = 520; + +/** + * Reference viewport width used to convert sidebar px bounds into + * `ResizablePanel` percentage min/max/default values. Percentages scale with + * the actual window while preserving the intended px limits at this width. + */ +export const LAYOUT_REFERENCE_VIEWPORT_WIDTH_PX = 1280; + +function sidebarWidthToPanelPercent(widthPx: number): number { + return (widthPx / LAYOUT_REFERENCE_VIEWPORT_WIDTH_PX) * 100; +} + +/** Default sidebar size as a fraction of the main horizontal split (percent). */ +export const SIDEBAR_DEFAULT_PANEL_PERCENT = sidebarWidthToPanelPercent(SIDEBAR_DEFAULT_WIDTH_PX); + +/** Minimum sidebar size for the resizable split (percent). */ +export const SIDEBAR_MIN_PANEL_PERCENT = sidebarWidthToPanelPercent(SIDEBAR_MIN_WIDTH_PX); + +/** Maximum sidebar size for the resizable split (percent). */ +export const SIDEBAR_MAX_PANEL_PERCENT = sidebarWidthToPanelPercent(SIDEBAR_MAX_WIDTH_PX); + +/** + * Sidebar container-query breakpoint (px). Below this inline width, non-essential + * sidebar chrome is hidden. Must match `@container sidebar (max-width: …)` in + * `src/index.css`. + */ +export const SIDEBAR_COMPACT_MAX_WIDTH_PX = 320; diff --git a/src/features/layout/sidebarPanelSize.ts b/src/features/layout/sidebarPanelSize.ts new file mode 100644 index 0000000..07a5e17 --- /dev/null +++ b/src/features/layout/sidebarPanelSize.ts @@ -0,0 +1,30 @@ +import { readPreferences } from '@/core/preferences/readWritePreferences'; +import type { PreferencePersistence } from '@/core/preferences/types'; +import { + LAYOUT_REFERENCE_VIEWPORT_WIDTH_PX, + SIDEBAR_DEFAULT_PANEL_PERCENT, + SIDEBAR_MAX_PANEL_PERCENT, + SIDEBAR_MIN_PANEL_PERCENT, +} from './layoutConstants'; + +export function clampSidebarPanelPercent(value: number): number { + return Math.min(SIDEBAR_MAX_PANEL_PERCENT, Math.max(SIDEBAR_MIN_PANEL_PERCENT, value)); +} + +export function getInitialSidebarPanelPercent(persistence: PreferencePersistence): number { + if (persistence !== 'localStorage') { + return SIDEBAR_DEFAULT_PANEL_PERCENT; + } + const preferences = readPreferences(); + if (preferences?.sidebarPanelPercent !== undefined) { + return clampSidebarPanelPercent(preferences.sidebarPanelPercent); + } + if (preferences?.sidebarWidth !== undefined) { + const viewportWidth = + typeof globalThis === 'undefined' || !('innerWidth' in globalThis) || !globalThis.innerWidth + ? LAYOUT_REFERENCE_VIEWPORT_WIDTH_PX + : globalThis.innerWidth; + return clampSidebarPanelPercent((preferences.sidebarWidth / viewportWidth) * 100); + } + return SIDEBAR_DEFAULT_PANEL_PERCENT; +} diff --git a/src/features/panels/PANEL_CONTRACT.md b/src/features/panels/PANEL_CONTRACT.md index a25e399..3312dd9 100644 --- a/src/features/panels/PANEL_CONTRACT.md +++ b/src/features/panels/PANEL_CONTRACT.md @@ -72,9 +72,10 @@ Panels may bring their own components instead. - Panel rendering goes through `PanelRuntimeShell` which seeds the config store and registers tab-header actions. - Each panel is wrapped by `PanelErrorBoundary`. -- Standardised tab-header actions: +- Standardised tab-header actions (implemented in `PanelTabHeader` / `PanelTabActions`): - Gear icon → opens Sidebar `Settings` tab (only when `renderSettings` is provided) - - `⋮` menu → `Reset panel`, `Copy panel ID`, `Duplicate panel`, `Close` + - Add panel (`+`) and Close (`×`) icon buttons when the tab row is wide enough (see `PANEL_TAB_EXPANDED_MIN_WIDTH_PX` in `src/features/layout/layoutConstants.ts`); otherwise collapsed into a single “more” dropdown — independent of how many tabs share the DockView group + - Right-click context menu → `Reset panel`, `Copy panel ID`, `Duplicate panel`, `Close` ## State and export rules diff --git a/src/features/viewer/RosViewContent.tsx b/src/features/viewer/RosViewContent.tsx index 326e5d6..af59561 100644 --- a/src/features/viewer/RosViewContent.tsx +++ b/src/features/viewer/RosViewContent.tsx @@ -23,36 +23,14 @@ import type { RosViewExtension } from '@/core/extensions/types'; import { buildExtensionContext } from '@/core/extensions/buildContext'; import type { FoxgloveLayoutData } from '@/core/preferences/foxgloveLayout'; import type { OpenPanelInput } from '@/features/layout/dockviewController'; - -const DEFAULT_SIDEBAR_WIDTH = 288; -const MIN_SIDEBAR_WIDTH = 240; -const MAX_SIDEBAR_WIDTH = 520; -const DEFAULT_LAYOUT_WIDTH = 1280; -const DEFAULT_SIDEBAR_PANEL_PERCENT = (DEFAULT_SIDEBAR_WIDTH / DEFAULT_LAYOUT_WIDTH) * 100; -const MIN_SIDEBAR_PANEL_PERCENT = (MIN_SIDEBAR_WIDTH / DEFAULT_LAYOUT_WIDTH) * 100; -const MAX_SIDEBAR_PANEL_PERCENT = (MAX_SIDEBAR_WIDTH / DEFAULT_LAYOUT_WIDTH) * 100; - -function clampSidebarPanelPercent(value: number): number { - return Math.min(MAX_SIDEBAR_PANEL_PERCENT, Math.max(MIN_SIDEBAR_PANEL_PERCENT, value)); -} - -function getInitialSidebarPanelPercent(persistence: PreferencePersistence): number { - if (persistence !== 'localStorage') { - return DEFAULT_SIDEBAR_PANEL_PERCENT; - } - const preferences = readPreferences(); - if (preferences?.sidebarPanelPercent !== undefined) { - return clampSidebarPanelPercent(preferences.sidebarPanelPercent); - } - if (preferences?.sidebarWidth !== undefined) { - const viewportWidth = - typeof globalThis === 'undefined' || !('innerWidth' in globalThis) || !globalThis.innerWidth - ? DEFAULT_LAYOUT_WIDTH - : globalThis.innerWidth; - return clampSidebarPanelPercent((preferences.sidebarWidth / viewportWidth) * 100); - } - return DEFAULT_SIDEBAR_PANEL_PERCENT; -} +import { + SIDEBAR_MAX_PANEL_PERCENT, + SIDEBAR_MIN_PANEL_PERCENT, +} from '@/features/layout/layoutConstants'; +import { + clampSidebarPanelPercent, + getInitialSidebarPanelPercent, +} from '@/features/layout/sidebarPanelSize'; interface RosViewContentProps { player: Player; @@ -327,8 +305,8 @@ export const RosViewContent: React.FC = ({ id="sidebar" className="flex h-full min-h-0 min-w-0 flex-col" defaultSize={`${sidebarPanelPercent}%`} - minSize={`${MIN_SIDEBAR_PANEL_PERCENT}%`} - maxSize={`${MAX_SIDEBAR_PANEL_PERCENT}%`} + minSize={`${SIDEBAR_MIN_PANEL_PERCENT}%`} + maxSize={`${SIDEBAR_MAX_PANEL_PERCENT}%`} > = ({ id="main" className="flex h-full min-h-0 min-w-0 flex-col" defaultSize={showSidebar ? `${100 - sidebarPanelPercent}%` : '100%'} - minSize={showSidebar ? `${100 - MAX_SIDEBAR_PANEL_PERCENT}%` : '100%'} + minSize={showSidebar ? `${100 - SIDEBAR_MAX_PANEL_PERCENT}%` : '100%'} >
{ + let container: HTMLDivElement; + let root: Root; + let resizeCallback: ResizeObserverCallback | undefined; + let observedElement: Element | undefined; + + beforeEach(() => { + resizeCallback = undefined; + observedElement = undefined; + vi.stubGlobal( + 'ResizeObserver', + class { + constructor(callback: ResizeObserverCallback) { + resizeCallback = callback; + } + observe(element: Element) { + observedElement = element; + } + disconnect = vi.fn(); + }, + ); + container = document.createElement('div'); + document.body.appendChild(container); + root = createRoot(container); + }); + + afterEach(() => { + act(() => { + root.unmount(); + }); + container.remove(); + vi.unstubAllGlobals(); + }); + + it('returns undefined before ResizeObserver reports a width', () => { + let width: number | undefined; + + function Probe() { + const ref = useRef(null); + width = useElementWidth(ref); + return
; + } + + act(() => { + root.render(); + }); + + expect(width).toBeUndefined(); + expect(observedElement).toBeDefined(); + }); + + it('updates width when ResizeObserver fires', () => { + const widths: Array = []; + + function Probe() { + const ref = useRef(null); + const width = useElementWidth(ref); + widths.push(width); + return
; + } + + act(() => { + root.render(); + }); + + act(() => { + resizeCallback?.( + [{ contentRect: { width: 240 } as DOMRectReadOnly } as ResizeObserverEntry], + {} as ResizeObserver, + ); + }); + + expect(widths.at(-1)).toBe(240); + }); +}); diff --git a/src/shared/hooks/useElementWidth.ts b/src/shared/hooks/useElementWidth.ts new file mode 100644 index 0000000..bba1fd8 --- /dev/null +++ b/src/shared/hooks/useElementWidth.ts @@ -0,0 +1,27 @@ +import { useEffect, useState, type RefObject } from 'react'; + +/** Observes an element's content width via ResizeObserver. Returns undefined until mounted. */ +export function useElementWidth(ref: RefObject): number | undefined { + const [width, setWidth] = useState(undefined); + + useEffect(() => { + const element = ref.current; + if (!element || typeof ResizeObserver === 'undefined') { + return; + } + + const observer = new ResizeObserver((entries) => { + const entry = entries[0]; + if (entry) { + setWidth(entry.contentRect.width); + } + }); + + observer.observe(element); + return () => { + observer.disconnect(); + }; + }, [ref]); + + return width; +} From ea1ee3ca043c0e5cba3903ac6d5fee4c646c5fc7 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 16:23:23 +0800 Subject: [PATCH 02/15] refactor(db3): replace deprecated @foxglove/sql.js with official sql.js Drop @foxglove/rosbag2-web and add a local SqliteDb adapter on top of @foxglove/rosbag2 so db3 parsing no longer pulls in the unmaintained Foxglove sql.js fork. --- NOTICE | 4 +- README.md | 2 +- README.zh.md | 2 +- docs/ARCHITECTURE.md | 8 +- docs/ARCHITECTURE.zh.md | 13 +- package-lock.json | 52 +++--- package.json | 4 +- src/features/viewer/RosViewerImpl.tsx | 2 +- src/infra/sources/RosDb3IterableSource.ts | 7 +- src/infra/sources/SqliteSqljsDb.ts | 214 ++++++++++++++++++++++ 10 files changed, 265 insertions(+), 43 deletions(-) create mode 100644 src/infra/sources/SqliteSqljsDb.ts diff --git a/NOTICE b/NOTICE index 5999abf..1aec4eb 100644 --- a/NOTICE +++ b/NOTICE @@ -10,10 +10,10 @@ THIRD-PARTY SOFTWARE NOTICES This product depends on third-party packages. Their licenses are reproduced below or can be found in each package's LICENSE file under node_modules/. -@foxglove/rosbag, @foxglove/rosbag2-web, @foxglove/rosmsg, +@foxglove/rosbag, @foxglove/rosbag2, @foxglove/rosmsg, @foxglove/rosmsg-serialization, @foxglove/rosmsg2-serialization, @foxglove/message-definition, @foxglove/omgidl-parser, -@foxglove/omgidl-serialization, @foxglove/ros2idl-parser, @foxglove/sql.js +@foxglove/omgidl-serialization, @foxglove/ros2idl-parser, sql.js Copyright 2021-2024 Foxglove Technologies Inc. Licensed under the MIT License. https://github.com/foxglove diff --git a/README.md b/README.md index d730201..d2ae422 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ export function FileLoader() { |--------|-----------|-------| | MCAP | `.mcap` | ROS 2 / robotics standard; zstd and lz4 compression | | ROS 1 bag | `.bag` | ROS 1 recording format | -| ROS 2 SQLite | `.db3` | ROS 2 default recording (via `@foxglove/sql.js` WASM) | +| ROS 2 SQLite | `.db3` | ROS 2 default recording (via `sql.js` WASM) | | HDF5 | `.h5`, `.hdf5` | Scientific data; partial read via `@ioai/hdf5` | | BVH | `.bvh` | Skeletal motion capture animation | diff --git a/README.zh.md b/README.zh.md index b4a7a7b..5371d20 100644 --- a/README.zh.md +++ b/README.zh.md @@ -155,7 +155,7 @@ export function FileLoader() { |------|--------|------| | MCAP | `.mcap` | ROS 2 / 机器人常用；zstd、lz4 压缩 | | ROS 1 bag | `.bag` | ROS 1 录制格式 | -| ROS 2 SQLite | `.db3` | ROS 2 默认录制（`@foxglove/sql.js` WASM） | +| ROS 2 SQLite | `.db3` | ROS 2 默认录制（`sql.js` WASM） | | HDF5 | `.h5`, `.hdf5` | 科学数据；`@ioai/hdf5` WASM 读取 | | BVH | `.bvh` | 动作捕捉骨骼动画 | diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1be4891..65e661f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -16,7 +16,7 @@ ROS View is a **browser-native** playback and visualization tool for robotics re |--------|-------------| | **MCAP** | Preferred format. Chunk-indexed for efficient range queries. | | **ROS1 `.bag`** | Chunk-indexed binary log format. | -| **ROS2 `.db3`** | SQLite-based format via `@foxglove/sql.js` WASM. | +| **ROS2 `.db3`** | SQLite-based format via `sql.js` WASM. | | **HDF5** | via `@ioai/hdf5` WASM. | | **BVH** | Motion-capture skeleton animation. | @@ -55,7 +55,7 @@ ROS View is a **browser-native** playback and visualization tool for robotics re |--------|---------|---------------|-------| | **MCAP** | `@mcap/core` + `@foxglove/mcap-support` | Full (chunk index) | Preferred; efficient interval queries | | **ROS1 `.bag`** | `@foxglove/rosbag` | Full (chunk index) | via `CachedFilelike` + `BrowserHttpReader` | -| **ROS2 `.db3`** | `@foxglove/rosbag2-web` + `@foxglove/sql.js` (WASM) | Local / remote requires full download | SQLite limitation; convert to MCAP for large remote files | +| **ROS2 `.db3`** | `@foxglove/rosbag2` + `sql.js` (WASM) | Local / remote requires full download | SQLite limitation; convert to MCAP for large remote files | **Supported message encodings:** @@ -242,12 +242,12 @@ Fixed-width sidebar (collapsible) on the left with three tabs: | `@mcap/core` | MCAP file format read/write core (indexed + streaming) | | `@foxglove/mcap-support` | MCAP channel parsing, schema-to-JS bridging, decompression handlers | | `@foxglove/rosbag` | ROS1 `.bag` reader (BlobReader + remote CachedFilelike) | -| `@foxglove/rosbag2-web` | ROS2 `.db3` reader (`@foxglove/sql.js` WASM) | +| `@foxglove/rosbag2` | ROS2 `.db3` reader (SQLite queries + CDR decode) | | `@foxglove/rosmsg` | ROS message definition parsing | | `@foxglove/rosmsg-serialization` | ROS1 message deserialization | | `@foxglove/rosmsg2-serialization` | ROS2 CDR deserialization | | `comlink` ^4.4 | Web Worker bidirectional RPC (Transferable support) | -| `@foxglove/sql.js` | SQLite WASM runtime (for `.db3` files) | +| `sql.js` | SQLite WASM runtime (for `.db3` files) | ### 4.5 Visualization diff --git a/docs/ARCHITECTURE.zh.md b/docs/ARCHITECTURE.zh.md index b5b23df..4d1c59c 100644 --- a/docs/ARCHITECTURE.zh.md +++ b/docs/ARCHITECTURE.zh.md @@ -47,7 +47,7 @@ |------|--------|------------|------| | **MCAP** | `@mcap/core` + `@foxglove/mcap-support` | 完整支持（索引读取） | 首选格式，有 chunk 索引可做高效区间查询 | | **ROS1 .bag** | `@foxglove/rosbag` | 完整支持（chunk 索引） | 通过 `CachedFilelike` + `BrowserHttpReader` 实现 | -| **ROS2 .db3** | `@foxglove/rosbag2-web` + `@foxglove/sql.js` (WASM) | 本地支持 / 远程需整文件下载 | SQLite 格式限制，远程大文件建议转 MCAP | +| **ROS2 .db3** | `@foxglove/rosbag2` + `sql.js` (WASM) | 本地支持 / 远程需整文件下载 | SQLite 格式限制，远程大文件建议转 MCAP | | **HDF5** | `@ioai/hdf5` (WASM) | 部分读取 | 科学数据；浏览器内解析 | | **BVH** | 内置解析 | 不适用（非 ROS bag 类流） | 骨骼动作捕捉动画回放 | @@ -235,12 +235,12 @@ | `@mcap/core` | MCAP 文件格式读写核心（索引读取 + 流式读取） | | `@foxglove/mcap-support` | MCAP 通道解析、Schema 到 JS 对象的桥接、解压处理器 | | `@foxglove/rosbag` | ROS1 .bag 文件读取（支持 BlobReader + 远程 CachedFilelike） | -| `@foxglove/rosbag2-web` | ROS2 .db3 文件读取（基于 `@foxglove/sql.js` WASM） | +| `@foxglove/rosbag2` | ROS2 .db3 文件读取（SQLite 查询 + CDR 解码） | | `@foxglove/rosmsg` | ROS 消息定义解析 | | `@foxglove/rosmsg-serialization` | ROS1 消息反序列化 | | `@foxglove/rosmsg2-serialization` | ROS2 CDR 消息反序列化 | | `comlink` ^4.4 | Web Worker 双向 RPC（支持 Transferable） | -| `@foxglove/sql.js` | SQLite WASM 运行时（用于 .db3 文件读取） | +| `sql.js` | SQLite WASM 运行时（用于 .db3 文件读取） | ### 4.5 可视化 @@ -878,11 +878,8 @@ rosview/ "@mcap/core": "^2.3.2", "@foxglove/mcap-support": "workspace:*", "@foxglove/rosbag": "^3.1.2", - "@foxglove/rosbag2-web": "^2.1.2", - "@foxglove/rosmsg": "^5.0.3", - "@foxglove/rosmsg-serialization": "^3.1.1", - "@foxglove/rosmsg2-serialization": "^3.0.3", - "@foxglove/sql.js": "^0.0.4", + "@foxglove/rosbag2": "^6.0.0", + "sql.js": "^1.14.1", "comlink": "^4.4.2", diff --git a/package-lock.json b/package-lock.json index 39fabad..a42719f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@ioai/rosview", - "version": "1.2.5", + "version": "1.2.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ioai/rosview", - "version": "1.2.5", + "version": "1.2.6", "license": "MIT", "devDependencies": { "@eslint/js": "^9.39.4", @@ -15,7 +15,7 @@ "@foxglove/omgidl-serialization": "^1.2.3", "@foxglove/ros2idl-parser": "^0.3.6", "@foxglove/rosbag": "^0.4.1", - "@foxglove/rosbag2-web": "^5.0.0", + "@foxglove/rosbag2": "^6.0.0", "@foxglove/rosmsg": "^5.0.5", "@foxglove/rosmsg-serialization": "^2.0.4", "@foxglove/rosmsg2-serialization": "^3.0.3", @@ -38,6 +38,7 @@ "@types/node": "^25.6.0", "@types/react": "^19.2.15", "@types/react-dom": "^19.2.3", + "@types/sql.js": "^1.4.11", "@types/three": "^0.173.0", "@vitejs/plugin-react": "^6.0.1", "autoprefixer": "^10.4.23", @@ -69,6 +70,7 @@ "react-resizable-panels": "^4.10.0", "rollup": "^4.52.5", "sonner": "^2.0.7", + "sql.js": "^1.14.1", "tailwind-merge": "^3.5.0", "tailwindcss": "^3.4.17", "three": "^0.173.0", @@ -1167,17 +1169,6 @@ "js-yaml": "^4.1.0" } }, - "node_modules/@foxglove/rosbag2-web": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@foxglove/rosbag2-web/-/rosbag2-web-5.0.0.tgz", - "integrity": "sha512-umV4KDKMgz2pm9D+Y6+Qu4HvfS3YvGHR7hzLecwwr42hM826ldyMgAHDCGvTIUa/fu6yWdsEdN/VxuXi4pcCwQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@foxglove/rosbag2": "^6.0.0", - "@foxglove/sql.js": "^0.0.4" - } - }, "node_modules/@foxglove/rosmsg": { "version": "5.0.5", "resolved": "https://registry.npmjs.org/@foxglove/rosmsg/-/rosmsg-5.0.5.tgz", @@ -1252,14 +1243,6 @@ "tslib": "^2" } }, - "node_modules/@foxglove/sql.js": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/@foxglove/sql.js/-/sql.js-0.0.4.tgz", - "integrity": "sha512-yqt0OAjlX2q8ZZcG3fTKHtA/5vdcaEjlWkkdh98pqtApJs9AqMuWWjxOU1SX1cF6D2LgH39sFcIq9y+FWIl6Ew==", - "deprecated": "This package is no longer maintained", - "dev": true, - "license": "MIT" - }, "node_modules/@humanfs/core": { "version": "0.19.2", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.2.tgz", @@ -3630,6 +3613,13 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.41.5", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.41.5.tgz", + "integrity": "sha512-cMQm7pxu6BxtHyqJ7mQZ2kXWV5SLmugybFdHCBbJ5eHzOo6VhBckEgAT3//rP5FwPHNPeEiq4SmQ5ucBwsOo4Q==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -3691,6 +3681,17 @@ "@types/react": "*" } }, + "node_modules/@types/sql.js": { + "version": "1.4.11", + "resolved": "https://registry.npmjs.org/@types/sql.js/-/sql.js-1.4.11.tgz", + "integrity": "sha512-QXIx38p2ZThJaK9vP5ZdqdlRe1FG9I8SmCZOS7FHfB/2qPAjZwkL7/vlfPg6N/oWHuuOaGg/P/IRwfP2W0kWVQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/emscripten": "*", + "@types/node": "*" + } + }, "node_modules/@types/stats.js": { "version": "0.17.4", "resolved": "https://registry.npmjs.org/@types/stats.js/-/stats.js-0.17.4.tgz", @@ -7516,6 +7517,13 @@ "dev": true, "license": "BSD-3-Clause" }, + "node_modules/sql.js": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.14.1.tgz", + "integrity": "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==", + "dev": true, + "license": "MIT" + }, "node_modules/stackback": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", diff --git a/package.json b/package.json index 7f5aa26..863757a 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "@foxglove/omgidl-serialization": "^1.2.3", "@foxglove/ros2idl-parser": "^0.3.6", "@foxglove/rosbag": "^0.4.1", - "@foxglove/rosbag2-web": "^5.0.0", + "@foxglove/rosbag2": "^6.0.0", "@foxglove/rosmsg": "^5.0.5", "@foxglove/rosmsg-serialization": "^2.0.4", "@foxglove/rosmsg2-serialization": "^3.0.3", @@ -115,6 +115,7 @@ "@types/node": "^25.6.0", "@types/react": "^19.2.15", "@types/react-dom": "^19.2.3", + "@types/sql.js": "^1.4.11", "@types/three": "^0.173.0", "@vitejs/plugin-react": "^6.0.1", "autoprefixer": "^10.4.23", @@ -146,6 +147,7 @@ "react-resizable-panels": "^4.10.0", "rollup": "^4.52.5", "sonner": "^2.0.7", + "sql.js": "^1.14.1", "tailwind-merge": "^3.5.0", "tailwindcss": "^3.4.17", "three": "^0.173.0", diff --git a/src/features/viewer/RosViewerImpl.tsx b/src/features/viewer/RosViewerImpl.tsx index ca3057f..41b20e5 100644 --- a/src/features/viewer/RosViewerImpl.tsx +++ b/src/features/viewer/RosViewerImpl.tsx @@ -65,7 +65,7 @@ import { resolveEmbedChrome, type RosViewerChrome, type RosViewerMode } from './ import { RosViewerLayoutProvider } from './RosViewerLayoutContext'; import type { RosViewExtension } from '@/core/extensions/types'; import { toast } from 'sonner'; -import sqlWasmUrl from "@foxglove/sql.js/dist/sql-wasm.wasm?url"; +import sqlWasmUrl from 'sql.js/dist/sql-wasm.wasm?url'; let sqlWasmBinaryPromise: Promise | null = null; diff --git a/src/infra/sources/RosDb3IterableSource.ts b/src/infra/sources/RosDb3IterableSource.ts index 20ca5dc..cce4293 100644 --- a/src/infra/sources/RosDb3IterableSource.ts +++ b/src/infra/sources/RosDb3IterableSource.ts @@ -1,4 +1,5 @@ -import { ROS2_TO_DEFINITIONS, Rosbag2, SqliteSqljs } from "@foxglove/rosbag2-web"; +import { ROS2_TO_DEFINITIONS, Rosbag2 } from '@foxglove/rosbag2'; +import { SqliteSqljsDb } from './SqliteSqljsDb'; import { stringify } from "@foxglove/rosmsg"; import type { Initialization, MessageEvent, RosDatatypes, Time, TopicInfo, TopicStats } from '@/core/types/ros'; import type { GetAdjacentMessageArgs, IIterableSource } from "./IIterableSource"; @@ -51,11 +52,11 @@ export class RosDb3IterableSource implements IIterableSource { } async initialize(): Promise { - await SqliteSqljs.Initialize({ + await SqliteSqljsDb.initialize({ ...(this._sqlWasmBinary ? { wasmBinary: this._sqlWasmBinary } : {}), }); - const dbs = this._params.files.map((file) => new SqliteSqljs(file)); + const dbs = this._params.files.map((file) => new SqliteSqljsDb(file)); const bag = new Rosbag2(dbs, { timeType: "sec,nsec" }); await bag.open(); this._bag = bag; diff --git a/src/infra/sources/SqliteSqljsDb.ts b/src/infra/sources/SqliteSqljsDb.ts new file mode 100644 index 0000000..09e6316 --- /dev/null +++ b/src/infra/sources/SqliteSqljsDb.ts @@ -0,0 +1,214 @@ +import { + type MessageReadOptions, + type MessageRow, + RawMessageIterator, + parseQosProfiles, + type RawMessage, + type SqliteDb, + type TopicDefinition, +} from '@foxglove/rosbag2'; +import { fromNanoSec, toNanoSec, type Time } from '@foxglove/rostime'; +import initSqlJs, { type Database, type SqlJsStatic, type Statement } from 'sql.js'; + +type DbContext = { + db: Database; + idToTopic: Map; + topicNameToId: Map; +}; + +type TopicRowArray = [ + id: number, + name: string, + type: string, + serialization_format: string, + offered_qos_profiles?: string, +]; + +type MessageRowArray = [topic_id: number, timestamp: string, data: Uint8Array]; + +export class SqliteSqljsDb implements SqliteDb { + #file?: Readonly; + #data?: Readonly; + #context?: DbContext; + + static #sqlInitialization?: Promise; + + static async initialize(config?: Partial): Promise { + if (SqliteSqljsDb.#sqlInitialization) { + return await SqliteSqljsDb.#sqlInitialization; + } + + SqliteSqljsDb.#sqlInitialization = initSqlJs(config); + return await SqliteSqljsDb.#sqlInitialization; + } + + constructor(data: File | Uint8Array) { + if (data instanceof File) { + this.#file = data; + } else if (data instanceof Uint8Array) { + this.#data = data; + } + } + + async open(): Promise { + const SQL = await SqliteSqljsDb.initialize(); + + let db: Database; + if (this.#file) { + const buffer = await this.#file.arrayBuffer(); + db = new SQL.Database(new Uint8Array(buffer)); + } else if (this.#data) { + db = new SQL.Database(this.#data); + } else { + db = new SQL.Database(); + } + + const idToTopic = new Map(); + const topicNameToId = new Map(); + const topicRows = (db.exec('select * from topics')[0]?.values ?? []) as TopicRowArray[]; + for (const row of topicRows) { + const [id, name, type, serializationFormat, qosProfilesStr] = row; + const offeredQosProfiles = parseQosProfiles(qosProfilesStr ?? '[]'); + const topic = { name, type, serializationFormat, offeredQosProfiles }; + const bigintId = BigInt(id); + idToTopic.set(bigintId, topic); + topicNameToId.set(name, bigintId); + } + + this.#context = { db, idToTopic, topicNameToId }; + } + + async close(): Promise { + if (this.#context != undefined) { + this.#context.db.close(); + this.#context = undefined; + } + } + + async readTopics(): Promise { + if (this.#context == undefined) { + throw new Error('Call open() before reading topics'); + } + return Array.from(this.#context.idToTopic.values()); + } + + readMessages(opts: MessageReadOptions = {}): AsyncIterableIterator { + if (this.#context == undefined) { + throw new Error('Call open() before reading messages'); + } + const db = this.#context.db; + const topicNameToId = this.#context.topicNameToId; + + let args: (string | number)[] = []; + let query = 'select topic_id,cast(timestamp as TEXT) as timestamp,data from messages'; + if (opts.startTime != undefined) { + query += ' where timestamp >= cast(? as INTEGER)'; + args.push(toNanoSec(opts.startTime).toString()); + } + if (opts.endTime != undefined) { + if (args.length === 0) { + query += ' where timestamp < cast(? as INTEGER)'; + } else { + query += ' and timestamp < cast(? as INTEGER)'; + } + args.push(toNanoSec(opts.endTime).toString()); + } + if (opts.topics != undefined) { + const topicIds: number[] = []; + for (const topicName of opts.topics) { + const topicId = topicNameToId.get(topicName); + if (topicId != undefined) { + topicIds.push(Number(topicId)); + } + } + + if (topicIds.length === 0) { + if (args.length === 0) { + query += ' where topic_id = NULL'; + } else { + query += ' and topic_id = NULL'; + } + } else if (topicIds.length === 1) { + if (args.length === 0) { + query += ' where topic_id = ?'; + } else { + query += ' and topic_id = ?'; + } + args.push(topicIds[0]!); + } else { + if (args.length === 0) { + query += ` where topic_id in (${topicIds.map(() => '?').join(',')})`; + } else { + query += ` and topic_id in (${topicIds.map(() => '?').join(',')})`; + } + args = args.concat(topicIds); + } + } + + const statement = db.prepare(query, args); + const dbIterator = new SqlJsMessageRowIterator(statement); + return new RawMessageIterator(dbIterator, this.#context.idToTopic); + } + + async timeRange(): Promise<[min: Time, max: Time]> { + if (this.#context == undefined) { + throw new Error('Call open() before retrieving the time range'); + } + const db = this.#context.db; + + const res = db.exec( + 'select cast(min(timestamp) as TEXT), cast(max(timestamp) as TEXT) from messages', + )[0]?.values[0] ?? ['0', '0']; + const [minNsec, maxNsec] = res as [string | null, string | null]; + return [fromNanoSec(BigInt(minNsec ?? 0n)), fromNanoSec(BigInt(maxNsec ?? 0n))]; + } + + async messageCounts(): Promise> { + if (this.#context == undefined) { + throw new Error('Call open() before retrieving message counts'); + } + const db = this.#context.db; + + const rows = + db.exec(` + select topics.name,count(*) + from messages + inner join topics on messages.topic_id = topics.id + group by topics.id`)[0]?.values ?? ([] as [string, number][]); + const counts = new Map(); + for (const [topicName, count] of rows) { + counts.set(topicName as string, count as number); + } + return counts; + } +} + +class SqlJsMessageRowIterator implements IterableIterator { + statement: Statement; + + constructor(statement: Statement) { + this.statement = statement; + } + + [Symbol.iterator](): IterableIterator { + return this; + } + + next(): IteratorResult { + if (!this.statement.step()) { + return { value: undefined, done: true }; + } + + const [topic_id, timestamp, data] = this.statement.get() as MessageRowArray; + return { + value: { topic_id: BigInt(topic_id), timestamp: BigInt(timestamp), data }, + done: false, + }; + } + + return(): IteratorResult { + this.statement.freemem(); + this.statement.free(); + return { value: undefined, done: true }; + } +} From 17326a18866bc6a11b711293d09d34dce298131c Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 16:23:28 +0800 Subject: [PATCH 03/15] fix(build): split test TypeScript project to restore npm run build Exclude Vitest files from tsconfig.app and typecheck them in tsconfig.test with Node globals so tsc -b no longer fails on node: imports in test-only code. --- tsconfig.app.json | 3 ++- tsconfig.json | 3 ++- tsconfig.test.json | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 tsconfig.test.json diff --git a/tsconfig.app.json b/tsconfig.app.json index 6e6deab..ab02191 100644 --- a/tsconfig.app.json +++ b/tsconfig.app.json @@ -28,5 +28,6 @@ "erasableSyntaxOnly": true, "noFallthroughCasesInSwitch": true }, - "include": ["src"] + "include": ["src"], + "exclude": ["src/**/*.test.ts", "src/**/*.test.tsx"] } diff --git a/tsconfig.json b/tsconfig.json index 1ffef60..01490aa 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,6 +2,7 @@ "files": [], "references": [ { "path": "./tsconfig.app.json" }, - { "path": "./tsconfig.node.json" } + { "path": "./tsconfig.node.json" }, + { "path": "./tsconfig.test.json" } ] } diff --git a/tsconfig.test.json b/tsconfig.test.json new file mode 100644 index 0000000..12eef17 --- /dev/null +++ b/tsconfig.test.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.app.json", + "compilerOptions": { + "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.test.tsbuildinfo", + "types": ["vitest/globals", "node", "vite/client"] + }, + "include": ["src/**/*.test.ts", "src/**/*.test.tsx", "tests/**/*.ts"] +} From cd02bf1b0809ef73ff457afc830c7c45b8c3c0d5 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 16:26:20 +0800 Subject: [PATCH 04/15] fix(db3): compute topic frequency from per-topic time spans Query min/max timestamps per topic during db3 initialization so sidebar stats match mcap and bag (Hz plus message count). --- src/infra/sources/RosDb3IterableSource.ts | 45 ++++++++++++++++++++++- src/infra/sources/SqliteSqljsDb.ts | 23 ++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/infra/sources/RosDb3IterableSource.ts b/src/infra/sources/RosDb3IterableSource.ts index cce4293..e885bb7 100644 --- a/src/infra/sources/RosDb3IterableSource.ts +++ b/src/infra/sources/RosDb3IterableSource.ts @@ -39,6 +39,33 @@ function toMessageEvent(msg: Rosbag2ReadRow): MessageEvent { }; } +function mergeTopicTimeRange( + existing: [Time, Time] | undefined, + next: [Time, Time], +): [Time, Time] { + if (!existing) return next; + const [minA, maxA] = existing; + const [minB, maxB] = next; + return [ + toNano(minB) < toNano(minA) ? minB : minA, + toNano(maxB) > toNano(maxA) ? maxB : maxA, + ]; +} + +function computeTopicMetrics( + messageCount: number, + start?: Time, + end?: Time, +): { durationSec?: number; frequency: number } { + const durationSec = + start && end && toNano(end) > toNano(start) + ? Number(toNano(end) - toNano(start)) / 1e9 + : undefined; + const frequency = + durationSec != null && durationSec > 0 && messageCount > 1 ? (messageCount - 1) / durationSec : 0; + return { durationSec, frequency: frequency > 0 ? frequency : 0 }; +} + export class RosDb3IterableSource implements IIterableSource { private _params: RosDb3SourceParams; private _sqlWasmBinary?: ArrayBuffer; @@ -65,6 +92,14 @@ export class RosDb3IterableSource implements IIterableSource { const topicDefs = await this._bag.readTopics(); const messageCounts = await this._bag.messageCounts(); + const timeRangeByTopic = new Map(); + for (const db of dbs) { + const ranges = await db.topicTimeRanges(); + for (const [topicName, range] of ranges) { + timeRangeByTopic.set(topicName, mergeTopicTimeRange(timeRangeByTopic.get(topicName), range)); + } + } + const topics: TopicInfo[] = []; const topicStats: Record = {}; const datatypes: RosDatatypes = {}; @@ -79,14 +114,22 @@ export class RosDb3IterableSource implements IIterableSource { for (const topicDef of topicDefs) { const numMessages = messageCounts.get(topicDef.name) ?? 0; + const [topicStart, topicEnd] = timeRangeByTopic.get(topicDef.name) ?? []; + const { durationSec, frequency } = computeTopicMetrics(numMessages, topicStart, topicEnd); const topic: TopicInfo = { name: topicDef.name, type: topicDef.type, messageCount: numMessages, + durationSec, + frequency: frequency > 0 ? frequency : undefined, }; topics.push(topic); - topicStats[topicDef.name] = { messageCount: numMessages, frequency: 0 }; + topicStats[topicDef.name] = { + messageCount: numMessages, + durationSec, + frequency, + }; const parsedMsgdef = ROS2_TO_DEFINITIONS.get(topicDef.type); if (parsedMsgdef) { diff --git a/src/infra/sources/SqliteSqljsDb.ts b/src/infra/sources/SqliteSqljsDb.ts index 09e6316..5d86e6b 100644 --- a/src/infra/sources/SqliteSqljsDb.ts +++ b/src/infra/sources/SqliteSqljsDb.ts @@ -181,6 +181,29 @@ export class SqliteSqljsDb implements SqliteDb { } return counts; } + + async topicTimeRanges(): Promise> { + if (this.#context == undefined) { + throw new Error('Call open() before retrieving topic time ranges'); + } + const db = this.#context.db; + + const rows = + db.exec(` + select topics.name,cast(min(messages.timestamp) as TEXT),cast(max(messages.timestamp) as TEXT) + from messages + inner join topics on messages.topic_id = topics.id + group by topics.id`)[0]?.values ?? []; + const ranges = new Map(); + for (const row of rows) { + const [topicName, minNsec, maxNsec] = row as [string, string | null, string | null]; + ranges.set(topicName, [ + fromNanoSec(BigInt(minNsec ?? 0n)), + fromNanoSec(BigInt(maxNsec ?? 0n)), + ]); + } + return ranges; + } } class SqlJsMessageRowIterator implements IterableIterator { From 6a3cf89077466ba12bff6dc53c95220b068e918f Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 16:44:06 +0800 Subject: [PATCH 05/15] feat(image): add CompressedVideo H264 playback and auto-layout support. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enable foxglove_msgs/msg/CompressedVideo in the Image panel with shared WebCodecs decoding, seek keyframe repair, and 2×3 auto-layout for multi-camera feeds while skipping 3D when no JointState or TF topics exist. --- .../applyDefaultRosDockLayout.test.ts | 43 +++++++-- .../autoLayout/applyDefaultRosDockLayout.ts | 53 ++++++++--- .../autoLayout/imageTopicSelection.test.ts | 12 +++ .../layout/autoLayout/imageTopicSelection.ts | 1 + .../autoLayout/planRosImageGrid.test.ts | 41 ++++++-- .../layout/autoLayout/planRosImageGrid.ts | 38 +++++--- src/features/panels/Image/Component.tsx | 71 +++----------- .../panels/Image/ImagePanelSettings.tsx | 4 +- src/features/panels/Image/definition.tsx | 12 ++- .../panels/Image/image-core/h264.test.ts | 12 +++ .../Image/image-core/h264SeekRepair.test.ts | 54 +++++++++++ .../panels/Image/image-core/h264SeekRepair.ts | 95 +++++++++++++++++++ .../Image/image-core/imageTypes.test.ts | 57 +++++++++++ .../panels/Image/image-core/imageTypes.ts | 58 +++++++++++ .../image-core/messageFrameAdapter.test.ts | 59 ++++++++++++ .../Image/image-core/messageFrameAdapter.ts | 68 +++++++++++++ src/shared/ros/rosMessageTypes.test.ts | 3 + src/shared/ros/rosMessageTypes.ts | 5 +- 18 files changed, 586 insertions(+), 100 deletions(-) create mode 100644 src/features/panels/Image/image-core/h264SeekRepair.test.ts create mode 100644 src/features/panels/Image/image-core/h264SeekRepair.ts create mode 100644 src/features/panels/Image/image-core/messageFrameAdapter.test.ts create mode 100644 src/features/panels/Image/image-core/messageFrameAdapter.ts diff --git a/src/features/layout/autoLayout/applyDefaultRosDockLayout.test.ts b/src/features/layout/autoLayout/applyDefaultRosDockLayout.test.ts index ba477fe..f8df307 100644 --- a/src/features/layout/autoLayout/applyDefaultRosDockLayout.test.ts +++ b/src/features/layout/autoLayout/applyDefaultRosDockLayout.test.ts @@ -29,6 +29,7 @@ describe('buildDefaultRosFoxgloveLayoutData', () => { { name: '/camera/top/depth/z/compressed', type: 'sensor_msgs/msg/CompressedImage' }, { name: '/io/pose/left', type: 'geometry_msgs/msg/PoseStamped [ros2msg]' }, { name: '/io/pose/right', type: 'geometry_msgs/msg/PoseStamped [jsonschema]' }, + { name: '/joint_states', type: 'sensor_msgs/msg/JointState' }, ]; const data = buildDefaultRosFoxgloveLayoutData(topics); @@ -99,7 +100,7 @@ describe('buildDefaultRosFoxgloveLayoutData', () => { expect(getPanelTypeFromId(pose3dRow.second as string)).toBe('3D'); }); - it('creates only full-width 3D row when no PoseStamped topics are present', () => { + it('omits 3D when only image topics are present', () => { const topics: TopicInfo[] = [ { name: '/camera/left/color/a/compressed', type: 'sensor_msgs/msg/CompressedImage' }, { name: '/camera/right/color/a/compressed', type: 'sensor_msgs/msg/CompressedImage' }, @@ -110,7 +111,38 @@ describe('buildDefaultRosFoxgloveLayoutData', () => { const ids = collectMosaicPanelIds(root); const panelTypes = ids.map((id) => getPanelTypeFromId(id)); expect(panelTypes.filter((type) => type === 'Pose')).toHaveLength(0); - expect(panelTypes.filter((type) => type === '3D')).toHaveLength(1); + expect(panelTypes.filter((type) => type === '3D')).toHaveLength(0); + expect(panelTypes.filter((type) => type === 'Image')).toHaveLength(3); + }); + + it('builds two image rows for six CompressedVideo streams without 3D', () => { + const topics: TopicInfo[] = [ + { name: '/head/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/head/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/left/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/left/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/right/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/right/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + ]; + const data = buildDefaultRosFoxgloveLayoutData(topics); + const ids = collectMosaicPanelIds(data.layout); + const panelTypes = ids.map((id) => getPanelTypeFromId(id)); + expect(panelTypes.filter((type) => type === 'Image')).toHaveLength(6); + expect(panelTypes.filter((type) => type === '3D')).toHaveLength(0); + expect(panelTypes.filter((type) => type === 'RawMessages')).toHaveLength(0); + + const imageTopics = Object.values(data.configById) + .map((config) => (config as { topic?: string }).topic) + .filter((topic): topic is string => Boolean(topic)) + .sort(); + expect(imageTopics).toEqual([ + '/head/color/image', + '/head/depth/image', + '/left/color/image', + '/left/depth/image', + '/right/color/image', + '/right/depth/image', + ]); }); it('BVH-only dataset: single 3D panel only; dockview root still wraps to branch for fromJSON', () => { @@ -132,18 +164,17 @@ describe('buildDefaultRosFoxgloveLayoutData', () => { expect(imported.dockviewState?.grid.root.type).toBe('branch'); }); - it('non-BVH single-stack layout still appends RawMessages when only 3D is eligible', () => { + it('non-BVH single-stack layout still appends RawMessages when no panels are eligible', () => { const topics: TopicInfo[] = [{ name: '/scan', type: 'sensor_msgs/msg/LaserScan' }]; const data = buildDefaultRosFoxgloveLayoutData(topics); const root = data.layout as FoxgloveMosaicNode; const ids = collectMosaicPanelIds(root); - expect(ids.length).toBeGreaterThanOrEqual(2); const panelTypes = ids.map((id) => getPanelTypeFromId(id)); - expect(panelTypes).toContain('3D'); + expect(panelTypes).not.toContain('3D'); expect(panelTypes).toContain('RawMessages'); const imported = importFoxgloveLayout(data, { unavailableComponent: 'Unavailable' }); - expect(imported.restored).toBeGreaterThanOrEqual(2); + expect(imported.restored).toBe(1); const rawSnapshots = Object.values(imported.panelStates).filter((s) => s.type === 'RawMessages'); expect(rawSnapshots).toHaveLength(1); expect((rawSnapshots[0]?.config as { topic?: string }).topic).toBe('/scan'); diff --git a/src/features/layout/autoLayout/applyDefaultRosDockLayout.ts b/src/features/layout/autoLayout/applyDefaultRosDockLayout.ts index 4cc77f4..b645ce4 100644 --- a/src/features/layout/autoLayout/applyDefaultRosDockLayout.ts +++ b/src/features/layout/autoLayout/applyDefaultRosDockLayout.ts @@ -18,7 +18,7 @@ import { import { planColorDepthCameraRows } from '@/features/layout/autoLayout/planRosImageGrid'; import { heuristicAudioInfoTopics } from '@/features/panels/Audio/audio-core/resolveAudioInfo'; import { getPanelDefinition } from '@/features/panels/registry'; -import { isAudioCommonInfoSchema, isRawAudioSchema, normalizeRosSchemaName } from '@/shared/ros/rosMessageTypes'; +import { isAudioCommonInfoSchema, isJointStateSchema, isRawAudioSchema, normalizeRosSchemaName } from '@/shared/ros/rosMessageTypes'; import { pickDefaultRawMessagesTopic } from '@/features/layout/autoLayout/pickDefaultRawMessagesTopic'; function imageTabTitle(topic: string): string { @@ -192,6 +192,24 @@ export interface BuildDefaultRosLayoutOptions { publishersByTopic?: ReadonlyMap>; } +function isTransformTopicType(type: string): boolean { + const normalized = type.trim().toLowerCase(); + return ( + normalized.includes('tfmessage') || + normalized.includes('tf2_msgs') || + normalized.includes('tf/tfmessage') || + normalized.includes('/transform') + ); +} + +/** 3D is useful when the bag carries robot motion / TF — not for pure camera feeds. */ +function shouldIncludeThreeDPanel(topics: ReadonlyArray): boolean { + if (isBvhOnlyDataset(topics)) { + return true; + } + return topics.some((topic) => isJointStateSchema(topic.type) || isTransformTopicType(topic.type)); +} + function isHdf5Dataset( topics: ReadonlyArray, publishersByTopic?: ReadonlyMap>, @@ -244,8 +262,8 @@ export function buildDefaultRosFoxgloveLayoutData( stackParts.push(rawId); } else { const poseTopics = collectTopicsForPanelSchemas(topics, 'Pose'); - const threeDId = createPanelInstanceId('3D'); - configById[threeDId] = {}; + const includeThreeD = shouldIncludeThreeDPanel(topics); + if (poseTopics.length > 0) { const poseId = createPanelInstanceId('Pose'); configById[poseId] = { @@ -255,21 +273,28 @@ export function buildDefaultRosFoxgloveLayoutData( enabled: true, })), }; - stackParts.push({ - direction: 'row', - first: poseId, - second: threeDId, - }); - } else { + if (includeThreeD) { + const threeDId = createPanelInstanceId('3D'); + configById[threeDId] = {}; + stackParts.push({ + direction: 'row', + first: poseId, + second: threeDId, + }); + } else { + stackParts.push(poseId); + } + } else if (includeThreeD) { + const threeDId = createPanelInstanceId('3D'); + configById[threeDId] = {}; stackParts.push(threeDId); } } - // BVH-only datasets just keep the single 3D panel; `mosaicToDockviewGrid` - // wraps the leaf into a branch internally so Dockview's `fromJSON` is happy. - // For other single-eligible-panel cases (e.g. non-BVH yielding only 3D), - // keep the historical RawMessages fallback so the user still sees raw data. - if (stackParts.length === 1 && !isBvhOnlyDataset(topics) && !hdf5Dataset) { + if (stackParts.length === 0 && !isBvhOnlyDataset(topics) && !hdf5Dataset) { + const rawId = appendFallbackRawMessagesPanel(topics, configById); + stackParts.push(rawId); + } else if (stackParts.length === 1 && !isBvhOnlyDataset(topics) && !hdf5Dataset) { const rawId = appendFallbackRawMessagesPanel(topics, configById); stackParts.push(rawId); } diff --git a/src/features/layout/autoLayout/imageTopicSelection.test.ts b/src/features/layout/autoLayout/imageTopicSelection.test.ts index b69a244..fd86f1a 100644 --- a/src/features/layout/autoLayout/imageTopicSelection.test.ts +++ b/src/features/layout/autoLayout/imageTopicSelection.test.ts @@ -42,6 +42,18 @@ describe('selectImageTopicsForAutoLayout', () => { expect(picked).toEqual(['/camera/left/color/image_raw/compressed']); }); + it('includes foxglove CompressedVideo topics', () => { + const topics: TopicInfo[] = [ + { name: '/camera/left/color/video', type: 'foxglove_msgs/msg/CompressedVideo' }, + { name: '/camera/right/color/video', type: 'foxglove_msgs/msg/CompressedVideo' }, + ]; + const picked = selectImageTopicsForAutoLayout(topics); + expect(picked).toEqual([ + '/camera/left/color/video', + '/camera/right/color/video', + ]); + }); + it('keeps distinct non-overlapping streams', () => { const topics: TopicInfo[] = [ { name: '/camera/left/color/image_resized/compressed', type: 'sensor_msgs/msg/CompressedImage' }, diff --git a/src/features/layout/autoLayout/imageTopicSelection.ts b/src/features/layout/autoLayout/imageTopicSelection.ts index 8196470..42787be 100644 --- a/src/features/layout/autoLayout/imageTopicSelection.ts +++ b/src/features/layout/autoLayout/imageTopicSelection.ts @@ -43,6 +43,7 @@ export function imageTopicPriorityScore(topicName: string): number { if (n.includes('metadata')) return -1_000; let s = 0; if (n.includes('compressed')) s += 80; + if (n.includes('video')) s += 70; if (n.includes('image_resized')) s += 40; if (n.includes('/depth/')) s -= 50; if (n.includes('image_rect_raw') && !n.includes('compressed')) s += 25; diff --git a/src/features/layout/autoLayout/planRosImageGrid.test.ts b/src/features/layout/autoLayout/planRosImageGrid.test.ts index 98f62d6..664a23b 100644 --- a/src/features/layout/autoLayout/planRosImageGrid.test.ts +++ b/src/features/layout/autoLayout/planRosImageGrid.test.ts @@ -19,6 +19,11 @@ describe('classifyCameraSide', () => { expect(classifyCameraSide('/sensor/Right_Gripper_Camera_0/image/compressed')).toBe('right'); expect(classifyCameraSide('/sensor/EgoCentric_Camera_0/image/compressed')).toBe('other'); }); + + it('maps /head/ streams to the top column', () => { + expect(classifyCameraSide('/head/color/image')).toBe('top'); + expect(classifyCameraSide('/head/depth/image')).toBe('top'); + }); }); describe('isDepthImageTopicName', () => { @@ -46,6 +51,30 @@ describe('planColorDepthCameraRows', () => { expect(classifyCameraSide(row[1]!)).toBe('top'); expect(classifyCameraSide(row[2]!)).toBe('right'); } + expect(colorRow.every((topic) => topic == null || !isDepthImageTopicName(topic))).toBe(true); + expect(depthRow.every((topic) => topic == null || isDepthImageTopicName(topic))).toBe(true); + }); + + it('lays out six head/left/right CompressedVideo streams as color row + depth row', () => { + const topics: TopicInfo[] = [ + { name: '/head/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/head/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/left/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/left/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/right/color/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + { name: '/right/depth/image', type: 'foxglove_msgs/msg/CompressedVideo [ros2msg]' }, + ]; + const { colorRow, depthRow } = planColorDepthCameraRows(topics); + expect(colorRow).toEqual([ + '/left/color/image', + '/head/color/image', + '/right/color/image', + ]); + expect(depthRow).toEqual([ + '/left/depth/image', + '/head/depth/image', + '/right/depth/image', + ]); }); it('shows up to six streams even when side tokens are missing', () => { @@ -53,9 +82,9 @@ describe('planColorDepthCameraRows', () => { { name: '/sensor/Left_Gripper_Camera_0/image/compressed', type: 'sensor_msgs/msg/CompressedImage' }, { name: '/sensor/Right_Gripper_Camera_0/image/compressed', type: 'sensor_msgs/msg/CompressedImage' }, { name: '/sensor/EgoCentric_Camera_0/image/compressed', type: 'sensor_msgs/msg/CompressedImage' }, - { name: '/sensor/EgoCentric_Camera_1/image/compressed', type: 'sensor_msgs/msg/CompressedImage' }, - { name: '/io/pose/Gripper/compressed', type: 'sensor_msgs/msg/CompressedImage' }, - { name: '/io/depth/EgoCentric_Camera/color/compressed', type: 'sensor_msgs/msg/CompressedImage' }, + { name: '/sensor/Left_Gripper_Camera_0/depth/compressed', type: 'sensor_msgs/msg/CompressedImage' }, + { name: '/sensor/Right_Gripper_Camera_0/depth/compressed', type: 'sensor_msgs/msg/CompressedImage' }, + { name: '/sensor/EgoCentric_Camera_0/depth/compressed', type: 'sensor_msgs/msg/CompressedImage' }, ]; const { colorRow, depthRow } = planColorDepthCameraRows(topics); const picked = [...colorRow, ...depthRow].filter((name): name is string => name != null); @@ -63,9 +92,9 @@ describe('planColorDepthCameraRows', () => { expect(picked).toContain('/sensor/Left_Gripper_Camera_0/image/compressed'); expect(picked).toContain('/sensor/Right_Gripper_Camera_0/image/compressed'); expect(picked).toContain('/sensor/EgoCentric_Camera_0/image/compressed'); - expect(picked).toContain('/sensor/EgoCentric_Camera_1/image/compressed'); - expect(picked).toContain('/io/pose/Gripper/compressed'); - expect(picked).toContain('/io/depth/EgoCentric_Camera/color/compressed'); + expect(picked).toContain('/sensor/Left_Gripper_Camera_0/depth/compressed'); + expect(picked).toContain('/sensor/Right_Gripper_Camera_0/depth/compressed'); + expect(picked).toContain('/sensor/EgoCentric_Camera_0/depth/compressed'); }); }); diff --git a/src/features/layout/autoLayout/planRosImageGrid.ts b/src/features/layout/autoLayout/planRosImageGrid.ts index b39b49a..9f1345f 100644 --- a/src/features/layout/autoLayout/planRosImageGrid.ts +++ b/src/features/layout/autoLayout/planRosImageGrid.ts @@ -11,12 +11,17 @@ export function isDepthImageTopicName(name: string): boolean { /** Infer camera column (left/top/right) from topic path for auto-layout. */ export function classifyCameraSide(topicName: string): CameraColumn | 'other' { - const m = topicName.match(/\/camera\/(left|right|top)\b/i); + const m = topicName.match(/\/camera\/(left|right|top|head)\b/i); if (m) { - return m[1].toLowerCase() as CameraColumn; + const side = m[1].toLowerCase(); + if (side === 'head') { + return 'top'; + } + return side as CameraColumn; } const lower = topicName.toLowerCase(); const tokenized = `/${lower.replace(/[_-]+/g, '/')}/`; + if (tokenized.includes('/head/')) return 'top'; if (tokenized.includes('/top/')) return 'top'; if (tokenized.includes('/left/') && !tokenized.includes('/right/')) return 'left'; if (tokenized.includes('/right/')) return 'right'; @@ -58,7 +63,12 @@ function takeNextAny(state: PickState): string | null { return next.name; } -function buildCandidates(topics: ReadonlyArray, topicNameFilter: (name: string) => boolean): Candidate[] { +function buildCandidates( + topics: ReadonlyArray, + topicNameFilter: (name: string) => boolean, + options?: { minScore?: number }, +): Candidate[] { + const minScore = options?.minScore ?? 0; return topics .filter((t) => isRosImageSchema(t.type) && topicNameFilter(t.name)) .map((t) => ({ @@ -66,7 +76,7 @@ function buildCandidates(topics: ReadonlyArray, topicNameFilter: (nam score: imageTopicPriorityScore(t.name), side: classifyCameraSide(t.name), })) - .filter((c) => c.score >= 0) + .filter((c) => c.score >= minScore) .sort((a, b) => b.score - a.score || a.name.localeCompare(b.name)); } @@ -85,21 +95,27 @@ function pickOneRow(state: PickState): (string | null)[] { } /** - * Plan two image rows with one unified pool (no depth-reserved row): - * up to 3 topics per row, columns left / top / right with fallback fill. + * Plan color and depth rows separately: each row fills left / top(head) / right columns. + * Depth streams are not dropped by the color-vs-depth priority score used elsewhere. */ export function planColorDepthCameraRows(topics: ReadonlyArray): { colorRow: (string | null)[]; depthRow: (string | null)[]; } { - const unifiedState: PickState = { - candidates: buildCandidates(topics, () => true), + const colorState: PickState = { + candidates: buildCandidates(topics, (name) => !isDepthImageTopicName(name)), + usedNames: new Set(), + usedBuckets: new Set(), + }; + const depthState: PickState = { + candidates: buildCandidates(topics, (name) => isDepthImageTopicName(name), { minScore: -100 }), usedNames: new Set(), usedBuckets: new Set(), }; - const colorRow = pickOneRow(unifiedState); - const depthRow = pickOneRow(unifiedState); - return { colorRow, depthRow }; + return { + colorRow: pickOneRow(colorState), + depthRow: pickOneRow(depthState), + }; } /** diff --git a/src/features/panels/Image/Component.tsx b/src/features/panels/Image/Component.tsx index f51fcef..8d8a242 100644 --- a/src/features/panels/Image/Component.tsx +++ b/src/features/panels/Image/Component.tsx @@ -9,15 +9,13 @@ import type { ImageRenderOptions, ImageRenderWorkerEvent, ImageRenderWorkerRequest, - ImageWorkerFrameEnvelope, } from './image-core/imageWorkerProtocol'; import { - getCompressedKind, - isCompressedImageMessage, - isRawImageMessage, - prepareImageWorkerBytes, + IMAGE_PANEL_TOPIC_INCLUDES, type ImageSurfaceStatus, } from './image-core/imageTypes'; +import { repairH264Seek } from './image-core/h264SeekRepair'; +import { isH264MessageEvent, toWorkerFrame } from './image-core/messageFrameAdapter'; import type { ImageConfig } from './defaults'; import { TopicQuickPicker } from '../framework/TopicQuickPicker'; import ImageRenderWorkerClass from './image-core/ImageRender.worker.ts?worker&inline'; @@ -242,17 +240,23 @@ export const ImagePanel: React.FC = (props) => { }; }, [player, mainConsumerId, h264ConsumerId, topic]); - // Reset on playback rewind + // Reset on playback rewind; for H264, rebuild decoder state from the nearest keyframe. useEffect(() => { return player.subscribeCurrentTime((time) => { const nowNs = toNano(time); const previousNs = lastPlaybackTimeNsRef.current; if (previousNs != null && nowNs + 5_000_000n < previousNs) { - workerRef.current?.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest); + const worker = workerRef.current; + if (worker && topic && h264ModeRef.current) { + worker.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest); + void repairH264Seek(player, worker, topic, time); + } else { + workerRef.current?.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest); + } } lastPlaybackTimeNsRef.current = nowNs; }); - }, [player]); + }, [player, topic]); // Send color/depth decode options when they change — triggers immediate redraw in worker useEffect(() => { @@ -303,7 +307,7 @@ export const ImagePanel: React.FC = (props) => { setConfig((prev) => ({ ...prev, topic: nextTopic }))} - typeIncludes={['image', 'CompressedImage']} + typeIncludes={[...IMAGE_PANEL_TOPIC_INCLUDES]} placeholder={formatMessage({ id: 'panels.framework.topicPicker.imagePlaceholder' })} className="min-w-0 flex-1" triggerClassName="border-zinc-700 bg-zinc-950 text-zinc-100 hover:bg-zinc-900 hover:text-zinc-50" @@ -359,50 +363,6 @@ function isUiStatusEqual(a: ImageSurfaceStatus, b: ImageSurfaceStatus): boolean ); } -type PreparedImageWorkerFrame = { - frame: ImageWorkerFrameEnvelope; - transfer: Transferable[]; -}; - -function toWorkerFrame(messageEvent: RosMessageEvent): PreparedImageWorkerFrame | null { - const message = messageEvent.message; - if (isCompressedImageMessage(message)) { - const payload = prepareImageWorkerBytes(message.data); - if (!payload) { - return null; - } - return { - frame: { - kind: 'compressed', - receiveTime: messageEvent.receiveTime, - format: message.format, - data: payload.data, - }, - transfer: payload.transfer, - }; - } - if (isRawImageMessage(message)) { - const payload = prepareImageWorkerBytes(message.data); - if (!payload) { - return null; - } - return { - frame: { - kind: 'raw', - receiveTime: messageEvent.receiveTime, - encoding: message.encoding, - width: message.width, - height: message.height, - step: message.step, - isBigEndian: message.is_bigendian, - data: payload.data, - }, - transfer: payload.transfer, - }; - } - return null; -} - function postImageFrame(worker: Worker, messageEvent: RosMessageEvent): void { const next = toWorkerFrame(messageEvent); if (!next) { @@ -413,8 +373,3 @@ function postImageFrame(worker: Worker, messageEvent: RosMessageEvent): void { next.transfer, ); } - -function isH264MessageEvent(messageEvent: RosMessageEvent): boolean { - const message = messageEvent.message; - return isCompressedImageMessage(message) && getCompressedKind(message.format) === 'h264'; -} diff --git a/src/features/panels/Image/ImagePanelSettings.tsx b/src/features/panels/Image/ImagePanelSettings.tsx index 96ed155..2706998 100644 --- a/src/features/panels/Image/ImagePanelSettings.tsx +++ b/src/features/panels/Image/ImagePanelSettings.tsx @@ -13,7 +13,7 @@ import { } from '../framework/settings'; import { messageBus } from '@/core/pipeline/messageBus'; import { useTopicSeq } from '@/core/pipeline/useMessageBus'; -import { isRawImageMessage, isRawImageTopicSchema } from './image-core/imageTypes'; +import { isRawImageMessage, isRawImageTopicSchema, IMAGE_PANEL_TOPIC_INCLUDES } from './image-core/imageTypes'; import type { ImageConfig } from './defaults'; const DEPTH_ENCODINGS = new Set(['mono16', '16uc1', '32fc1']); @@ -111,7 +111,7 @@ export function ImagePanelSettings({ value={config.topic} onChange={(topic) => setConfig({ ...config, topic })} topics={topics} - typeIncludes={['image', 'CompressedImage']} + typeIncludes={[...IMAGE_PANEL_TOPIC_INCLUDES]} placeholder={formatMessage({ id: 'panels.image.settings.field.topic.placeholder' })} /> diff --git a/src/features/panels/Image/definition.tsx b/src/features/panels/Image/definition.tsx index b5cc35d..2875cd9 100644 --- a/src/features/panels/Image/definition.tsx +++ b/src/features/panels/Image/definition.tsx @@ -3,7 +3,11 @@ import type { PanelDefinition } from '../framework/types'; import { PanelSuspense } from '../framework/panelSuspense'; import { defaultImageConfig, type ImageConfig } from './defaults'; import { parseImageConfig } from './schema'; -import { ROS_MSG_SENSOR_COMPRESSED_IMAGE, ROS_MSG_SENSOR_IMAGE } from '@/shared/ros/rosMessageTypes'; +import { + ROS_MSG_FOXGLOVE_COMPRESSED_VIDEO, + ROS_MSG_SENSOR_COMPRESSED_IMAGE, + ROS_MSG_SENSOR_IMAGE, +} from '@/shared/ros/rosMessageTypes'; import { ImagePanelSettings } from './ImagePanelSettings'; const ImagePanel = lazy(async () => { @@ -15,7 +19,11 @@ export const imagePanelDefinition: PanelDefinition = { type: 'Image', defaultTitle: 'Image', schemaSupport: { - supportedSchemas: [ROS_MSG_SENSOR_IMAGE, ROS_MSG_SENSOR_COMPRESSED_IMAGE], + supportedSchemas: [ + ROS_MSG_SENSOR_IMAGE, + ROS_MSG_SENSOR_COMPRESSED_IMAGE, + ROS_MSG_FOXGLOVE_COMPRESSED_VIDEO, + ], }, createDefaultConfig: defaultImageConfig, configSchema: { version: 5, parse: parseImageConfig }, diff --git a/src/features/panels/Image/image-core/h264.test.ts b/src/features/panels/Image/image-core/h264.test.ts index 961bc30..2de2ab6 100644 --- a/src/features/panels/Image/image-core/h264.test.ts +++ b/src/features/panels/Image/image-core/h264.test.ts @@ -20,4 +20,16 @@ describe('H.264 NAL parsing', () => { expect(scanH264NalTypes(new Uint8Array([0x65, 1, 2]))).toEqual([5]); expect(getH264ChunkType(new Uint8Array([0x41, 1, 2]))).toBe('delta'); }); + + it('treats Foxglove keyframe chunks with SPS + IDR as key frames', () => { + // Per foxglove_msgs/CompressedVideo: keyframes must include SPS and IDR NAL units. + const chunk = new Uint8Array([ + 0, 0, 0, 1, 0x67, 0x42, 0x00, 0x1e, + 0, 0, 0, 1, 0x68, 0xce, 0x3c, 0x80, + 0, 0, 0, 1, 0x65, 0x88, 0x84, + ]); + + expect(scanH264NalTypes(chunk)).toEqual([7, 8, 5]); + expect(getH264ChunkType(chunk)).toBe('key'); + }); }); diff --git a/src/features/panels/Image/image-core/h264SeekRepair.test.ts b/src/features/panels/Image/image-core/h264SeekRepair.test.ts new file mode 100644 index 0000000..3e45365 --- /dev/null +++ b/src/features/panels/Image/image-core/h264SeekRepair.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import type { MessageEvent as RosMessageEvent } from '@/core/types/ros'; +import { findLatestH264KeyFrameIndex, selectH264SeekRepairFrames } from './h264SeekRepair'; + +const keyChunk = new Uint8Array([0, 0, 0, 1, 0x67, 1, 2, 0, 0, 1, 0x65, 3, 4]); +const deltaChunk = new Uint8Array([0, 0, 1, 0x41, 9, 9]); + +function makeEvent(sec: number, data: Uint8Array, format = 'h264'): RosMessageEvent { + const receiveTime = { sec, nsec: 0 }; + return { + topic: '/camera/video', + receiveTime, + publishTime: receiveTime, + message: { format, data }, + schemaName: 'foxglove_msgs/msg/CompressedVideo', + }; +} + +describe('h264SeekRepair', () => { + it('findLatestH264KeyFrameIndex returns the last keyframe index', () => { + const messages = [makeEvent(1, keyChunk), makeEvent(2, deltaChunk), makeEvent(3, deltaChunk)]; + expect(findLatestH264KeyFrameIndex(messages)).toBe(0); + }); + + it('selectH264SeekRepairFrames returns frames from keyframe through target time', () => { + const messages = [ + makeEvent(1, keyChunk), + makeEvent(2, deltaChunk), + makeEvent(3, deltaChunk), + makeEvent(4, deltaChunk), + ]; + const repair = selectH264SeekRepairFrames(messages, { sec: 3, nsec: 0 }); + + expect(repair).toHaveLength(3); + expect(repair.map((event) => event.receiveTime.sec)).toEqual([1, 2, 3]); + }); + + it('selectH264SeekRepairFrames ignores messages after the target time', () => { + const messages = [ + makeEvent(1, keyChunk), + makeEvent(2, deltaChunk), + makeEvent(4, deltaChunk), + ]; + const repair = selectH264SeekRepairFrames(messages, { sec: 2, nsec: 0 }); + + expect(repair).toHaveLength(2); + expect(repair.map((event) => event.receiveTime.sec)).toEqual([1, 2]); + }); + + it('returns empty when no keyframe exists in the window', () => { + const messages = [makeEvent(1, deltaChunk), makeEvent(2, deltaChunk)]; + expect(selectH264SeekRepairFrames(messages, { sec: 2, nsec: 0 })).toEqual([]); + }); +}); diff --git a/src/features/panels/Image/image-core/h264SeekRepair.ts b/src/features/panels/Image/image-core/h264SeekRepair.ts new file mode 100644 index 0000000..e6ef1b9 --- /dev/null +++ b/src/features/panels/Image/image-core/h264SeekRepair.ts @@ -0,0 +1,95 @@ +import type { Player } from '@/core/types/player'; +import type { MessageEvent as RosMessageEvent, Time } from '@/core/types/ros'; +import { addMs, toNano } from '@/shared/utils/time'; +import { getH264ChunkType } from './h264'; +import type { ImageRenderWorkerRequest } from './imageWorkerProtocol'; +import { getH264MessagePayload, isH264MessageEvent, toWorkerFrame } from './messageFrameAdapter'; + +/** Progressive lookback windows when searching for a keyframe before a seek target. */ +export const H264_SEEK_WINDOWS_MS = [2000, 5000, 10_000, 30_000] as const; + +/** Cap frames fed during seek repair so high-FPS streams stay responsive. */ +export const H264_SEEK_MAX_FRAMES = 180; + +export function findLatestH264KeyFrameIndex(messages: RosMessageEvent[]): number { + for (let i = messages.length - 1; i >= 0; i -= 1) { + const event = messages[i]; + if (!event) { + continue; + } + const payload = getH264MessagePayload(event); + if (payload && getH264ChunkType(payload) === 'key') { + return i; + } + } + return -1; +} + +export function selectH264SeekRepairFrames( + messages: RosMessageEvent[], + targetTime: Time, +): RosMessageEvent[] { + const targetNs = toNano(targetTime); + const h264Messages = messages + .filter((event) => isH264MessageEvent(event) && toNano(event.receiveTime) <= targetNs) + .sort((a, b) => { + const diff = toNano(a.receiveTime) - toNano(b.receiveTime); + if (diff < 0n) { + return -1; + } + if (diff > 0n) { + return 1; + } + return 0; + }); + + const keyIndex = findLatestH264KeyFrameIndex(h264Messages); + if (keyIndex < 0) { + return []; + } + + return h264Messages.slice(keyIndex).slice(-H264_SEEK_MAX_FRAMES); +} + +export async function repairH264Seek( + player: Player, + worker: Worker, + topic: string, + targetTime: Time, +): Promise { + if (!player.getMessagesInTimeRange) { + return false; + } + + for (const windowMs of H264_SEEK_WINDOWS_MS) { + const start = addMs(targetTime, -windowMs); + const messages = await player.getMessagesInTimeRange({ + start, + end: targetTime, + topics: [topic], + }); + + const repairFrames = selectH264SeekRepairFrames( + messages.filter((event) => event.topic === topic), + targetTime, + ); + if (repairFrames.length === 0) { + continue; + } + + worker.postMessage({ type: 'reset' } satisfies ImageRenderWorkerRequest); + for (const event of repairFrames) { + const next = toWorkerFrame(event); + if (!next) { + continue; + } + worker.postMessage( + { type: 'frame', frame: next.frame } satisfies ImageRenderWorkerRequest, + next.transfer, + ); + } + return true; + } + + return false; +} diff --git a/src/features/panels/Image/image-core/imageTypes.test.ts b/src/features/panels/Image/image-core/imageTypes.test.ts index 471c590..005fe34 100644 --- a/src/features/panels/Image/image-core/imageTypes.test.ts +++ b/src/features/panels/Image/image-core/imageTypes.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest'; import { getCompressedKind, + isCompressedVideoMessage, + isH264CompressedFrameMessage, + isImagePanelTopicSchema, isRawImageTopicSchema, normalizeCompressedMime, prepareImageWorkerBytes, @@ -51,12 +54,66 @@ describe('isRawImageTopicSchema', () => { expect(isRawImageTopicSchema('sensor_msgs/CompressedImage')).toBe(false); }); + it('rejects CompressedVideo', () => { + expect(isRawImageTopicSchema('foxglove_msgs/msg/CompressedVideo')).toBe(false); + }); + it('rejects unrelated types', () => { expect(isRawImageTopicSchema('')).toBe(false); expect(isRawImageTopicSchema('sensor_msgs/msg/CameraInfo')).toBe(false); }); }); +describe('isCompressedVideoMessage', () => { + it('accepts foxglove CompressedVideo payloads', () => { + expect( + isCompressedVideoMessage({ + timestamp: { sec: 1, nsec: 0 }, + frame_id: 'camera', + format: 'h264', + data: new Uint8Array([1, 2, 3]), + }), + ).toBe(true); + }); + + it('rejects payloads without binary data', () => { + expect(isCompressedVideoMessage({ format: 'h264', data: [1, 2, 3] })).toBe(false); + }); +}); + +describe('isH264CompressedFrameMessage', () => { + it('recognizes h264 CompressedVideo format tokens', () => { + expect( + isH264CompressedFrameMessage({ + format: 'h264', + data: new Uint8Array([0]), + }), + ).toBe(true); + }); + + it('does not treat other CompressedVideo codecs as h264', () => { + expect( + isH264CompressedFrameMessage({ + format: 'h265', + data: new Uint8Array([0]), + }), + ).toBe(false); + expect(getCompressedKind('vp9')).toBeNull(); + }); +}); + +describe('isImagePanelTopicSchema', () => { + it('accepts raw, compressed image, and compressed video schemas', () => { + expect(isImagePanelTopicSchema('sensor_msgs/msg/Image')).toBe(true); + expect(isImagePanelTopicSchema('sensor_msgs/msg/CompressedImage')).toBe(true); + expect(isImagePanelTopicSchema('foxglove_msgs/msg/CompressedVideo')).toBe(true); + }); + + it('rejects unrelated schemas', () => { + expect(isImagePanelTopicSchema('sensor_msgs/msg/CameraInfo')).toBe(false); + }); +}); + describe('normalizeCompressedMime', () => { it('falls back to JPEG for raw-style format tokens on CompressedImage', () => { expect(getCompressedKind('bgr8')).toBeNull(); diff --git a/src/features/panels/Image/image-core/imageTypes.ts b/src/features/panels/Image/image-core/imageTypes.ts index 1b97904..efe389d 100644 --- a/src/features/panels/Image/image-core/imageTypes.ts +++ b/src/features/panels/Image/image-core/imageTypes.ts @@ -14,6 +14,14 @@ export interface CompressedImageMessage { data: Uint8Array; } +/** foxglove_msgs/msg/CompressedVideo — Annex B H264/H265/VP9/AV1 bitstream chunks. */ +export interface CompressedVideoMessage { + timestamp?: Time; + frame_id?: string; + format: string; + data: Uint8Array; +} + export interface ImageSurfaceStatus { phase: 'idle' | 'decoding' | 'ready' | 'error'; width?: number; @@ -65,6 +73,33 @@ export function isCompressedImageMessage(message: unknown): message is Compresse ); } +export function isCompressedVideoMessage(message: unknown): message is CompressedVideoMessage { + return Boolean( + message && + typeof message === 'object' && + 'format' in message && + 'data' in message && + (message as { data?: unknown }).data instanceof Uint8Array, + ); +} + +export type CompressedFrameMessage = CompressedImageMessage | CompressedVideoMessage; + +export function isCompressedFrameMessage(message: unknown): message is CompressedFrameMessage { + return isCompressedImageMessage(message) || isCompressedVideoMessage(message); +} + +export function getCompressedFrameFormat(message: CompressedFrameMessage): string { + return message.format; +} + +export function isH264CompressedFrameMessage(message: unknown): boolean { + if (!isCompressedFrameMessage(message)) { + return false; + } + return getCompressedKind(getCompressedFrameFormat(message)) === 'h264'; +} + export function isRawImageMessage(message: unknown): message is RawImageMessage { return Boolean( message && @@ -157,5 +192,28 @@ export function isRawImageTopicSchema(schemaName: string): boolean { if (lower.includes('compressedimage')) { return false; } + if (lower.includes('compressedvideo')) { + return false; + } return /(^|\/)sensor_msgs\/(msg\/)?image$/i.test(trimmed); } + +/** Topic type tokens accepted by the Image panel topic picker. */ +export const IMAGE_PANEL_TOPIC_INCLUDES = ['image', 'CompressedImage', 'CompressedVideo'] as const; + +export function isImagePanelTopicSchema(schemaName: string): boolean { + const lower = schemaName.trim().toLowerCase(); + if (!lower) { + return false; + } + if (isRawImageTopicSchema(schemaName)) { + return true; + } + if (lower.includes('compressedimage')) { + return true; + } + if (lower.includes('compressedvideo')) { + return true; + } + return false; +} diff --git a/src/features/panels/Image/image-core/messageFrameAdapter.test.ts b/src/features/panels/Image/image-core/messageFrameAdapter.test.ts new file mode 100644 index 0000000..efdb863 --- /dev/null +++ b/src/features/panels/Image/image-core/messageFrameAdapter.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from 'vitest'; +import type { MessageEvent as RosMessageEvent } from '@/core/types/ros'; +import { isH264MessageEvent, toWorkerFrame } from './messageFrameAdapter'; + +const receiveTime = { sec: 10, nsec: 0 }; + +function makeCompressedImageEvent(data: Uint8Array, format = 'h264'): RosMessageEvent { + return { + topic: '/camera/compressed', + receiveTime, + publishTime: receiveTime, + message: { format, data }, + schemaName: 'sensor_msgs/msg/CompressedImage', + }; +} + +function makeCompressedVideoEvent(data: Uint8Array, format = 'h264'): RosMessageEvent { + return { + topic: '/camera/video', + receiveTime, + publishTime: receiveTime, + message: { + timestamp: receiveTime, + frame_id: 'camera_optical', + format, + data, + }, + schemaName: 'foxglove_msgs/msg/CompressedVideo', + }; +} + +describe('messageFrameAdapter', () => { + it('maps CompressedImage and CompressedVideo h264 payloads to the same worker envelope shape', () => { + const payload = new Uint8Array([0, 0, 0, 1, 0x65, 1, 2, 3]); + const fromImage = toWorkerFrame(makeCompressedImageEvent(payload)); + const fromVideo = toWorkerFrame(makeCompressedVideoEvent(payload)); + + expect(fromImage).not.toBeNull(); + expect(fromVideo).not.toBeNull(); + expect(fromImage!.frame).toMatchObject({ + kind: 'compressed', + receiveTime, + format: 'h264', + }); + expect(fromVideo!.frame).toMatchObject({ + kind: 'compressed', + receiveTime, + format: 'h264', + }); + expect(Array.from(fromImage!.frame.data)).toEqual(Array.from(fromVideo!.frame.data)); + }); + + it('detects h264 for both CompressedImage and CompressedVideo', () => { + const payload = new Uint8Array([1]); + expect(isH264MessageEvent(makeCompressedImageEvent(payload))).toBe(true); + expect(isH264MessageEvent(makeCompressedVideoEvent(payload))).toBe(true); + expect(isH264MessageEvent(makeCompressedVideoEvent(payload, 'vp9'))).toBe(false); + }); +}); diff --git a/src/features/panels/Image/image-core/messageFrameAdapter.ts b/src/features/panels/Image/image-core/messageFrameAdapter.ts new file mode 100644 index 0000000..ca8dcfa --- /dev/null +++ b/src/features/panels/Image/image-core/messageFrameAdapter.ts @@ -0,0 +1,68 @@ +import type { MessageEvent as RosMessageEvent } from '@/core/types/ros'; +import type { ImageWorkerFrameEnvelope } from './imageWorkerProtocol'; +import { + getCompressedFrameFormat, + isCompressedFrameMessage, + isH264CompressedFrameMessage, + isRawImageMessage, + prepareImageWorkerBytes, +} from './imageTypes'; + +export type PreparedImageWorkerFrame = { + frame: ImageWorkerFrameEnvelope; + transfer: Transferable[]; +}; + +export function toWorkerFrame(messageEvent: RosMessageEvent): PreparedImageWorkerFrame | null { + const message = messageEvent.message; + if (isCompressedFrameMessage(message)) { + const payload = prepareImageWorkerBytes(message.data); + if (!payload) { + return null; + } + return { + frame: { + kind: 'compressed', + receiveTime: messageEvent.receiveTime, + format: getCompressedFrameFormat(message), + data: payload.data, + }, + transfer: payload.transfer, + }; + } + if (isRawImageMessage(message)) { + const payload = prepareImageWorkerBytes(message.data); + if (!payload) { + return null; + } + return { + frame: { + kind: 'raw', + receiveTime: messageEvent.receiveTime, + encoding: message.encoding, + width: message.width, + height: message.height, + step: message.step, + isBigEndian: message.is_bigendian, + data: payload.data, + }, + transfer: payload.transfer, + }; + } + return null; +} + +export function isH264MessageEvent(messageEvent: RosMessageEvent): boolean { + return isH264CompressedFrameMessage(messageEvent.message); +} + +export function getH264MessagePayload(messageEvent: RosMessageEvent): Uint8Array | null { + const message = messageEvent.message; + if (!isCompressedFrameMessage(message)) { + return null; + } + if (!isH264CompressedFrameMessage(message)) { + return null; + } + return message.data; +} diff --git a/src/shared/ros/rosMessageTypes.test.ts b/src/shared/ros/rosMessageTypes.test.ts index 910cab9..0127677 100644 --- a/src/shared/ros/rosMessageTypes.test.ts +++ b/src/shared/ros/rosMessageTypes.test.ts @@ -7,6 +7,7 @@ import { isRosImageSchema, matchesRosSchema, normalizeRosSchemaName, + ROS_MSG_FOXGLOVE_COMPRESSED_VIDEO, ROS_MSG_FOXGLOVE_RAW_AUDIO, ROS_MSG_POSE_STAMPED, ROS_MSG_SENSOR_COMPRESSED_IMAGE, @@ -41,6 +42,7 @@ describe('isRosImageSchema', () => { it('supports source-annotated schema labels', () => { expect(isRosImageSchema('sensor_msgs/msg/Image [ros2msg]')).toBe(true); expect(isRosImageSchema('sensor_msgs/msg/CompressedImage [jsonschema]')).toBe(true); + expect(isRosImageSchema(`${ROS_MSG_FOXGLOVE_COMPRESSED_VIDEO} [ros2msg]`)).toBe(true); }); }); @@ -60,6 +62,7 @@ describe('rosMessageTypes', () => { it('detects image schemas', () => { expect(isRosImageSchema('sensor_msgs/msg/Image')).toBe(true); expect(isRosImageSchema('sensor_msgs/msg/CompressedImage')).toBe(true); + expect(isRosImageSchema(ROS_MSG_FOXGLOVE_COMPRESSED_VIDEO)).toBe(true); expect(isRosImageSchema('sensor_msgs/msg/CameraInfo')).toBe(false); }); diff --git a/src/shared/ros/rosMessageTypes.ts b/src/shared/ros/rosMessageTypes.ts index afc880c..dbb605b 100644 --- a/src/shared/ros/rosMessageTypes.ts +++ b/src/shared/ros/rosMessageTypes.ts @@ -2,6 +2,7 @@ export const ROS_MSG_SENSOR_IMAGE = 'sensor_msgs/msg/Image' as const; export const ROS_MSG_SENSOR_COMPRESSED_IMAGE = 'sensor_msgs/msg/CompressedImage' as const; +export const ROS_MSG_FOXGLOVE_COMPRESSED_VIDEO = 'foxglove_msgs/msg/CompressedVideo' as const; export const ROS_MSG_JOINT_STATE = 'sensor_msgs/msg/JointState' as const; export const ROS_MSG_POSE_STAMPED = 'geometry_msgs/msg/PoseStamped' as const; export const ROS_MSG_FOXGLOVE_RAW_AUDIO = 'foxglove_msgs/msg/RawAudio' as const; @@ -34,7 +35,9 @@ export function isJointStateSchema(type: string): boolean { export function isRosImageSchema(type: string): boolean { return ( - matchesRosSchema(type, ROS_MSG_SENSOR_IMAGE) || matchesRosSchema(type, ROS_MSG_SENSOR_COMPRESSED_IMAGE) + matchesRosSchema(type, ROS_MSG_SENSOR_IMAGE) || + matchesRosSchema(type, ROS_MSG_SENSOR_COMPRESSED_IMAGE) || + matchesRosSchema(type, ROS_MSG_FOXGLOVE_COMPRESSED_VIDEO) ); } From 30d6eb31054275065e10af33359c40b55f22aa8c Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 16:45:58 +0800 Subject: [PATCH 06/15] fix(lint): include test files in ESLint tsconfig and clean up SqliteSqljsDb. Clear inherited app tsconfig excludes so type-checked ESLint covers src tests, and resolve require-await violations in the sql.js db adapter. --- src/infra/sources/SqliteSqljsDb.ts | 21 +++++++++++---------- tsconfig.eslint.json | 5 +++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/infra/sources/SqliteSqljsDb.ts b/src/infra/sources/SqliteSqljsDb.ts index 5d86e6b..bc49161 100644 --- a/src/infra/sources/SqliteSqljsDb.ts +++ b/src/infra/sources/SqliteSqljsDb.ts @@ -78,18 +78,19 @@ export class SqliteSqljsDb implements SqliteDb { this.#context = { db, idToTopic, topicNameToId }; } - async close(): Promise { + close(): Promise { if (this.#context != undefined) { this.#context.db.close(); this.#context = undefined; } + return Promise.resolve(); } - async readTopics(): Promise { + readTopics(): Promise { if (this.#context == undefined) { throw new Error('Call open() before reading topics'); } - return Array.from(this.#context.idToTopic.values()); + return Promise.resolve(Array.from(this.#context.idToTopic.values())); } readMessages(opts: MessageReadOptions = {}): AsyncIterableIterator { @@ -134,7 +135,7 @@ export class SqliteSqljsDb implements SqliteDb { } else { query += ' and topic_id = ?'; } - args.push(topicIds[0]!); + args.push(topicIds[0]); } else { if (args.length === 0) { query += ` where topic_id in (${topicIds.map(() => '?').join(',')})`; @@ -150,7 +151,7 @@ export class SqliteSqljsDb implements SqliteDb { return new RawMessageIterator(dbIterator, this.#context.idToTopic); } - async timeRange(): Promise<[min: Time, max: Time]> { + timeRange(): Promise<[min: Time, max: Time]> { if (this.#context == undefined) { throw new Error('Call open() before retrieving the time range'); } @@ -160,10 +161,10 @@ export class SqliteSqljsDb implements SqliteDb { 'select cast(min(timestamp) as TEXT), cast(max(timestamp) as TEXT) from messages', )[0]?.values[0] ?? ['0', '0']; const [minNsec, maxNsec] = res as [string | null, string | null]; - return [fromNanoSec(BigInt(minNsec ?? 0n)), fromNanoSec(BigInt(maxNsec ?? 0n))]; + return Promise.resolve([fromNanoSec(BigInt(minNsec ?? 0n)), fromNanoSec(BigInt(maxNsec ?? 0n))]); } - async messageCounts(): Promise> { + messageCounts(): Promise> { if (this.#context == undefined) { throw new Error('Call open() before retrieving message counts'); } @@ -179,10 +180,10 @@ export class SqliteSqljsDb implements SqliteDb { for (const [topicName, count] of rows) { counts.set(topicName as string, count as number); } - return counts; + return Promise.resolve(counts); } - async topicTimeRanges(): Promise> { + topicTimeRanges(): Promise> { if (this.#context == undefined) { throw new Error('Call open() before retrieving topic time ranges'); } @@ -202,7 +203,7 @@ export class SqliteSqljsDb implements SqliteDb { fromNanoSec(BigInt(maxNsec ?? 0n)), ]); } - return ranges; + return Promise.resolve(ranges); } } diff --git a/tsconfig.eslint.json b/tsconfig.eslint.json index ceeff84..8bcd93f 100644 --- a/tsconfig.eslint.json +++ b/tsconfig.eslint.json @@ -1,7 +1,8 @@ { "extends": "./tsconfig.app.json", "compilerOptions": { - "types": ["vite/client", "node", "@playwright/test"] + "types": ["vite/client", "node", "@playwright/test", "vitest/globals"] }, - "include": ["src/**/*.ts", "src/**/*.tsx", "tests/**/*.ts"] + "include": ["src/**/*.ts", "src/**/*.tsx", "tests/**/*.ts"], + "exclude": [] } From 9811d47a9fd0f1bc67fa40365512927688401131 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:01:06 +0800 Subject: [PATCH 07/15] fix(playback): prevent stale async results from overwriting backward seeks. Introduce a playback epoch in IterablePlayer so late nextBatch/backfill/cursor work is discarded after seek, and add regression tests for backward seek and out-of-order backfill. --- src/core/players/IterablePlayer.test.ts | 129 ++++++++++++ src/core/players/IterablePlayer.ts | 253 +++++++++++++++++------- 2 files changed, 315 insertions(+), 67 deletions(-) diff --git a/src/core/players/IterablePlayer.test.ts b/src/core/players/IterablePlayer.test.ts index d126273..c32953b 100644 --- a/src/core/players/IterablePlayer.test.ts +++ b/src/core/players/IterablePlayer.test.ts @@ -50,6 +50,14 @@ function makeImageMessage(): MessageEvent { }; } +function makeImageMessageAt(sec: number): MessageEvent { + return { + ...makeImageMessage(), + receiveTime: { sec, nsec: 0 }, + publishTime: { sec, nsec: 0 }, + }; +} + function makeSource(messages: MessageEvent[]): WorkerSerializedSource { return { initialize: vi.fn(async () => makeInitialization()), @@ -83,6 +91,20 @@ async function flushAsyncWork(): Promise { await new Promise((resolve) => setTimeout(resolve, 0)); } +function deferred(): { + promise: Promise; + resolve: (value: T) => void; + reject: (reason?: unknown) => void; +} { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + afterEach(() => { messageBus.reset(); }); @@ -348,6 +370,113 @@ describe('IterablePlayer playback clock', () => { globalThis.cancelAnimationFrame = oldCancelAnimationFrame; } }); + + it('does not let an in-flight playback batch overwrite a backward seek', async () => { + let now = 0; + let nextRafId = 1; + const oldPerformanceNow = performance.now; + const oldRequestAnimationFrame = globalThis.requestAnimationFrame; + const oldCancelAnimationFrame = globalThis.cancelAnimationFrame; + const rafCallbacks = new Map(); + Object.defineProperty(performance, 'now', { + configurable: true, + value: () => now, + }); + globalThis.requestAnimationFrame = vi.fn((cb: FrameRequestCallback) => { + const id = nextRafId++; + rafCallbacks.set(id, cb); + return id; + }); + globalThis.cancelAnimationFrame = vi.fn((id: number) => { + rafCallbacks.delete(id); + }); + + const delayedBatch = deferred(); + const source = makeSource([]); + const cursor = { + nextBatch: vi.fn(() => delayedBatch.promise), + end: vi.fn(async () => undefined), + }; + vi.mocked(source.getMessageCursor).mockResolvedValue(cursor as never); + const player = new IterablePlayer(source); + const seenTimes: number[] = []; + + try { + await player.initialize({}); + player.registerSubscriptions('panel', [{ topic: TOPIC, subscriberId: 'panel' }]); + await flushAsyncWork(); + player.seek({ sec: 7, nsec: 0 }); + await flushAsyncWork(); + player.subscribeCurrentTime((time) => { + seenTimes.push(time.sec + time.nsec / 1e9); + }); + player.play(); + + now = 100; + rafCallbacks.get(1)?.(now); + await Promise.resolve(); + expect(cursor.nextBatch).toHaveBeenCalled(); + + player.seek({ sec: 5, nsec: 0 }); + await flushAsyncWork(); + expect(seenTimes.at(-1)).toBe(5); + + delayedBatch.resolve([makeImageMessageAt(7)]); + await flushAsyncWork(); + + expect(seenTimes.at(-1)).toBe(5); + expect(seenTimes).not.toContain(7.1); + } finally { + player.close(); + Object.defineProperty(performance, 'now', { + configurable: true, + value: oldPerformanceNow, + }); + globalThis.requestAnimationFrame = oldRequestAnimationFrame; + globalThis.cancelAnimationFrame = oldCancelAnimationFrame; + } + }); + + it('keeps the latest seek when older backfill finishes later', async () => { + const source = makeSource([]); + const firstBackfill = deferred(); + const secondBackfill = deferred(); + vi.mocked(source.getBackfillMessages).mockImplementation(async ({ time }) => { + if (time.sec === 5) { + return await firstBackfill.promise; + } + if (time.sec === 4) { + return await secondBackfill.promise; + } + return []; + }); + const player = new IterablePlayer(source); + const seenTimes: number[] = []; + + try { + await player.initialize({}); + player.registerSubscriptions('panel', [{ topic: TOPIC, subscriberId: 'panel' }]); + await flushAsyncWork(); + player.subscribeCurrentTime((time) => { + seenTimes.push(time.sec + time.nsec / 1e9); + }); + + player.seek({ sec: 5, nsec: 0 }); + await Promise.resolve(); + player.seek({ sec: 4, nsec: 0 }); + await Promise.resolve(); + + secondBackfill.resolve([makeImageMessageAt(4)]); + await flushAsyncWork(); + firstBackfill.resolve([makeImageMessageAt(5)]); + await flushAsyncWork(); + + expect(seenTimes.at(-1)).toBe(4); + expect(messageBus.getLastMessage(TOPIC)?.receiveTime).toEqual({ sec: 4, nsec: 0 }); + } finally { + player.close(); + } + }); }); describe('IterablePlayer topic metadata', () => { diff --git a/src/core/players/IterablePlayer.ts b/src/core/players/IterablePlayer.ts index a9a2e2b..6517a4b 100644 --- a/src/core/players/IterablePlayer.ts +++ b/src/core/players/IterablePlayer.ts @@ -155,6 +155,8 @@ export class IterablePlayer implements Player { private _isBuffering = false; private _topicLastMessageNs = new Map(); private _highFrequencyConsumerSignature = ""; + private _playbackEpoch = 0; + private _fetchEpoch: number | undefined; private _debugEnabled = typeof window !== "undefined" && new URLSearchParams(window.location.search).get("debugPlayback") === "1"; @@ -296,10 +298,16 @@ export class IterablePlayer implements Player { if (!topicsChanged) { return; } - if (this._cursor) { - await this._cursor.end(); - this._cursor = undefined; + const epoch = this._advancePlaybackEpoch(); + const cursor = this._cursor; + this._cursor = undefined; + if (cursor) { + await cursor.end(); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } } + this._scheduleNextTick(); } private async _handleHighFrequencyConsumerChange( @@ -309,8 +317,14 @@ export class IterablePlayer implements Player { const before = new Set(previousTopics); await this._handleTopicSetChange(before); if (this._cursor && previousSignature !== this._highFrequencyConsumerSignature) { - await this._cursor.end(); + const epoch = this._advancePlaybackEpoch(); + const cursor = this._cursor; this._cursor = undefined; + await cursor.end(); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } + this._scheduleNextTick(); } if (!this._isPlaying && this._initialization) { await this._backfillCurrentTime(); @@ -318,14 +332,19 @@ export class IterablePlayer implements Player { } private async _backfillCurrentTime(): Promise { + const epoch = this._playbackEpoch; + const referenceTime = this._currentTime; const topics = this._currentTopics(); if (topics.length === 0) return; try { const messages = await this._source.getBackfillMessages({ - time: this._currentTime, + time: referenceTime, topics, }); - this._distributeMessages(messages, this._currentTime); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } + this._distributeMessages(messages, referenceTime); this._lastPipelineEmitMs = 0; this._emitState(); } catch (err) { @@ -339,6 +358,7 @@ export class IterablePlayer implements Player { this._emitState(); try { + this._advancePlaybackEpoch(); this._initialization = await this._source.initialize(args); this._topicLastMessageNs.clear(); this._currentTime = this._initialization.start; @@ -417,6 +437,7 @@ export class IterablePlayer implements Player { play(): void { if (this._isPlaying) return; + this._advancePlaybackEpoch(); this._isPlaying = true; const now = performance.now(); this._clock.play(this._currentTime, this._speedFactor(), now); @@ -430,13 +451,11 @@ export class IterablePlayer implements Player { void this._refreshLoadProgress(); } this._emitState(); - this._cancelRaf(); - if (!this._pageSuspended) { - this._rafId = requestAnimationFrame(this._tickLoop); - } + this._scheduleNextTick(); } pause(): void { + this._advancePlaybackEpoch(); const now = performance.now(); this._clock.pause(now); this._currentTime = this._clock.getTime(now); @@ -463,23 +482,50 @@ export class IterablePlayer implements Player { } private async _seekAsync(time: Time): Promise { - this._currentTime = this._clampToRange(time); - this._clock.seek(this._currentTime, performance.now()); + const epoch = this._advancePlaybackEpoch(); + const seekTime = this._clampToRange(time); + const now = performance.now(); + this._cancelRaf(); + this._currentTime = seekTime; + this._clock.seek(seekTime, now); + this._lastTickWallMs = now; this._topicLastMessageNs.clear(); - if (this._cursor) { - await this._cursor.end(); + this._notifyTimeSubscribers(seekTime); + this._lastPipelineEmitMs = 0; + this._emitState(); + + try { + const cursor = this._cursor; this._cursor = undefined; - } + if (cursor) { + await cursor.end(); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } + } - const topics = this._currentTopics(); - if (topics.length > 0) { - const messages = await this._source.getBackfillMessages({ time, topics }); - this._distributeMessages(messages, this._currentTime); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } + const topics = this._currentTopics(); + if (topics.length > 0) { + const messages = await this._source.getBackfillMessages({ time: seekTime, topics }); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } + this._distributeMessages(messages, seekTime); + } + } catch (err) { + if (this._isPlaybackEpochCurrent(epoch)) { + console.warn("IterablePlayer: seek failed", err); + } + } finally { + if (this._isPlaybackEpochCurrent(epoch)) { + this._lastPipelineEmitMs = 0; + this._emitState(); + this._scheduleNextTick(); + } } - - this._notifyTimeSubscribers(this._currentTime); - this._lastPipelineEmitMs = 0; - this._emitState(); } stepBy(deltaMs: number): void { @@ -492,18 +538,27 @@ export class IterablePlayer implements Player { } private async _stepMessageAsync(direction: -1 | 1): Promise { + const epoch = this._advancePlaybackEpoch(); + const referenceTime = this._currentTime; const topics = this._currentTopics(); - if (topics.length === 0) return; + this._cancelRaf(); + if (topics.length === 0) { + this._scheduleNextTick(); + return; + } try { const msg = await this._source.getAdjacentMessage({ - time: this._currentTime, + time: referenceTime, topics, direction: direction === 1 ? "next" : "prev", }); + if (!this._isPlaybackEpochCurrent(epoch)) return; if (!msg) return; - if (this._cursor) { - await this._cursor.end(); - this._cursor = undefined; + const cursor = this._cursor; + this._cursor = undefined; + if (cursor) { + await cursor.end(); + if (!this._isPlaybackEpochCurrent(epoch)) return; } this._currentTime = this._clampToRange(msg.receiveTime); this._clock.seek(this._currentTime, performance.now()); @@ -512,8 +567,13 @@ export class IterablePlayer implements Player { this._notifyTimeSubscribers(this._currentTime); this._lastPipelineEmitMs = 0; this._emitState(); + this._scheduleNextTick(); } catch (err) { console.warn("IterablePlayer: stepMessage failed", err); + } finally { + if (this._isPlaybackEpochCurrent(epoch)) { + this._scheduleNextTick(); + } } } @@ -551,6 +611,7 @@ export class IterablePlayer implements Player { } close(): void { + this._advancePlaybackEpoch(); this._clock.pause(performance.now()); this._isPlaying = false; this._pageSuspended = false; @@ -564,6 +625,7 @@ export class IterablePlayer implements Player { this._initialization = undefined; this._clock = new PlaybackClock(); this._cursor = undefined; + this._fetchEpoch = undefined; this._topicLastMessageNs.clear(); this._highFrequencyConsumersById.clear(); this._highFrequencyConsumersByTopic.clear(); @@ -577,6 +639,22 @@ export class IterablePlayer implements Player { } } + private _advancePlaybackEpoch(): number { + this._playbackEpoch += 1; + return this._playbackEpoch; + } + + private _isPlaybackEpochCurrent(epoch: number): boolean { + return epoch === this._playbackEpoch && this._state.presence !== "closed"; + } + + private _scheduleNextTick(): void { + this._cancelRaf(); + if (this._isPlaying && !this._pageSuspended) { + this._rafId = requestAnimationFrame(this._tickLoop); + } + } + private _notifyTimeSubscribers(time: Time): void { for (const cb of this._timeSubscribers) { cb(time); @@ -809,8 +887,7 @@ export class IterablePlayer implements Player { this._clock.resume(now); this._lastTickWallMs = now; this._pageSuspended = false; - this._cancelRaf(); - this._rafId = requestAnimationFrame(this._tickLoop); + this._scheduleNextTick(); } private async _tickAsync(): Promise { @@ -818,28 +895,23 @@ export class IterablePlayer implements Player { const now = performance.now(); if (this._isFetching) { this._clock.seek(this._currentTime, now); - if (this._isPlaying) { - this._rafId = requestAnimationFrame(this._tickLoop); - } + this._scheduleNextTick(); return; } const tickDurationMs = 1000 / this._samplingFps; if (now - this._lastTickWallMs < tickDurationMs) { - if (this._isPlaying) { - this._rafId = requestAnimationFrame(this._tickLoop); - } + this._scheduleNextTick(); return; } + const epoch = this._playbackEpoch; this._lastTickWallMs = now; const nextTime = this._clampToRange(this._clock.getTime(now)); const currentNs = toNano(this._currentTime); const nextNs = toNano(nextTime); if (nextNs <= currentNs) { - if (this._isPlaying) { - this._rafId = requestAnimationFrame(this._tickLoop); - } + this._scheduleNextTick(); return; } const batchDurationMs = Math.max(1, Number(nextNs - currentNs) / 1e6); @@ -847,18 +919,29 @@ export class IterablePlayer implements Player { if (!this._cursor) { const topics = this._currentTopics(); if (topics.length > 0) { - this._cursor = await this._source.getMessageCursor({ - startTime: this._currentTime, + const cursorStartTime = this._currentTime; + const cursor = await this._source.getMessageCursor({ + startTime: cursorStartTime, topics, latestOnlyTopics: this._latestOnlyHighFrequencyTopics(), }); + if (!this._isPlaybackEpochCurrent(epoch) || toNano(this._currentTime) !== toNano(cursorStartTime)) { + await cursor.end(); + return; + } + this._cursor = cursor; } } if (this._cursor) { this._isFetching = true; + this._fetchEpoch = epoch; try { - const messages = await this._cursor.nextBatch(batchDurationMs, { endTime: nextTime }); + const cursor = this._cursor; + const messages = await cursor.nextBatch(batchDurationMs, { endTime: nextTime }); + if (!this._isPlaybackEpochCurrent(epoch) || cursor !== this._cursor) { + return; + } if (messages.length > 0) { this._emptyBatchStreak = 0; this._distributeMessages(messages); @@ -870,37 +953,54 @@ export class IterablePlayer implements Player { })); } } else { - await this._handleEmptyBatch(now); + await this._handleEmptyBatch(now, epoch); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } } } catch (err) { console.error("Failed to fetch messages", err); } finally { - this._isFetching = false; + if (this._fetchEpoch === epoch) { + this._isFetching = false; + this._fetchEpoch = undefined; + } } } + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } this._currentTime = nextTime; this._clock.seek(this._currentTime, performance.now()); - await this._refreshStaleTopicsFromBackfill(now); + await this._refreshStaleTopicsFromBackfill(now, epoch); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } if (this._initialization && toNano(this._currentTime) >= toNano(this._initialization.end)) { if (this._isLooping) { this._currentTime = this._initialization.start; this._clock.seek(this._currentTime, performance.now()); - if (this._cursor) { - await this._cursor.end(); - this._cursor = undefined; + const cursor = this._cursor; + this._cursor = undefined; + if (cursor) { + await cursor.end(); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } } const topics = this._currentTopics(); if (topics.length > 0) { const messages = await this._source.getBackfillMessages({ time: this._currentTime, topics }); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } this._distributeMessages(messages, this._currentTime); } this._notifyTimeSubscribers(this._currentTime); this._emitState(); - if (this._isPlaying) { - this._rafId = requestAnimationFrame(this._tickLoop); - } + this._scheduleNextTick(); return; } this._currentTime = this._initialization.end; @@ -913,9 +1013,7 @@ export class IterablePlayer implements Player { this._notifyTimeSubscribers(this._currentTime); this._maybeEmitPipelineState(); - if (this._isPlaying) { - this._rafId = requestAnimationFrame(this._tickLoop); - } + this._scheduleNextTick(); } private _clampToRange(time: Time): Time { @@ -1019,7 +1117,7 @@ export class IterablePlayer implements Player { } } - private async _handleEmptyBatch(nowMs: number): Promise { + private async _handleEmptyBatch(nowMs: number, epoch: number): Promise { this._emptyBatchStreak += 1; this._state.progress = { ...this._state.progress, @@ -1029,7 +1127,10 @@ export class IterablePlayer implements Player { }; if (this._emptyBatchStreak === 1) { - await this._rebuildCursorFromCurrentTime(); + await this._rebuildCursorFromCurrentTime(epoch); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } if (this._debugEnabled) { console.debug("[Playback] empty batch -> rebuild cursor " + JSON.stringify({ streak: this._emptyBatchStreak, @@ -1046,15 +1147,19 @@ export class IterablePlayer implements Player { } this._lastFallbackBackfillMs = nowMs; this._fallbackBackfillCount += 1; + const referenceTime = this._currentTime; const topics = this._currentTopics(); if (topics.length === 0) { return; } try { const messages = await this._source.getBackfillMessages({ - time: this._currentTime, + time: referenceTime, topics, }); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } // For latched topics (URDF, static TF), the backfill returns the same // message we already distributed – skip those so downstream panels don't // needlessly rebuild (URDF rebuild fetches + parses meshes, which was @@ -1063,7 +1168,7 @@ export class IterablePlayer implements Player { if (freshMessages.length === 0) { return; } - this._distributeMessages(freshMessages, this._currentTime); + this._distributeMessages(freshMessages, referenceTime); if (this._debugEnabled) { console.debug("[Playback] empty batch -> fallback backfill " + JSON.stringify({ streak: this._emptyBatchStreak, @@ -1077,22 +1182,32 @@ export class IterablePlayer implements Player { } } - private async _rebuildCursorFromCurrentTime(): Promise { - if (this._cursor) { - await this._cursor.end(); - this._cursor = undefined; + private async _rebuildCursorFromCurrentTime(epoch: number): Promise { + const previousCursor = this._cursor; + this._cursor = undefined; + if (previousCursor) { + await previousCursor.end(); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } } const topics = this._currentTopics(); if (topics.length === 0) return; this._cursorRebuildCount += 1; - this._cursor = await this._source.getMessageCursor({ - startTime: this._currentTime, + const cursorStartTime = this._currentTime; + const cursor = await this._source.getMessageCursor({ + startTime: cursorStartTime, topics, latestOnlyTopics: this._latestOnlyHighFrequencyTopics(), }); + if (!this._isPlaybackEpochCurrent(epoch) || toNano(this._currentTime) !== toNano(cursorStartTime)) { + await cursor.end(); + return; + } + this._cursor = cursor; } - private async _refreshStaleTopicsFromBackfill(nowMs: number): Promise { + private async _refreshStaleTopicsFromBackfill(nowMs: number, epoch: number): Promise { if (nowMs - this._lastStaleRefreshMs < STALE_TOPIC_REFRESH_COOLDOWN_MS) { return; } @@ -1110,11 +1225,15 @@ export class IterablePlayer implements Player { return; } this._lastStaleRefreshMs = nowMs; + const referenceTime = this._currentTime; try { const messages = await this._source.getBackfillMessages({ - time: this._currentTime, + time: referenceTime, topics: staleTopics, }); + if (!this._isPlaybackEpochCurrent(epoch)) { + return; + } // Same reasoning as in _handleEmptyBatch: latched topics would otherwise // get re-delivered on every refresh tick (~5 Hz), causing panels like // the 3D/URDF renderer to rebuild from scratch and leak GPU buffers. @@ -1122,7 +1241,7 @@ export class IterablePlayer implements Player { if (freshMessages.length === 0) { return; } - this._distributeMessages(freshMessages, this._currentTime); + this._distributeMessages(freshMessages, referenceTime); if (this._debugEnabled) { console.debug("[Playback] stale refresh " + JSON.stringify({ staleTopicCount: staleTopics.length, From 9bf3152ab7336add79503baaafe5b92e4cc7211c Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:02:13 +0800 Subject: [PATCH 08/15] fix(tsconfig): remove deprecated baseUrl and ignoreDeprecations for TS 6. Paths already resolve without baseUrl, which avoids IDE errors on ignoreDeprecations 6.0 and clears TS 7 migration warnings. --- tsconfig.app.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/tsconfig.app.json b/tsconfig.app.json index ab02191..10792ed 100644 --- a/tsconfig.app.json +++ b/tsconfig.app.json @@ -1,7 +1,5 @@ { "compilerOptions": { - "ignoreDeprecations": "6.0", - "baseUrl": ".", "paths": { "@/*": ["./src/*"] }, From a2860a57de3aac5c8cc9e85567c64ea982de18f8 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:04:58 +0800 Subject: [PATCH 09/15] test(urdf-debug): run MCAP processor tests in memory Extract processMcapBuffer for in-process Vitest checks instead of spawning node with mkdtemp under the repo root, and ignore stray .tmp-urdf-debug-* directories. --- .gitignore | 3 +- .../panels/UrdfDebug/mcapProcessor.ts | 275 ++++++++++++++++++ .../panels/UrdfDebug/scriptTemplates.test.ts | 74 ++--- 3 files changed, 303 insertions(+), 49 deletions(-) create mode 100644 src/features/panels/UrdfDebug/mcapProcessor.ts diff --git a/.gitignore b/.gitignore index 27de6c6..22dbe92 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,5 @@ yarn.lock # test files /public/examples/ /test-results/ -/playwright-report/ \ No newline at end of file +/playwright-report/ +.tmp-urdf-debug-*/ \ No newline at end of file diff --git a/src/features/panels/UrdfDebug/mcapProcessor.ts b/src/features/panels/UrdfDebug/mcapProcessor.ts new file mode 100644 index 0000000..c4caf50 --- /dev/null +++ b/src/features/panels/UrdfDebug/mcapProcessor.ts @@ -0,0 +1,275 @@ +import { McapIndexedReader, McapWriter, type Channel, type Schema } from '@mcap/core'; +import { MessageReader, MessageWriter } from '@foxglove/rosmsg2-serialization'; +import rosmsg from '@foxglove/rosmsg'; +import { JointState2TF } from './embedded/fkEngine.js'; +import { applyJointMapping } from './jointStateMapping'; +import type { UrdfDebugRecipe } from './recipe'; +import { applyUrdfVisualCorrection } from './urdfVisualCorrection'; + +const { parseMessageDefinition } = rosmsg; + +const ROS2_DEFINITIONS = [ + { name: 'builtin_interfaces/msg/Time', definitions: [{ name: 'sec', type: 'int32' }, { name: 'nanosec', type: 'uint32' }] }, + { name: 'std_msgs/msg/Header', definitions: [{ name: 'stamp', type: 'builtin_interfaces/msg/Time', isComplex: true }, { name: 'frame_id', type: 'string' }] }, + { name: 'geometry_msgs/msg/Vector3', definitions: [{ name: 'x', type: 'float64' }, { name: 'y', type: 'float64' }, { name: 'z', type: 'float64' }] }, + { name: 'geometry_msgs/msg/Quaternion', definitions: [{ name: 'x', type: 'float64' }, { name: 'y', type: 'float64' }, { name: 'z', type: 'float64' }, { name: 'w', type: 'float64' }] }, + { name: 'geometry_msgs/msg/Transform', definitions: [{ name: 'translation', type: 'geometry_msgs/msg/Vector3', isComplex: true }, { name: 'rotation', type: 'geometry_msgs/msg/Quaternion', isComplex: true }] }, + { name: 'geometry_msgs/msg/TransformStamped', definitions: [{ name: 'header', type: 'std_msgs/msg/Header', isComplex: true }, { name: 'child_frame_id', type: 'string' }, { name: 'transform', type: 'geometry_msgs/msg/Transform', isComplex: true }] }, + { name: 'tf2_msgs/msg/TFMessage', definitions: [{ name: 'transforms', type: 'geometry_msgs/msg/TransformStamped', isArray: true, isComplex: true }] }, + { name: 'std_msgs/msg/String', definitions: [{ name: 'data', type: 'string' }] }, + { name: 'sensor_msgs/msg/JointState', definitions: [{ name: 'header', type: 'std_msgs/msg/Header', isComplex: true }, { name: 'name', type: 'string', isArray: true }, { name: 'position', type: 'float64', isArray: true }, { name: 'velocity', type: 'float64', isArray: true }, { name: 'effort', type: 'float64', isArray: true }] }, +]; + +class BufferReadable { + constructor(private readonly buffer: Uint8Array) {} + + size() { + return BigInt(this.buffer.byteLength); + } + + async read(offset: bigint, size: bigint) { + const start = Number(offset); + return this.buffer.subarray(start, start + Number(size)); + } +} + +class BufferWritable { + #chunks: Buffer[] = []; + #pos = 0n; + + position() { + return this.#pos; + } + + async write(buffer: Uint8Array) { + const chunk = Buffer.from(buffer); + this.#chunks.push(chunk); + this.#pos += BigInt(chunk.byteLength); + } + + toBuffer() { + return Buffer.concat(this.#chunks); + } +} + +function prepareUrdfFromRecipe(urdfXml: string, recipe: UrdfDebugRecipe): string { + const urdf = recipe.urdf ?? { rotateMeshVisuals: false, visualRpyOffset: [0, 0, 0] as [number, number, number] }; + return applyUrdfVisualCorrection(urdfXml, { + rotateMeshVisuals: !!urdf.rotateMeshVisuals, + visualRpyOffset: Array.isArray(urdf.visualRpyOffset) + ? (urdf.visualRpyOffset as [number, number, number]) + : [0, 0, 0], + }); +} + +function normalizeJointState(raw: unknown) { + const msg = raw as { + name?: unknown; + position?: unknown; + header?: { stamp?: { sec?: number; nanosec?: number }; frame_id?: string }; + }; + const name = Array.isArray(msg?.name) ? msg.name.map(String) : []; + const position = Array.isArray(msg?.position) ? msg.position.map((v) => Number(v) || 0) : []; + const header = + msg?.header && typeof msg.header === 'object' + ? msg.header + : { stamp: { sec: 0, nanosec: 0 }, frame_id: '' }; + return { header, name, position }; +} + +function buildChannelDeserializer(channel: Channel, schema: Schema | undefined) { + if (channel.messageEncoding === 'json') { + const decoder = new TextDecoder(); + return (data: Uint8Array) => JSON.parse(decoder.decode(data)) as unknown; + } + if (!schema?.data?.length) { + throw new Error(`Missing schema for ${channel.topic}`); + } + const text = new TextDecoder().decode(schema.data); + const reader = new MessageReader(parseMessageDefinition(text)); + return (data: Uint8Array) => reader.readMessage(data); +} + +function buildChannelSerializer(schemaName: string, writers: Record) { + const writer = writers[schemaName]; + if (!writer) throw new Error(`Missing writer for ${schemaName}`); + return (msg: unknown) => writer.writeMessage(msg); +} + +export type ProcessMcapBufferOptions = { + input: Uint8Array; + recipe: UrdfDebugRecipe; + urdfXml: string; + overwriteTopics?: boolean; +}; + +/** In-memory MCAP processor (same logic as exported TypeScript scripts). */ +export async function processMcapBuffer({ + input, + recipe, + urdfXml, + overwriteTopics = false, +}: ProcessMcapBufferOptions): Promise<{ output: Uint8Array; processedJointStates: number }> { + const tfTopic = recipe.outputTfTopic ?? '/tf'; + const robotDescTopic = recipe.outputRobotDescriptionTopic ?? '/robot_description'; + const jointTopic = recipe.jointStateTopic; + if (!jointTopic) throw new Error('recipe.jointStateTopic is required'); + + const reader = await McapIndexedReader.Initialize({ + readable: new BufferReadable(input), + }); + + let hasTf = false; + let hasRobotDesc = false; + let jointChannel: Channel | undefined; + for (const channel of reader.channelsById.values()) { + if (channel.topic === tfTopic) hasTf = true; + if (channel.topic === robotDescTopic) hasRobotDesc = true; + if (channel.topic === jointTopic) jointChannel = channel; + } + if (!jointChannel) throw new Error(`JointState topic not found: ${jointTopic}`); + if (!overwriteTopics && (hasTf || hasRobotDesc)) { + throw new Error( + 'Input already contains /tf or /robot_description. Pass --overwrite-topics to replace them.', + ); + } + + const writable = new BufferWritable(); + const writer = new McapWriter({ writable }); + await writer.start({ + profile: reader.header?.profile ?? 'ros2', + library: 'urdf-debug-processor', + }); + + const schemaMap = new Map(); + const channelMap = new Map(); + for (const schema of reader.schemasById.values()) { + schemaMap.set(schema.id, await writer.registerSchema(schema)); + } + for (const channel of reader.channelsById.values()) { + if (overwriteTopics && (channel.topic === tfTopic || channel.topic === robotDescTopic)) continue; + const mapped = { ...channel, schemaId: schemaMap.get(channel.schemaId) ?? 0 }; + channelMap.set(channel.id, await writer.registerChannel(mapped)); + } + + const outEncoding = jointChannel.messageEncoding ?? 'json'; + const preparedUrdf = prepareUrdfFromRecipe(urdfXml, recipe); + const fkEngine = JointState2TF.fromXml({ xml: preparedUrdf }); + const jointSchema = reader.schemasById.get(jointChannel.schemaId); + const deserializeJoint = buildChannelDeserializer(jointChannel, jointSchema); + + const writers: Record = { + 'tf2_msgs/msg/TFMessage': new MessageWriter(ROS2_DEFINITIONS), + 'std_msgs/msg/String': new MessageWriter(ROS2_DEFINITIONS), + }; + + let tfChannelId: number; + let robotDescChannelId: number; + if (outEncoding === 'json') { + const tfSchemaId = await writer.registerSchema({ + name: 'tf2_msgs/msg/TFMessage', + encoding: 'jsonschema', + data: new TextEncoder().encode('{"type":"object"}'), + }); + const robotSchemaId = await writer.registerSchema({ + name: 'std_msgs/msg/String', + encoding: 'jsonschema', + data: new TextEncoder().encode('{"type":"object"}'), + }); + tfChannelId = await writer.registerChannel({ + schemaId: tfSchemaId, + topic: tfTopic, + messageEncoding: 'json', + metadata: new Map(), + }); + robotDescChannelId = await writer.registerChannel({ + schemaId: robotSchemaId, + topic: robotDescTopic, + messageEncoding: 'json', + metadata: new Map(), + }); + } else { + const tfSchemaId = await writer.registerSchema({ + name: 'tf2_msgs/msg/TFMessage', + encoding: 'ros2msg', + data: new TextEncoder().encode('geometry_msgs/TransformStamped[] transforms\n'), + }); + const robotSchemaId = await writer.registerSchema({ + name: 'std_msgs/msg/String', + encoding: 'ros2msg', + data: new TextEncoder().encode('string data\n'), + }); + tfChannelId = await writer.registerChannel({ + schemaId: tfSchemaId, + topic: tfTopic, + messageEncoding: 'cdr', + metadata: new Map(), + }); + robotDescChannelId = await writer.registerChannel({ + schemaId: robotSchemaId, + topic: robotDescTopic, + messageEncoding: 'cdr', + metadata: new Map(), + }); + } + + const serializeTf = + outEncoding === 'json' + ? (msg: unknown) => new TextEncoder().encode(JSON.stringify(msg)) + : buildChannelSerializer('tf2_msgs/msg/TFMessage', writers); + const serializeString = + outEncoding === 'json' + ? (msg: unknown) => new TextEncoder().encode(JSON.stringify(msg)) + : buildChannelSerializer('std_msgs/msg/String', writers); + + let robotDescWritten = false; + let tfSeq = 0; + let processedJointStates = 0; + + for await (const message of reader.readMessages()) { + const channel = reader.channelsById.get(message.channelId); + if (!channel) continue; + if (overwriteTopics && (channel.topic === tfTopic || channel.topic === robotDescTopic)) continue; + + const mappedChannelId = channelMap.get(message.channelId); + if (mappedChannelId != null) { + await writer.addMessage({ ...message, channelId: mappedChannelId }); + } + + if (channel.id !== jointChannel.id) continue; + + const rawJoint = normalizeJointState(deserializeJoint(message.data)); + const mapped = applyJointMapping( + { name: rawJoint.name, position: rawJoint.position }, + recipe.rules ?? [], + ); + const tfMsg = fkEngine.computeFromJointState( + { header: rawJoint.header, name: mapped.name, position: mapped.position }, + { publishTimeNs: message.logTime }, + ); + + if (!robotDescWritten) { + await writer.addMessage({ + channelId: robotDescChannelId, + sequence: 0, + logTime: message.logTime, + publishTime: message.publishTime, + data: serializeString({ data: preparedUrdf }), + }); + robotDescWritten = true; + } + + tfSeq += 1; + await writer.addMessage({ + channelId: tfChannelId, + sequence: tfSeq, + logTime: message.logTime, + publishTime: message.publishTime, + data: serializeTf(tfMsg), + }); + processedJointStates += 1; + } + + await writer.end(); + return { output: writable.toBuffer(), processedJointStates }; +} diff --git a/src/features/panels/UrdfDebug/scriptTemplates.test.ts b/src/features/panels/UrdfDebug/scriptTemplates.test.ts index 0b907c6..4bb2612 100644 --- a/src/features/panels/UrdfDebug/scriptTemplates.test.ts +++ b/src/features/panels/UrdfDebug/scriptTemplates.test.ts @@ -1,8 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { execFileSync } from 'node:child_process'; -import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; -import { join } from 'node:path'; import { McapStreamReader, McapWriter } from '@mcap/core'; +import { processMcapBuffer } from './mcapProcessor'; import { generatePythonScript, generateTypeScriptScript } from './scriptTemplates'; import type { UrdfDebugRecipe } from './recipe'; @@ -41,7 +39,7 @@ class BufferWritable { return this.#pos; } - write(buffer: Uint8Array) { + async write(buffer: Uint8Array) { const b = Buffer.from(buffer); this.#chunks.push(b); this.#pos += BigInt(b.byteLength); @@ -53,7 +51,7 @@ class BufferWritable { } } -async function writeFixtureMcap(outputPath: string): Promise { +async function buildFixtureMcap(): Promise { const writable = new BufferWritable(); const writer = new McapWriter({ writable, @@ -95,12 +93,12 @@ async function writeFixtureMcap(outputPath: string): Promise { }); await writer.end(); - writeFileSync(outputPath, writable.getBuffer()); + return writable.getBuffer(); } -function listTopics(mcapPath: string): string[] { +function listTopics(mcap: Uint8Array): string[] { const reader = new McapStreamReader(); - reader.append(readFileSync(mcapPath)); + reader.append(mcap); const topics = new Set(); for (let record = reader.nextRecord(); record; record = reader.nextRecord()) { if (record.type === 'Channel') { @@ -110,9 +108,9 @@ function listTopics(mcapPath: string): string[] { return [...topics].sort(); } -function readRobotDescription(mcapPath: string): string { +function readRobotDescription(mcap: Uint8Array): string { const reader = new McapStreamReader(); - reader.append(readFileSync(mcapPath)); + reader.append(mcap); const topicsByChannel = new Map(); let robotDescription = ''; for (let record = reader.nextRecord(); record; record = reader.nextRecord()) { @@ -166,25 +164,15 @@ describe('scriptTemplates', () => { }); it('TypeScript processor rewrites test MCAP with /tf and /robot_description', async () => { - const dir = mkdtempSync(join(process.cwd(), '.tmp-urdf-debug-')); - const inputPath = join(dir, 'input.mcap'); - const outputPath = join(dir, 'out.mcap'); - const urdfPath = join(dir, 'test.urdf'); - const scriptPath = join(dir, 'process.mjs'); - const recipePath = join(dir, 'recipe.json'); - - await writeFixtureMcap(inputPath); - writeFileSync(urdfPath, FIXTURE_URDF); - writeFileSync(recipePath, JSON.stringify(sampleRecipe, null, 2)); - writeFileSync(scriptPath, generateTypeScriptScript(sampleRecipe)); - - execFileSync( - 'node', - [scriptPath, inputPath, outputPath, recipePath, urdfPath, '--overwrite-topics'], - { stdio: 'pipe', cwd: process.cwd() }, - ); - - const topics = listTopics(outputPath); + const input = await buildFixtureMcap(); + const { output } = await processMcapBuffer({ + input, + recipe: sampleRecipe, + urdfXml: FIXTURE_URDF, + overwriteTopics: true, + }); + + const topics = listTopics(output); expect(topics).toContain('/tf'); expect(topics).toContain('/robot_description'); expect(topics).toContain('/joint_states'); @@ -195,25 +183,15 @@ describe('scriptTemplates', () => { ...sampleRecipe, urdf: { rotateMeshVisuals: true, visualRpyOffset: [0, 0, 0] }, }; - const dir = mkdtempSync(join(process.cwd(), '.tmp-urdf-debug-')); - const inputPath = join(dir, 'input.mcap'); - const outputPath = join(dir, 'out.mcap'); - const urdfPath = join(dir, 'test.urdf'); - const scriptPath = join(dir, 'process.mjs'); - const recipePath = join(dir, 'recipe.json'); - - await writeFixtureMcap(inputPath); - writeFileSync(urdfPath, NO_ORIGIN_MESH_URDF); - writeFileSync(recipePath, JSON.stringify(recipe, null, 2)); - writeFileSync(scriptPath, generateTypeScriptScript(recipe)); - - execFileSync( - 'node', - [scriptPath, inputPath, outputPath, recipePath, urdfPath, '--overwrite-topics'], - { stdio: 'pipe', cwd: process.cwd() }, - ); - - const robotDescription = readRobotDescription(outputPath); + const input = await buildFixtureMcap(); + const { output } = await processMcapBuffer({ + input, + recipe, + urdfXml: NO_ORIGIN_MESH_URDF, + overwriteTopics: true, + }); + + const robotDescription = readRobotDescription(output); expect(robotDescription).toContain('rpy="-1.5707963267948966 0 0"'); }); }); From 273e1849fd98bd22171e31a62678d7d09ac0f016 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:09:17 +0800 Subject: [PATCH 10/15] perf(raw-messages): keep high-frequency updates off the React render path Route topic updates through refs and shared scheduleFrame DOM patching so image topics no longer trigger per-message reconciliation and layouts. --- src/features/panels/RawMessages/Component.tsx | 577 +++++++++++------- .../panels/RawMessages/describeValue.test.ts | 23 + 2 files changed, 394 insertions(+), 206 deletions(-) create mode 100644 src/features/panels/RawMessages/describeValue.test.ts diff --git a/src/features/panels/RawMessages/Component.tsx b/src/features/panels/RawMessages/Component.tsx index 18ebacd..f63e282 100644 --- a/src/features/panels/RawMessages/Component.tsx +++ b/src/features/panels/RawMessages/Component.tsx @@ -4,11 +4,12 @@ import { useIntl } from 'react-intl'; import { toast } from 'sonner'; import { messageBus } from '@/core/pipeline/messageBus'; import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline'; -import { useTopicSeq } from '@/core/pipeline/useMessageBus'; import type { MessageEvent } from '@/core/types/ros'; import type { MessagePipelineState } from '@/core/pipeline/store'; import type { Player } from '@/core/types/player'; import { pickDefaultRawMessagesTopic } from '@/features/layout/autoLayout/pickDefaultRawMessagesTopic'; +import { isRosImageSchema } from '@/shared/ros/rosMessageTypes'; +import { scheduleFrame } from '@/shared/utils/rafScheduler'; import { TopicQuickPicker } from '../framework/TopicQuickPicker'; import type { RawMessagesConfig } from './defaults'; import { buildRowsForShape, type FlatRow } from './shapeTree'; @@ -35,12 +36,22 @@ interface ValueVisual { kind: ValueKind; } -interface StreamStats { - incomingUpdates: number; - displayedUpdates: number; - droppedUpdates: number; - shapeMisses: number; - framePatchMs: number; +interface ScrollWindow { + startRow: number; + endRow: number; + totalRows: number; +} + +interface DescribeValueOptions { + hideBinaryHex?: boolean; +} + +interface RawMessageRowProps { + row: FlatRow; + expanded: boolean; + onToggle: (path: string) => void; + onCopy: (path: string) => void; + registerValueNode: (path: string, node: HTMLSpanElement | null) => void; } const ROW_HEIGHT = 22; @@ -48,6 +59,7 @@ const OVERSCAN_ROWS = 8; const MAX_VISIBLE_PATCH_ROWS = 1200; const MAX_OBJECT_PREVIEW_FIELDS = 3; const MAX_PREVIEW_STRING_LENGTH = 80; +const LARGE_BINARY_THRESHOLD = 1024; function toHex(data: Uint8Array): string { let out = ''; @@ -109,6 +121,17 @@ function getVisibleRows(rows: FlatRow[], expandedPaths: Set): FlatRow[] return out; } +function computeScrollWindow( + scrollTop: number, + viewportHeight: number, + totalRows: number, +): Pick { + const startRow = Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - OVERSCAN_ROWS); + const visibleCount = Math.ceil(viewportHeight / ROW_HEIGHT) + OVERSCAN_ROWS * 2; + const endRow = Math.min(totalRows, startRow + visibleCount); + return { startRow, endRow }; +} + function previewPrimitive(value: unknown): string { if (value === null) return 'null'; if (typeof value === 'string') { @@ -144,8 +167,18 @@ function previewObject(value: Record): string | null { return `{${fields.join(',')}${keys.length > MAX_OBJECT_PREVIEW_FIELDS ? ',...' : ''}}`; } -function describeValue(value: unknown, maxBinaryPreviewBytes: number): ValueVisual { +export function describeValue( + value: unknown, + maxBinaryPreviewBytes: number, + options?: DescribeValueOptions, +): ValueVisual { if (value instanceof Uint8Array) { + if (options?.hideBinaryHex || value.byteLength > LARGE_BINARY_THRESHOLD) { + return { + text: `Uint8Array(${value.byteLength}) [preview hidden]`, + kind: 'binary', + }; + } const head = value.subarray(0, Math.min(value.byteLength, maxBinaryPreviewBytes)); return { text: `Uint8Array(${value.byteLength}) 0x${toHex(head)}${value.byteLength > head.byteLength ? '...' : ''}`, @@ -153,7 +186,7 @@ function describeValue(value: unknown, maxBinaryPreviewBytes: number): ValueVisu }; } if (value instanceof ArrayBuffer) { - return describeValue(new Uint8Array(value), maxBinaryPreviewBytes); + return describeValue(new Uint8Array(value), maxBinaryPreviewBytes, options); } if (Array.isArray(value)) return { text: `Array(${value.length})`, kind: 'array' }; if (isPlainObject(value)) { @@ -224,61 +257,62 @@ async function copyText(text: string): Promise { } } -function useScheduledTopicMessage( - topic: string, - topicSeq: number, - uiRefreshHz: number, - paused: boolean, - latestOnly: boolean, -) { - const [displayed, setDisplayed] = useState(() => messageBus.getLastMessage(topic)); - const [stats, setStats] = useState({ - incomingUpdates: 0, - displayedUpdates: 0, - droppedUpdates: 0, - shapeMisses: 0, - framePatchMs: 0, - }); - const latestRef = useRef(messageBus.getLastMessage(topic)); - const pendingRef = useRef(0); - const lastDisplayedAtRef = useRef(0); - - useEffect(() => { - latestRef.current = messageBus.getLastMessage(topic); - pendingRef.current = latestOnly ? 1 : pendingRef.current + 1; - setStats((prev) => ({ ...prev, incomingUpdates: prev.incomingUpdates + 1 })); - }, [latestOnly, topic, topicSeq]); - - useEffect(() => { - pendingRef.current = 0; - setDisplayed(messageBus.getLastMessage(topic)); - lastDisplayedAtRef.current = performance.now(); - }, [topic]); +function applyValueVisual(node: HTMLSpanElement, visual: ValueVisual): void { + node.textContent = visual.text; + if (node.dataset.kind !== visual.kind) { + node.dataset.kind = visual.kind; + node.style.color = valueColor(visual.kind); + } +} - useEffect(() => { - let rafId = 0; - const minInterval = 1000 / Math.max(1, uiRefreshHz); - const tick = () => { - const now = performance.now(); - if (!paused && pendingRef.current > 0 && now - lastDisplayedAtRef.current >= minInterval) { - const dropped = Math.max(0, pendingRef.current - 1); - pendingRef.current = 0; - setDisplayed(latestRef.current); - setStats((prev) => ({ - ...prev, - displayedUpdates: prev.displayedUpdates + 1, - droppedUpdates: prev.droppedUpdates + dropped, - })); - lastDisplayedAtRef.current = now; - } - rafId = requestAnimationFrame(tick); - }; - rafId = requestAnimationFrame(tick); - return () => cancelAnimationFrame(rafId); - }, [paused, uiRefreshHz]); +const RawMessageRow = React.memo(function RawMessageRow({ + row, + expanded, + onToggle, + onCopy, + registerValueNode, +}: RawMessageRowProps) { + const handleToggle = useCallback(() => { + if (row.expandable) onToggle(row.path); + }, [onToggle, row.expandable, row.path]); + + const handleCopy = useCallback(() => { + void onCopy(row.path); + }, [onCopy, row.path]); + + const valueRef = useCallback( + (node: HTMLSpanElement | null) => { + registerValueNode(row.path, node); + }, + [registerValueNode, row.path], + ); - return { displayed, stats, setStats }; -} + return ( +
+ + {row.key}: + + +
+ ); +}); export const RawMessagesPanel: React.FC = ({ player, @@ -296,6 +330,74 @@ export const RawMessagesPanel: React.FC = ({ const { formatMessage } = useIntl(); const topics = useMessagePipeline((state: MessagePipelineState) => state.sortedTopics); const didAutoPickTopicRef = useRef(false); + const isImageTopic = useMemo(() => { + const topicType = topics.find((entry) => entry.name === topic)?.type ?? ''; + return isRosImageSchema(topicType); + }, [topic, topics]); + + const latestRef = useRef(messageBus.getLastMessage(topic)); + const pendingRef = useRef(0); + const lastDisplayedAtRef = useRef(0); + const pausedRef = useRef(pauseUpdates); + const uiRefreshHzRef = useRef(uiRefreshHz); + const latestOnlyRef = useRef(latestOnly); + const hasMessageRef = useRef(!!messageBus.getLastMessage(topic)); + const shapeRowsRef = useRef([]); + const shapeSignatureRef = useRef(''); + const expandedPathsRef = useRef>(new Set(['message'])); + const scrollTopRef = useRef(0); + const viewportHeightRef = useRef(240); + const scrollWindowRef = useRef({ startRow: 0, endRow: 0, totalRows: 0 }); + const configRef = useRef({ + maxExpandedDepth, + maxRows, + maxBinaryPreviewBytes, + isImageTopic, + }); + const viewportRef = useRef(null); + const valueNodeRefs = useRef>(new Map()); + const latestValueVisualRef = useRef>(new Map()); + const pendingPatchRef = useRef>(new Map()); + const didInitializeExpansionRef = useRef(false); + + const [hasMessage, setHasMessage] = useState(() => !!messageBus.getLastMessage(topic)); + const [expandedPaths, setExpandedPaths] = useState>(() => new Set(['message'])); + const [shapeRows, setShapeRows] = useState([]); + const [shapeSignature, setShapeSignature] = useState(''); + const [scrollWindow, setScrollWindow] = useState({ startRow: 0, endRow: 0, totalRows: 0 }); + + useEffect(() => { + pausedRef.current = pauseUpdates; + }, [pauseUpdates]); + + useEffect(() => { + uiRefreshHzRef.current = uiRefreshHz; + }, [uiRefreshHz]); + + useEffect(() => { + latestOnlyRef.current = latestOnly; + }, [latestOnly]); + + useEffect(() => { + configRef.current = { + maxExpandedDepth, + maxRows, + maxBinaryPreviewBytes, + isImageTopic, + }; + }, [isImageTopic, maxBinaryPreviewBytes, maxExpandedDepth, maxRows]); + + useEffect(() => { + expandedPathsRef.current = expandedPaths; + }, [expandedPaths]); + + useEffect(() => { + shapeRowsRef.current = shapeRows; + }, [shapeRows]); + + useEffect(() => { + shapeSignatureRef.current = shapeSignature; + }, [shapeSignature]); useEffect(() => { if (didAutoPickTopicRef.current) return; @@ -319,47 +421,95 @@ export const RawMessagesPanel: React.FC = ({ return () => player.unregisterSubscriptions(panelId); }, [player, panelId, topic]); - const topicSeq = useTopicSeq(topic); - const { displayed: displayMessage, setStats } = useScheduledTopicMessage( - topic, - topicSeq, - uiRefreshHz, - pauseUpdates, - latestOnly, + const applyDomPatch = useCallback(() => { + for (const [path, visual] of pendingPatchRef.current) { + const node = valueNodeRefs.current.get(path); + if (node) { + applyValueVisual(node, visual); + } + } + pendingPatchRef.current.clear(); + }, []); + + const patchVisibleValues = useCallback( + (message: MessageEvent) => { + const visible = getVisibleRows(shapeRowsRef.current, expandedPathsRef.current); + const { startRow, endRow } = scrollWindowRef.current; + const patchRows = visible.slice(startRow, endRow); + const maxPatchRows = Math.min(patchRows.length, MAX_VISIBLE_PATCH_ROWS); + const { maxBinaryPreviewBytes: previewBytes, isImageTopic: hideHex } = configRef.current; + + for (let i = 0; i < maxPatchRows; i++) { + const row = patchRows[i]; + if (!row) continue; + const value = readValueAtPath(message.message, row.path); + const visual = describeValue(value, previewBytes, { hideBinaryHex: hideHex }); + latestValueVisualRef.current.set(row.path, visual); + const previousText = valueNodeRefs.current.get(row.path)?.textContent ?? null; + if (previousText !== visual.text) { + pendingPatchRef.current.set(row.path, visual); + } + } + + if (pendingPatchRef.current.size > 0) { + scheduleFrame(applyDomPatch); + } + }, + [applyDomPatch], ); - const [expandedPaths, setExpandedPaths] = useState>(() => new Set(['message'])); - const [shapeRows, setShapeRows] = useState([]); - const [shapeSignature, setShapeSignature] = useState(''); - const [viewportHeight, setViewportHeight] = useState(240); - const [scrollTop, setScrollTop] = useState(0); - const viewportRef = useRef(null); - const valueNodeRefs = useRef>(new Map()); - const latestValueVisualRef = useRef>(new Map()); - const pendingPatchRef = useRef>(new Map()); - const patchRafRef = useRef(null); - const didInitializeExpansionRef = useRef(false); - const visibleRows = useMemo(() => { - if (shapeRows.length === 0) return []; - return getVisibleRows(shapeRows, expandedPaths); - }, [expandedPaths, shapeRows]); + const applyScrollWindow = useCallback(() => { + const visible = getVisibleRows(shapeRowsRef.current, expandedPathsRef.current); + const totalRows = visible.length; + const nextWindow = computeScrollWindow(scrollTopRef.current, viewportHeightRef.current, totalRows); + const next: ScrollWindow = { ...nextWindow, totalRows }; + + setScrollWindow((prev) => { + if ( + prev.startRow === next.startRow && + prev.endRow === next.endRow && + prev.totalRows === next.totalRows + ) { + return prev; + } + scrollWindowRef.current = next; + return next; + }); + }, []); - const totalRows = visibleRows.length; - const startRow = Math.max(0, Math.floor(scrollTop / ROW_HEIGHT) - OVERSCAN_ROWS); - const visibleCount = Math.ceil(viewportHeight / ROW_HEIGHT) + OVERSCAN_ROWS * 2; - const endRow = Math.min(totalRows, startRow + visibleCount); - const windowRows = visibleRows.slice(startRow, endRow); + const flushPending = useCallback(() => { + const now = performance.now(); + const minInterval = 1000 / Math.max(1, uiRefreshHzRef.current); - useEffect(() => { - if (!displayMessage) return; - const nextShape = buildRowsForShape(displayMessage.message, maxExpandedDepth, maxRows); - const shapeChanged = nextShape.signature !== shapeSignature; - let patchRows = visibleRows.slice(startRow, endRow); - if (shapeChanged) { + if (pausedRef.current) { + return; + } + + if (pendingRef.current <= 0) { + return; + } + + if (now - lastDisplayedAtRef.current < minInterval) { + scheduleFrame(flushPending); + return; + } + + pendingRef.current = 0; + lastDisplayedAtRef.current = now; + + const message = latestRef.current; + if (!message) { + return; + } + + const { maxExpandedDepth: depth, maxRows: rowLimit } = configRef.current; + const nextShape = buildRowsForShape(message.message, depth, rowLimit); + if (nextShape.signature !== shapeSignatureRef.current) { + shapeSignatureRef.current = nextShape.signature; + shapeRowsRef.current = nextShape.rows; setShapeRows(nextShape.rows); setShapeSignature(nextShape.signature); - setStats((prev) => ({ ...prev, shapeMisses: prev.shapeMisses + 1 })); - let expandedForShape = expandedPaths; + if (!didInitializeExpansionRef.current) { const nextExpanded = new Set(['message']); for (const row of nextShape.rows) { @@ -367,106 +517,145 @@ export const RawMessagesPanel: React.FC = ({ nextExpanded.add(row.path); } } + expandedPathsRef.current = nextExpanded; setExpandedPaths(nextExpanded); - expandedForShape = nextExpanded; didInitializeExpansionRef.current = true; } - patchRows = getVisibleRows(nextShape.rows, expandedForShape).slice(startRow, startRow + visibleCount); + + applyScrollWindow(); + scheduleFrame(() => { + if (latestRef.current) { + patchVisibleValues(latestRef.current); + } + }); + return; } - const patchMap = new Map(); - const maxPatchRows = Math.min(patchRows.length, MAX_VISIBLE_PATCH_ROWS); - for (let i = 0; i < maxPatchRows; i++) { - const row = patchRows[i]; - if (!row) continue; - const value = readValueAtPath(displayMessage.message, row.path); - const visual = describeValue(value, maxBinaryPreviewBytes); - latestValueVisualRef.current.set(row.path, visual); - const previousText = valueNodeRefs.current.get(row.path)?.textContent ?? null; - if (previousText !== visual.text) { - patchMap.set(row.path, visual); - } + patchVisibleValues(message); + }, [applyScrollWindow, patchVisibleValues]); + + useEffect(() => { + if (!pauseUpdates && pendingRef.current > 0) { + scheduleFrame(flushPending); } - if (patchMap.size > 0) { - for (const [path, visual] of patchMap) { - pendingPatchRef.current.set(path, visual); - } - if (patchRafRef.current == null) { - patchRafRef.current = requestAnimationFrame(() => { - const frameStarted = performance.now(); - for (const [path, visual] of pendingPatchRef.current) { - const node = valueNodeRefs.current.get(path); - if (node) { - node.textContent = visual.text; - if (node.dataset.kind !== visual.kind) { - node.dataset.kind = visual.kind; - node.style.color = valueColor(visual.kind); - } - } - } - pendingPatchRef.current.clear(); - patchRafRef.current = null; - setStats((prev) => ({ ...prev, framePatchMs: performance.now() - frameStarted })); - }); - } + }, [flushPending, pauseUpdates]); + + useEffect(() => { + latestRef.current = messageBus.getLastMessage(topic); + pendingRef.current = 0; + lastDisplayedAtRef.current = performance.now(); + + const initial = latestRef.current; + if (initial) { + pendingRef.current = 1; + scheduleFrame(flushPending); } - }, [ - displayMessage, - endRow, - expandedPaths, - maxBinaryPreviewBytes, - maxExpandedDepth, - maxRows, - setStats, - shapeSignature, - startRow, - visibleCount, - visibleRows, - ]); - - useEffect( - () => () => { - if (patchRafRef.current != null) { - cancelAnimationFrame(patchRafRef.current); + const unsubscribe = messageBus.subscribeTopic(topic, () => { + latestRef.current = messageBus.getLastMessage(topic); + pendingRef.current = latestOnlyRef.current ? 1 : pendingRef.current + 1; + if (latestRef.current && !hasMessageRef.current) { + hasMessageRef.current = true; + setHasMessage(true); } - }, - [], - ); + scheduleFrame(flushPending); + }); + + return unsubscribe; + }, [flushPending, topic]); useEffect(() => { if (!viewportRef.current) return; + + let cancelScheduledResize: (() => void) | null = null; const observer = new ResizeObserver((entries) => { const height = entries[0]?.contentRect.height; - if (height && height > 0) { - setViewportHeight(height); - } + if (!height || height <= 0) return; + viewportHeightRef.current = height; + cancelScheduledResize?.(); + cancelScheduledResize = scheduleFrame(applyScrollWindow); }); + observer.observe(viewportRef.current); - return () => observer.disconnect(); - }, []); + return () => { + cancelScheduledResize?.(); + observer.disconnect(); + }; + }, [applyScrollWindow]); useEffect(() => { + latestRef.current = messageBus.getLastMessage(topic); + hasMessageRef.current = !!latestRef.current; + setHasMessage(!!latestRef.current); setShapeSignature(''); setShapeRows([]); + shapeRowsRef.current = []; + shapeSignatureRef.current = ''; latestValueVisualRef.current.clear(); valueNodeRefs.current.clear(); + pendingPatchRef.current.clear(); + scrollTopRef.current = 0; + scrollWindowRef.current = { startRow: 0, endRow: 0, totalRows: 0 }; + setScrollWindow({ startRow: 0, endRow: 0, totalRows: 0 }); + expandedPathsRef.current = new Set(['message']); setExpandedPaths(new Set(['message'])); didInitializeExpansionRef.current = false; - }, [topic]); + pendingRef.current = latestRef.current ? 1 : 0; + lastDisplayedAtRef.current = performance.now(); + if (latestRef.current) { + scheduleFrame(flushPending); + } + }, [flushPending, topic]); + + useEffect(() => { + applyScrollWindow(); + }, [applyScrollWindow, expandedPaths, shapeRows]); + + useEffect(() => { + if (!latestRef.current) return; + latestValueVisualRef.current.clear(); + pendingRef.current = Math.max(pendingRef.current, 1); + scheduleFrame(flushPending); + }, [flushPending, isImageTopic, maxBinaryPreviewBytes, maxExpandedDepth, maxRows]); + + const visibleRows = useMemo(() => { + if (shapeRows.length === 0) return []; + return getVisibleRows(shapeRows, expandedPaths); + }, [expandedPaths, shapeRows]); + + const windowRows = visibleRows.slice(scrollWindow.startRow, scrollWindow.endRow); + + const registerValueNode = useCallback((path: string, node: HTMLSpanElement | null) => { + if (node) { + valueNodeRefs.current.set(path, node); + let initial = latestValueVisualRef.current.get(path); + if (initial == null && latestRef.current) { + const value = readValueAtPath(latestRef.current.message, path); + const { maxBinaryPreviewBytes: previewBytes, isImageTopic: hideHex } = configRef.current; + initial = describeValue(value, previewBytes, { hideBinaryHex: hideHex }); + latestValueVisualRef.current.set(path, initial); + } + if (initial != null) { + applyValueVisual(node, initial); + } + } else { + valueNodeRefs.current.delete(path); + } + }, []); const toggleExpand = useCallback((path: string) => { setExpandedPaths((prev) => { const next = new Set(prev); if (next.has(path)) next.delete(path); else next.add(path); + expandedPathsRef.current = next; return next; }); }, []); const copyField = useCallback( async (path: string) => { - const value = readValueAtPath(displayMessage?.message, path); + const value = readValueAtPath(latestRef.current?.message, path); const serialized = serializeForCopy(value, binaryCopyFormat); const text = typeof serialized === 'string' || typeof serialized === 'number' || typeof serialized === 'boolean' @@ -479,7 +668,15 @@ export const RawMessagesPanel: React.FC = ({ toast.error(formatMessage({ id: 'panels.rawMessages.copy.error' })); } }, - [binaryCopyFormat, displayMessage?.message, formatMessage], + [binaryCopyFormat, formatMessage], + ); + + const handleScroll = useCallback( + (event: React.UIEvent) => { + scrollTopRef.current = event.currentTarget.scrollTop; + scheduleFrame(applyScrollWindow); + }, + [applyScrollWindow], ); return ( @@ -487,6 +684,7 @@ export const RawMessagesPanel: React.FC = ({
setConfig((prev) => ({ ...prev, topic: nextTopic }))} placeholder={formatMessage({ id: 'panels.framework.topicPicker.placeholder' })} className="min-w-0 w-full" @@ -496,53 +694,20 @@ export const RawMessagesPanel: React.FC = ({
setScrollTop(event.currentTarget.scrollTop)} + onScroll={handleScroll} > - {displayMessage && totalRows > 0 ? ( -
-
+ {hasMessage && scrollWindow.totalRows > 0 ? ( +
+
{windowRows.map((row) => ( -
- - {row.key}: - { - if (node) { - valueNodeRefs.current.set(row.path, node); - const initial = latestValueVisualRef.current.get(row.path); - if (initial != null) { - node.textContent = initial.text; - node.dataset.kind = initial.kind; - node.style.color = valueColor(initial.kind); - } - } else { - valueNodeRefs.current.delete(row.path); - } - }} - className="truncate" - /> - -
+ row={row} + expanded={expandedPaths.has(row.path)} + onToggle={toggleExpand} + onCopy={copyField} + registerValueNode={registerValueNode} + /> ))}
diff --git a/src/features/panels/RawMessages/describeValue.test.ts b/src/features/panels/RawMessages/describeValue.test.ts new file mode 100644 index 0000000..0f6ede3 --- /dev/null +++ b/src/features/panels/RawMessages/describeValue.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest'; +import { describeValue } from './Component'; + +describe('describeValue', () => { + it('shows hex preview for small binary buffers', () => { + const value = new Uint8Array([0xde, 0xad, 0xbe, 0xef]); + const visual = describeValue(value, 256); + expect(visual.text).toBe('Uint8Array(4) 0xdeadbeef'); + expect(visual.kind).toBe('binary'); + }); + + it('hides hex preview for large binary buffers', () => { + const value = new Uint8Array(2048); + const visual = describeValue(value, 256); + expect(visual.text).toBe('Uint8Array(2048) [preview hidden]'); + }); + + it('hides hex preview for image topics regardless of size', () => { + const value = new Uint8Array([1, 2, 3]); + const visual = describeValue(value, 256, { hideBinaryHex: true }); + expect(visual.text).toBe('Uint8Array(3) [preview hidden]'); + }); +}); From dd0e707b0743f2d5b381f1ffdda49ace15d3c58a Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:09:43 +0800 Subject: [PATCH 11/15] feat(raw-messages): show log_time and publish_time beside message payload Surface MCAP receive/publish timestamps as top-level rows parallel to message so topics without headers still expose both clock sources. --- src/features/panels/RawMessages/Component.tsx | 38 ++++++++++++++----- .../panels/RawMessages/describeValue.test.ts | 6 +++ .../panels/RawMessages/shapeTree.test.ts | 19 ++++++++++ src/features/panels/RawMessages/shapeTree.ts | 23 +++++++++++ 4 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 src/features/panels/RawMessages/shapeTree.test.ts diff --git a/src/features/panels/RawMessages/Component.tsx b/src/features/panels/RawMessages/Component.tsx index f63e282..2cfe689 100644 --- a/src/features/panels/RawMessages/Component.tsx +++ b/src/features/panels/RawMessages/Component.tsx @@ -4,15 +4,16 @@ import { useIntl } from 'react-intl'; import { toast } from 'sonner'; import { messageBus } from '@/core/pipeline/messageBus'; import { useMessagePipeline } from '@/core/pipeline/useMessagePipeline'; -import type { MessageEvent } from '@/core/types/ros'; +import type { MessageEvent, Time } from '@/core/types/ros'; import type { MessagePipelineState } from '@/core/pipeline/store'; import type { Player } from '@/core/types/player'; import { pickDefaultRawMessagesTopic } from '@/features/layout/autoLayout/pickDefaultRawMessagesTopic'; import { isRosImageSchema } from '@/shared/ros/rosMessageTypes'; +import { formatLocalTimestamp } from '@/shared/utils/time'; import { scheduleFrame } from '@/shared/utils/rafScheduler'; import { TopicQuickPicker } from '../framework/TopicQuickPicker'; import type { RawMessagesConfig } from './defaults'; -import { buildRowsForShape, type FlatRow } from './shapeTree'; +import { buildRowsForMessageEvent, type FlatRow } from './shapeTree'; interface RawMessagesPanelProps { player: Player; @@ -88,10 +89,24 @@ function pathToParts(path: string): string[] { return path.split('.').filter((part) => part.length > 0); } -function readValueAtPath(root: unknown, path: string): unknown { - if (!path || path === 'message') return root; +function isRosTime(value: unknown): value is Time { + if (!isPlainObject(value)) return false; + return typeof value.sec === 'number' && typeof value.nsec === 'number'; +} + +function formatRosTime(time: Time): string { + const nsecPadded = time.nsec.toString().padStart(9, '0'); + return `${time.sec}.${nsecPadded} (${formatLocalTimestamp(time)})`; +} + +function readValueAtPath(event: MessageEvent | null | undefined, path: string): unknown { + if (!event) return undefined; + if (path === 'log_time') return event.receiveTime; + if (path === 'publish_time') return event.publishTime; + if (!path || path === 'message') return event.message; + if (!path.startsWith('message.')) return undefined; const parts = pathToParts(path.replace(/^message\./, '')); - let current: unknown = root; + let current: unknown = event.message; for (const part of parts) { if (Array.isArray(current)) { const idx = Number(part); @@ -172,6 +187,9 @@ export function describeValue( maxBinaryPreviewBytes: number, options?: DescribeValueOptions, ): ValueVisual { + if (isRosTime(value)) { + return { text: formatRosTime(value), kind: 'number' }; + } if (value instanceof Uint8Array) { if (options?.hideBinaryHex || value.byteLength > LARGE_BINARY_THRESHOLD) { return { @@ -442,7 +460,7 @@ export const RawMessagesPanel: React.FC = ({ for (let i = 0; i < maxPatchRows; i++) { const row = patchRows[i]; if (!row) continue; - const value = readValueAtPath(message.message, row.path); + const value = readValueAtPath(message, row.path); const visual = describeValue(value, previewBytes, { hideBinaryHex: hideHex }); latestValueVisualRef.current.set(row.path, visual); const previousText = valueNodeRefs.current.get(row.path)?.textContent ?? null; @@ -503,7 +521,7 @@ export const RawMessagesPanel: React.FC = ({ } const { maxExpandedDepth: depth, maxRows: rowLimit } = configRef.current; - const nextShape = buildRowsForShape(message.message, depth, rowLimit); + const nextShape = buildRowsForMessageEvent(message, depth, rowLimit); if (nextShape.signature !== shapeSignatureRef.current) { shapeSignatureRef.current = nextShape.signature; shapeRowsRef.current = nextShape.rows; @@ -513,7 +531,7 @@ export const RawMessagesPanel: React.FC = ({ if (!didInitializeExpansionRef.current) { const nextExpanded = new Set(['message']); for (const row of nextShape.rows) { - if (row.depth === 1 && row.expandable) { + if (row.depth === 1 && row.expandable && row.path.startsWith('message.')) { nextExpanded.add(row.path); } } @@ -630,7 +648,7 @@ export const RawMessagesPanel: React.FC = ({ valueNodeRefs.current.set(path, node); let initial = latestValueVisualRef.current.get(path); if (initial == null && latestRef.current) { - const value = readValueAtPath(latestRef.current.message, path); + const value = readValueAtPath(latestRef.current, path); const { maxBinaryPreviewBytes: previewBytes, isImageTopic: hideHex } = configRef.current; initial = describeValue(value, previewBytes, { hideBinaryHex: hideHex }); latestValueVisualRef.current.set(path, initial); @@ -655,7 +673,7 @@ export const RawMessagesPanel: React.FC = ({ const copyField = useCallback( async (path: string) => { - const value = readValueAtPath(latestRef.current?.message, path); + const value = readValueAtPath(latestRef.current, path); const serialized = serializeForCopy(value, binaryCopyFormat); const text = typeof serialized === 'string' || typeof serialized === 'number' || typeof serialized === 'boolean' diff --git a/src/features/panels/RawMessages/describeValue.test.ts b/src/features/panels/RawMessages/describeValue.test.ts index 0f6ede3..e128fee 100644 --- a/src/features/panels/RawMessages/describeValue.test.ts +++ b/src/features/panels/RawMessages/describeValue.test.ts @@ -20,4 +20,10 @@ describe('describeValue', () => { const visual = describeValue(value, 256, { hideBinaryHex: true }); expect(visual.text).toBe('Uint8Array(3) [preview hidden]'); }); + + it('formats ROS time values', () => { + const visual = describeValue({ sec: 100, nsec: 500_000_000 }, 256); + expect(visual.text).toContain('100.500000000'); + expect(visual.kind).toBe('number'); + }); }); diff --git a/src/features/panels/RawMessages/shapeTree.test.ts b/src/features/panels/RawMessages/shapeTree.test.ts new file mode 100644 index 0000000..cb3c529 --- /dev/null +++ b/src/features/panels/RawMessages/shapeTree.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest'; +import type { MessageEvent } from '@/core/types/ros'; +import { buildRowsForMessageEvent } from './shapeTree'; + +describe('buildRowsForMessageEvent', () => { + it('includes log_time and publish_time rows before message', () => { + const event: MessageEvent = { + topic: '/camera/image', + receiveTime: { sec: 10, nsec: 1 }, + publishTime: { sec: 9, nsec: 2 }, + message: { width: 640, height: 480 }, + schemaName: 'sensor_msgs/msg/Image', + }; + + const shape = buildRowsForMessageEvent(event, 4, 2000); + expect(shape.rows.slice(0, 3).map((row) => row.key)).toEqual(['log_time', 'publish_time', 'message']); + expect(shape.signature.startsWith('log_time:time|publish_time:time|')).toBe(true); + }); +}); diff --git a/src/features/panels/RawMessages/shapeTree.ts b/src/features/panels/RawMessages/shapeTree.ts index 0d6d749..65c32c5 100644 --- a/src/features/panels/RawMessages/shapeTree.ts +++ b/src/features/panels/RawMessages/shapeTree.ts @@ -1,3 +1,5 @@ +import type { MessageEvent } from '@/core/types/ros'; + export interface FlatRow { id: string; path: string; @@ -12,6 +14,11 @@ export interface ShapeBuildResult { rows: FlatRow[]; } +const MESSAGE_EVENT_META_ROWS: FlatRow[] = [ + { id: 'log_time', path: 'log_time', key: 'log_time', depth: 0, expandable: false, parentIsArray: false }, + { id: 'publish_time', path: 'publish_time', key: 'publish_time', depth: 0, expandable: false, parentIsArray: false }, +]; + function isPlainObject(value: unknown): value is Record { if (!value || typeof value !== 'object') return false; const proto = Object.getPrototypeOf(value) as object | null; @@ -81,3 +88,19 @@ export function buildRowsForShape( walk(root, 'message', 'message', 0, false); return { rows, signature: signatureParts.join('|') }; } + +export function buildRowsForMessageEvent( + event: MessageEvent, + maxExpandedDepth: number, + maxRows: number, +): ShapeBuildResult { + const messageShape = buildRowsForShape( + event.message, + maxExpandedDepth, + Math.max(0, maxRows - MESSAGE_EVENT_META_ROWS.length), + ); + return { + rows: [...MESSAGE_EVENT_META_ROWS, ...messageShape.rows], + signature: `log_time:time|publish_time:time|${messageShape.signature}`, + }; +} From 0bfb73b5e107857232b1227fed60f3d6901da2eb Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:11:59 +0800 Subject: [PATCH 12/15] fix(raw-messages): show first 8 bytes of binary previews instead of hiding Use a compact hex head for image topics and large Uint8Array fields so payload format can still be identified without full-buffer formatting. --- src/features/panels/RawMessages/Component.tsx | 13 ++++++------- .../panels/RawMessages/describeValue.test.ts | 12 +++++++----- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/features/panels/RawMessages/Component.tsx b/src/features/panels/RawMessages/Component.tsx index 2cfe689..8ae99ae 100644 --- a/src/features/panels/RawMessages/Component.tsx +++ b/src/features/panels/RawMessages/Component.tsx @@ -61,6 +61,7 @@ const MAX_VISIBLE_PATCH_ROWS = 1200; const MAX_OBJECT_PREVIEW_FIELDS = 3; const MAX_PREVIEW_STRING_LENGTH = 80; const LARGE_BINARY_THRESHOLD = 1024; +const COMPACT_BINARY_PREVIEW_BYTES = 8; function toHex(data: Uint8Array): string { let out = ''; @@ -191,13 +192,11 @@ export function describeValue( return { text: formatRosTime(value), kind: 'number' }; } if (value instanceof Uint8Array) { - if (options?.hideBinaryHex || value.byteLength > LARGE_BINARY_THRESHOLD) { - return { - text: `Uint8Array(${value.byteLength}) [preview hidden]`, - kind: 'binary', - }; - } - const head = value.subarray(0, Math.min(value.byteLength, maxBinaryPreviewBytes)); + const previewLength = + options?.hideBinaryHex || value.byteLength > LARGE_BINARY_THRESHOLD + ? COMPACT_BINARY_PREVIEW_BYTES + : Math.min(value.byteLength, maxBinaryPreviewBytes); + const head = value.subarray(0, previewLength); return { text: `Uint8Array(${value.byteLength}) 0x${toHex(head)}${value.byteLength > head.byteLength ? '...' : ''}`, kind: 'binary', diff --git a/src/features/panels/RawMessages/describeValue.test.ts b/src/features/panels/RawMessages/describeValue.test.ts index e128fee..a63b948 100644 --- a/src/features/panels/RawMessages/describeValue.test.ts +++ b/src/features/panels/RawMessages/describeValue.test.ts @@ -9,16 +9,18 @@ describe('describeValue', () => { expect(visual.kind).toBe('binary'); }); - it('hides hex preview for large binary buffers', () => { + it('shows compact hex preview for large binary buffers', () => { const value = new Uint8Array(2048); + value[0] = 0xff; + value[7] = 0x00; const visual = describeValue(value, 256); - expect(visual.text).toBe('Uint8Array(2048) [preview hidden]'); + expect(visual.text).toBe('Uint8Array(2048) 0xff00000000000000...'); }); - it('hides hex preview for image topics regardless of size', () => { - const value = new Uint8Array([1, 2, 3]); + it('shows compact hex preview for image topics regardless of size', () => { + const value = new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]); const visual = describeValue(value, 256, { hideBinaryHex: true }); - expect(visual.text).toBe('Uint8Array(3) [preview hidden]'); + expect(visual.text).toBe('Uint8Array(9) 0x0102030405060708...'); }); it('formats ROS time values', () => { From a575b6b9cf4bcb049c5f7f9713764dcad4f27fce Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:13:44 +0800 Subject: [PATCH 13/15] fix(raw-messages): increase compact binary preview bytes from 8 to 32 Update the compact binary preview size to allow for a more comprehensive representation of binary data, enhancing the identification of payload formats for image topics and large Uint8Array fields. --- src/features/panels/RawMessages/Component.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/features/panels/RawMessages/Component.tsx b/src/features/panels/RawMessages/Component.tsx index 8ae99ae..096894d 100644 --- a/src/features/panels/RawMessages/Component.tsx +++ b/src/features/panels/RawMessages/Component.tsx @@ -61,7 +61,7 @@ const MAX_VISIBLE_PATCH_ROWS = 1200; const MAX_OBJECT_PREVIEW_FIELDS = 3; const MAX_PREVIEW_STRING_LENGTH = 80; const LARGE_BINARY_THRESHOLD = 1024; -const COMPACT_BINARY_PREVIEW_BYTES = 8; +const COMPACT_BINARY_PREVIEW_BYTES = 32; function toHex(data: Uint8Array): string { let out = ''; From 03747add990e214ca603b2fb23417c072b248c64 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:19:47 +0800 Subject: [PATCH 14/15] chore: release v1.3.0 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index a42719f..20c4ddc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@ioai/rosview", - "version": "1.2.6", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@ioai/rosview", - "version": "1.2.6", + "version": "1.3.0", "license": "MIT", "devDependencies": { "@eslint/js": "^9.39.4", diff --git a/package.json b/package.json index 863757a..7b731b4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@ioai/rosview", - "version": "1.2.6", + "version": "1.3.0", "description": "High-performance robotics data visualization for MCAP, ROS bag, ROS2 db3, HDF5 and BVH — embeddable React component and standalone SPA", "keywords": [ "ros", From bdcbbe39e7256050b2c802876841b7e849b12de4 Mon Sep 17 00:00:00 2001 From: joaner Date: Tue, 26 May 2026 17:33:37 +0800 Subject: [PATCH 15/15] fix: restore build and e2e after mcapProcessor and RawMessages updates Fix TypeScript and lint blockers that prevented Playwright webServer from starting, and align binary preview tests with the 32-byte compact limit. --- src/features/panels/RawMessages/Component.tsx | 9 +++- .../panels/RawMessages/describeValue.test.ts | 13 +++-- .../panels/UrdfDebug/embedded/fkEngine.d.ts | 7 +++ .../panels/UrdfDebug/mcapProcessor.ts | 47 +++++++++++-------- 4 files changed, 53 insertions(+), 23 deletions(-) create mode 100644 src/features/panels/UrdfDebug/embedded/fkEngine.d.ts diff --git a/src/features/panels/RawMessages/Component.tsx b/src/features/panels/RawMessages/Component.tsx index 096894d..02a6da7 100644 --- a/src/features/panels/RawMessages/Component.tsx +++ b/src/features/panels/RawMessages/Component.tsx @@ -375,6 +375,7 @@ export const RawMessagesPanel: React.FC = ({ const valueNodeRefs = useRef>(new Map()); const latestValueVisualRef = useRef>(new Map()); const pendingPatchRef = useRef>(new Map()); + const flushPendingRef = useRef<() => void>(() => {}); const didInitializeExpansionRef = useRef(false); const [hasMessage, setHasMessage] = useState(() => !!messageBus.getLastMessage(topic)); @@ -507,7 +508,9 @@ export const RawMessagesPanel: React.FC = ({ } if (now - lastDisplayedAtRef.current < minInterval) { - scheduleFrame(flushPending); + scheduleFrame(() => { + flushPendingRef.current(); + }); return; } @@ -551,6 +554,10 @@ export const RawMessagesPanel: React.FC = ({ patchVisibleValues(message); }, [applyScrollWindow, patchVisibleValues]); + useEffect(() => { + flushPendingRef.current = flushPending; + }); + useEffect(() => { if (!pauseUpdates && pendingRef.current > 0) { scheduleFrame(flushPending); diff --git a/src/features/panels/RawMessages/describeValue.test.ts b/src/features/panels/RawMessages/describeValue.test.ts index a63b948..a89821e 100644 --- a/src/features/panels/RawMessages/describeValue.test.ts +++ b/src/features/panels/RawMessages/describeValue.test.ts @@ -14,13 +14,20 @@ describe('describeValue', () => { value[0] = 0xff; value[7] = 0x00; const visual = describeValue(value, 256); - expect(visual.text).toBe('Uint8Array(2048) 0xff00000000000000...'); + expect(visual.text).toBe( + 'Uint8Array(2048) 0xff00000000000000000000000000000000000000000000000000000000000000...', + ); }); it('shows compact hex preview for image topics regardless of size', () => { - const value = new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09]); + const value = new Uint8Array(40); + for (let i = 0; i < value.length; i++) { + value[i] = i + 1; + } const visual = describeValue(value, 256, { hideBinaryHex: true }); - expect(visual.text).toBe('Uint8Array(9) 0x0102030405060708...'); + expect(visual.text).toBe( + 'Uint8Array(40) 0x0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20...', + ); }); it('formats ROS time values', () => { diff --git a/src/features/panels/UrdfDebug/embedded/fkEngine.d.ts b/src/features/panels/UrdfDebug/embedded/fkEngine.d.ts new file mode 100644 index 0000000..d45e91d --- /dev/null +++ b/src/features/panels/UrdfDebug/embedded/fkEngine.d.ts @@ -0,0 +1,7 @@ +export class JointState2TF { + static fromXml(opts: { xml: string }): JointState2TF; + computeFromJointState( + jointState: { header: unknown; name: string[]; position: number[] }, + options?: { publishTimeNs?: bigint }, + ): { transforms: unknown[] }; +} diff --git a/src/features/panels/UrdfDebug/mcapProcessor.ts b/src/features/panels/UrdfDebug/mcapProcessor.ts index c4caf50..5d86937 100644 --- a/src/features/panels/UrdfDebug/mcapProcessor.ts +++ b/src/features/panels/UrdfDebug/mcapProcessor.ts @@ -1,13 +1,11 @@ -import { McapIndexedReader, McapWriter, type Channel, type Schema } from '@mcap/core'; +import { McapIndexedReader, McapWriter, McapTypes, type Channel, type Schema } from '@mcap/core'; import { MessageReader, MessageWriter } from '@foxglove/rosmsg2-serialization'; -import rosmsg from '@foxglove/rosmsg'; +import { parse as parseMessageDefinition } from '@foxglove/rosmsg'; import { JointState2TF } from './embedded/fkEngine.js'; import { applyJointMapping } from './jointStateMapping'; import type { UrdfDebugRecipe } from './recipe'; import { applyUrdfVisualCorrection } from './urdfVisualCorrection'; -const { parseMessageDefinition } = rosmsg; - const ROS2_DEFINITIONS = [ { name: 'builtin_interfaces/msg/Time', definitions: [{ name: 'sec', type: 'int32' }, { name: 'nanosec', type: 'uint32' }] }, { name: 'std_msgs/msg/Header', definitions: [{ name: 'stamp', type: 'builtin_interfaces/msg/Time', isComplex: true }, { name: 'frame_id', type: 'string' }] }, @@ -20,35 +18,46 @@ const ROS2_DEFINITIONS = [ { name: 'sensor_msgs/msg/JointState', definitions: [{ name: 'header', type: 'std_msgs/msg/Header', isComplex: true }, { name: 'name', type: 'string', isArray: true }, { name: 'position', type: 'float64', isArray: true }, { name: 'velocity', type: 'float64', isArray: true }, { name: 'effort', type: 'float64', isArray: true }] }, ]; -class BufferReadable { - constructor(private readonly buffer: Uint8Array) {} +class BufferReadable implements McapTypes.IReadable { + private readonly buffer: Uint8Array; + + constructor(buffer: Uint8Array) { + this.buffer = buffer; + } - size() { - return BigInt(this.buffer.byteLength); + size(): Promise { + return Promise.resolve(BigInt(this.buffer.byteLength)); } - async read(offset: bigint, size: bigint) { + read(offset: bigint, size: bigint): Promise { const start = Number(offset); - return this.buffer.subarray(start, start + Number(size)); + return Promise.resolve(this.buffer.subarray(start, start + Number(size))); } } class BufferWritable { - #chunks: Buffer[] = []; + #chunks: Uint8Array[] = []; #pos = 0n; position() { return this.#pos; } - async write(buffer: Uint8Array) { - const chunk = Buffer.from(buffer); - this.#chunks.push(chunk); - this.#pos += BigInt(chunk.byteLength); + write(buffer: Uint8Array) { + this.#chunks.push(buffer); + this.#pos += BigInt(buffer.byteLength); + return Promise.resolve(); } - toBuffer() { - return Buffer.concat(this.#chunks); + toUint8Array(): Uint8Array { + const total = Number(this.#pos); + const out = new Uint8Array(total); + let offset = 0; + for (const chunk of this.#chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; } } @@ -57,7 +66,7 @@ function prepareUrdfFromRecipe(urdfXml: string, recipe: UrdfDebugRecipe): string return applyUrdfVisualCorrection(urdfXml, { rotateMeshVisuals: !!urdf.rotateMeshVisuals, visualRpyOffset: Array.isArray(urdf.visualRpyOffset) - ? (urdf.visualRpyOffset as [number, number, number]) + ? urdf.visualRpyOffset : [0, 0, 0], }); } @@ -271,5 +280,5 @@ export async function processMcapBuffer({ } await writer.end(); - return { output: writable.toBuffer(), processedJointStates }; + return { output: writable.toUint8Array(), processedJointStates }; }