Add a video plot that supports video and animated image files

client/component/package.json

@@ -55,7 +55,7 @@
     "three": ">=0.182.0"
   },
   "devDependencies": {
-    "@h5web/lib": "^16.0.0",
+    "@h5web/lib": "^17.0.0",
     "@react-three/drei": "^9.122.0",
     "@react-three/fiber": "^8.18.0",
     "@types/cwise": "^1.0.6",

client/component/src/ConnectedPlot.tsx

@@ -147,12 +147,20 @@ interface ClientScatterParametersMessage {
   pointSize: number;
 }
 
+/**
+ * A client status message
+ */
+interface ClientConfigMessage {
+  config: Map<string, string>;
+}
+
 type ClientMessage =
   | ClientStatusMessage
   | ClientSelectionMessage
   | ClientLineParametersMessage
   | ClientScatterParametersMessage
   | ClearSelectionsMessage
+  | ClientConfigMessage
   | BatonRequestMessage
   | BatonDonateMessage;
 
@@ -224,6 +232,8 @@ interface ConnectedPlotProps {
   hostname?: string;
   /** The port */
   port?: string;
+  /** A configuration map */
+  config?: object;
   /** The universally unique identifier */
   uuid: string;
 }
@@ -238,6 +248,7 @@ function ConnectedPlot({
   plotId = 'plot_0',
   hostname = '127.0.0.1',
   port = '80',
+  config,
   uuid,
 }: ConnectedPlotProps) {
   const [plotProps, setPlotProps] = useState<AnyPlotProps | null>();
@@ -246,6 +257,7 @@ function ConnectedPlot({
     useState<PlotConfig>(defaultPlotConfig);
   const [scatterData, setScatterData] = useState<ScatterData>();
 
+  const [configMap, setConfigMap] = useState(config);
   const mountState = useRef('');
   const plotServerURL = `ws://${hostname}:${port}/plot/${uuid}/${plotId}`;
   const { sendMessage, lastMessage, readyState, getWebSocket } = useWebSocket(
@@ -271,6 +283,10 @@ function ConnectedPlot({
     };
   }, []);
 
+  if (config != configMap) {
+    setConfigMap(config);
+  }
+
   const sendClientMessage = useCallback(
     (data: ClientMessage) => {
       console.log('%s: sending', plotId, data);
@@ -338,6 +354,12 @@ function ConnectedPlot({
     }
   }, [getWebSocket, plotId, readyState, sendStatusMessage]);
 
+  useEffect(() => {
+    if (readyState === ReadyState.OPEN && configMap) {
+      sendClientMessage({config: configMap} as ClientConfigMessage);
+    }
+  }, [readyState, configMap]);
+
   const clearLineData = () => {
     setLineData([]);
     setLinePlotConfig(defaultPlotConfig);

client/component/src/LabelledInput.tsx

@@ -1,4 +1,4 @@
-import '@h5web/lib/dist/styles.css';
+import '@h5web/lib/styles.css';
 
 import styles from './LabelledInput.module.css';
 

client/component/src/VideoPlot.tsx

@@ -0,0 +1,216 @@
+import {
+  DefaultInteractions,
+  type ModifierKey,
+  ResetZoomButton,
+  TooltipMesh,
+  VisMesh,
+  VisCanvas,
+  getVisDomain,
+  FloatingControl,
+} from '@h5web/lib';
+import { useEffect, useMemo, useState, type ReactElement } from 'react';
+import { useFrame, useThree } from '@react-three/fiber';
+import { useTexture, useVideoTexture } from '@react-three/drei';
+
+import SelectionComponent from './SelectionComponent';
+import type { PlotBaseProps } from './models';
+import { createInteractionsConfig, InteractionModeType } from './utils';
+import {
+  PlotCustomizationContextProvider,
+  usePlotCustomizationContext,
+} from './PlotCustomizationContext';
+import { AnyToolbar } from './PlotToolbar';
+
+/**
+ * Props for the `VideoPlot` component.
+ */
+interface VideoPlotProps extends PlotBaseProps {
+  /** video source url */
+  sourceURL: string;
+  /** if true, source is an animated image (MJPEG, animated GIF, etc) */
+  isImage?: boolean;
+}
+
+function ImageMeshMaterial(props: {sourceUrl: string, play: boolean, setSize: (size: [number, number]) => void}) {
+  const { setFrameloop } = useThree();
+  const texture = useTexture(props.sourceUrl);
+  const data = texture.source.data as HTMLImageElement;
+  const height = data.height;
+  const width = data.width;
+  useEffect(() => {
+    props.setSize([height, width]);
+  }, [height, width]);
+
+  useEffect(() => {
+    return () => {
+      setFrameloop('demand');
+    };
+  }, []);
+
+  useEffect(() => {
+    // dial down frame loop when no more frames
+    if (data.complete) {
+      setFrameloop('demand');
+    }
+  }, [data.complete]);
+
+  useEffect(() => {
+    setFrameloop(props.play ? 'always' : 'demand');
+  }, [props.play]);
+  useFrame(() => texture.needsUpdate = props.play);
+  return <meshBasicMaterial map={texture} />;
+}
+
+function VideoMeshMaterial(props: {sourceUrl: string, play: boolean, setSize: (size: [number, number]) => void}) {
+  const { setFrameloop } = useThree();
+  const texture = useVideoTexture(props.sourceUrl);
+  const data = texture.source.data;
+
+  useEffect(() => {
+    data.pause(); // start paused
+    return () => {
+      setFrameloop('demand');
+    };
+  }, []);
+
+  const height = data.videoHeight;
+  const width = data.videoWidth;
+  useEffect(() => {
+    props.setSize([height, width]);
+  }, [height, width]);
+
+  useEffect(() => {
+    if (props.play) {
+      data.play();
+    } else {
+      data.pause();
+    }
+    setFrameloop(props.play ? 'always' : 'demand');
+  }, [props.play]);
+  return <meshBasicMaterial map={texture} />;
+}
+
+
+function PlayButton(props: {clickPlay: () => void, text: string}) {
+  return (
+    <FloatingControl>
+      <button
+        type="button"
+        onClick={() => props.clickPlay()}
+      >
+        <span>{props.text}</span>
+      </button>
+    </FloatingControl>
+  );
+}
+
+export function VideoVisCanvas(props: VideoPlotProps) {
+  const {
+    title,
+    showGrid,
+    xCustomDomain,
+    setXCustomDomain,
+    xDomain,
+    xLabel,
+    yCustomDomain,
+    setYCustomDomain,
+    yDomain,
+    yLabel,
+    mode,
+    batonProps,
+    canSelect,
+    selectionMax,
+    selectionType,
+    updateSelection,
+    selections,
+  } = usePlotCustomizationContext();
+  const { sourceURL, isImage=false } = {...props};
+  const [isPlaying, setPlaying] = useState(false);
+
+  const interactionsConfig = createInteractionsConfig(mode);
+  const tooltipText = (x: number, y: number): ReactElement<string> => {
+    return (
+      <p>
+        {x.toPrecision(8)}, {y.toPrecision(8)}
+      </p>
+    );
+  };
+
+  const [size, setSize] = useState([1,1]);
+
+  useEffect(() => {
+    setXCustomDomain([0, size[1]]);
+    setYCustomDomain([0, size[0]]);
+  }, [size, setSize]);
+  const xVisDomain = useMemo(() => {
+    return getVisDomain(xCustomDomain, xDomain);
+  }, [xCustomDomain, xDomain]);
+
+  const yVisDomain = useMemo(() => {
+    return getVisDomain(yCustomDomain, yDomain);
+  }, [yCustomDomain, yDomain]);
+
+  return (
+    <VisCanvas
+      title={title ?? ''}
+      aspect={"equal"}
+      abscissaConfig={{
+        visDomain: xVisDomain,
+        showGrid,
+        isIndexAxis: true,
+        label: xLabel,
+        nice: true,
+      }}
+      ordinateConfig={{
+        visDomain: yVisDomain,
+        showGrid,
+        isIndexAxis: true,
+        label: yLabel,
+        nice: true,
+      }}
+    >
+      <DefaultInteractions {...interactionsConfig} />
+      <ResetZoomButton />
+      <PlayButton clickPlay={() => setPlaying((p) => !p)} text={isPlaying ? "Pause" : "Play"}/>
+      <TooltipMesh renderTooltip={tooltipText} />
+      <VisMesh>
+        {isImage ? <ImageMeshMaterial sourceUrl={sourceURL} play={isPlaying} setSize={setSize} /> : <VideoMeshMaterial sourceUrl={sourceURL} play={isPlaying} setSize={setSize} />}
+      </VisMesh>
+      {canSelect && (
+        <SelectionComponent
+          modifierKey={[] as ModifierKey[]}
+          batonProps={batonProps}
+          disabled={mode !== InteractionModeType.selectRegion}
+          selectionMax={selectionMax}
+          selectionType={selectionType}
+          updateSelection={updateSelection}
+          selections={selections}
+        />
+      )}
+    </VisCanvas>
+  );
+}
+
+/**
+ * Render a video plot.
+ * @param {VideoPlotProps} props - The component props.
+ * @returns {JSX.Element} The rendered component.
+ */
+function VideoPlot(props: VideoPlotProps) {
+  return (
+    <div
+      style={{
+        display: 'grid',
+        position: 'relative',
+      }}
+    >
+      <PlotCustomizationContextProvider {...props}>
+        <AnyToolbar>{props.customToolbarChildren}</AnyToolbar>
+        <VideoVisCanvas {...props} />
+      </PlotCustomizationContextProvider>
+    </div>
+  );
+}
+
+export default VideoPlot;
+export type { VideoPlotProps };

client/component/src/index.ts

@@ -21,6 +21,9 @@ export type { ImageData, ImagePlotProps } from './ImagePlot';
 export { default as LinePlot } from './LinePlot';
 export type { LineData, LineParams, LinePlotProps } from './LinePlot';
 
+export { default as VideoPlot } from './VideoPlot';
+export type { VideoPlotProps } from './VideoPlot';
+
 export {
   PlotCustomizationContextProvider,
   usePlotCustomizationContext,

client/example/src/App.tsx

@@ -131,6 +131,7 @@ class AppMain extends React.Component<AppMainProps, AppMainStates> {
       ...heatmapProps.plotConfig,
       title: 'Sample Heatmap Plot (no toolbar)',
     };
+    const config = {pvName: 'MY_DEVICE'};
 
     const host = import.meta.env.VITE_WS_HOST ?? window.location.hostname;
     const port = import.meta.env.VITE_WS_PORT ?? window.location.port;
@@ -157,6 +158,7 @@ class AppMain extends React.Component<AppMainProps, AppMainStates> {
               uuid={this.uuid}
               hostname={host}
               port={port}
+              config={config}
             />
           </div>
           <div style={{ display: 'grid', height: '49vh' }}>

client/example/vite.config.ts

@@ -7,6 +7,7 @@ export default defineConfig({
   base: './',
   define: {
     global: 'window', // this fixes global is not defined
+    "process.env.DRAGGABLE_DEBUG": "undefined", // fixes react-draggable issue in its log function
   },
   build: {
     outDir: '../../server/example-client/sdist',