Move to offscreen canvas in setDesktopwithCameraSource

karma.conf.cjs

@@ -7,8 +7,8 @@ module.exports = function(config) {
 
 	],
 
-	proxies: {
-	  '/volume-meter-processor.js': '/base/src/main/js/volume-meter-processor.js'
+	proxies: {'/volume-meter-processor.js': '/base/src/main/js/volume-meter-processor.js',
+		'/draw-desktop-with-camera-source-worker.js': '/base/src/main/js/draw-desktop-with-camera-source-worker.js'
 	},
 
     reporters: ['progress', 'coverage'],

rollup.config.module.cjs

@@ -17,6 +17,7 @@ const builds = {
 			 'src/main/js/utility.js',
 			 'src/main/js/media_manager.js',
 			 'src/main/js/stream_merger.js',
+			 'src/main/js/draw-desktop-with-camera-source-worker.js',
 			],
 	output: [{
 		dir: 'dist',

src/main/js/draw-desktop-with-camera-source-worker.js

@@ -0,0 +1,67 @@
+let canvas, ctx;
+let canvasWidth = 1920;
+let canvasHeight = 1080;
+let overlayWidth, overlayHeight;
+
+// Handle messages from the main thread
+self.onmessage = function (event) {
+    const { type, data } = event.data;
+
+    switch (type) {
+        case 'init':
+            initializeCanvas(data.canvas, data.width, data.height);
+            break;
+        case 'frame':
+            drawFrame(data.screenFrame, data.cameraFrame);
+            break;
+        case 'resize':
+            resizeCanvas(data.width, data.height);
+            break;
+        default:
+            console.warn('Unknown message type:', type);
+    }
+};
+
+function initializeCanvas(offscreenCanvas, width, height) {
+    canvas = offscreenCanvas;
+    ctx = canvas.getContext('2d');
+    canvasWidth = width;
+    canvasHeight = height;
+
+    canvas.width = canvasWidth;
+    canvas.height = canvasHeight;
+
+    overlayWidth = canvasWidth / 4;
+    console.log('Canvas initialized in worker.');
+}
+
+function resizeCanvas(width, height) {
+    canvasWidth = width;
+    canvasHeight = height;
+    canvas.width = canvasWidth;
+    canvas.height = canvasHeight;
+
+    overlayWidth = canvasWidth / 4;
+    console.log('Canvas resized in worker:', canvasWidth, canvasHeight);
+}
+
+function drawFrame(screenFrame, cameraFrame) {
+    if (!ctx) return;
+
+    // Clear the canvas
+    ctx.clearRect(0, 0, canvasWidth, canvasHeight);
+
+    // Draw the screen video frame
+    ctx.drawImage(screenFrame, 0, 0, canvasWidth, canvasHeight);
+
+    // Draw the camera overlay
+    const overlayHeight = overlayWidth * (cameraFrame.height / cameraFrame.width);
+    const positionX = canvasWidth - overlayWidth - 20;
+    const positionY = canvasHeight - overlayHeight - 20;
+
+    ctx.drawImage(cameraFrame, positionX, positionY, overlayWidth, overlayHeight);
+
+    // Release ImageBitmap resources
+    screenFrame.close();
+    cameraFrame.close();
+}

src/main/js/media_manager.js

@@ -354,63 +354,89 @@ export class MediaManager {
      */
     setDesktopwithCameraSource(stream, streamId, onEndedCallback) {
         this.desktopStream = stream;
-        return this.navigatorUserMedia({video: true, audio: false}, cameraStream => {
-            this.smallVideoTrack = cameraStream.getVideoTracks()[0];
 
-            //create a canvas element
-            var canvas = document.createElement("canvas");
-            var canvasContext = canvas.getContext("2d");
+        return this.navigatorUserMedia({ video: true, audio: false }, cameraStream => {
+            this.smallVideoTrack = cameraStream.getVideoTracks()[0];
 
-            //create video element for screen
-            //var screenVideo = document.getElementById('sourceVideo');
-            var screenVideo = document.createElement('video');
+            // Create OffscreenCanvas
+            const canvas = document.createElement("canvas");
+            const offscreenCanvas = canvas.transferControlToOffscreen();
 
+            // Create video elements for streams
+            const screenVideo = document.createElement("video");
             screenVideo.srcObject = stream;
-            screenVideo.play();
-            //create video element for camera
-            var cameraVideo = document.createElement('video');
 
+            const cameraVideo = document.createElement("video");
             cameraVideo.srcObject = cameraStream;
-            cameraVideo.play();
-            var canvasStream = canvas.captureStream(15);
 
-            if (onEndedCallback != null) {
-                stream.getVideoTracks()[0].onended = function (event) {
+            // Start playing videos only after metadata is loaded
+            const setupVideo = video =>
+                new Promise(resolve => {
+                    video.onloadedmetadata = () => {
+                        video.play();
+                        resolve(video);
+                    };
+                });
+
+            // Handle stream end callback
+            if (onEndedCallback) {
+                stream.getVideoTracks()[0].onended = event => {
                     onEndedCallback(event);
-                }
-            }
-            var promise;
-            if (this.localStream == null) {
-                promise = this.gotStream(canvasStream);
-            } else {
-                promise = this.updateVideoTrack(canvasStream, streamId, onended, null);
+                };
             }
 
-            promise.then(() => {
+            // Create a MediaStream from the canvas
+            const canvasStream = canvas.captureStream(15); // Capture at 15 fps
 
-                //update the canvas
-                this.desktopCameraCanvasDrawerTimer = setInterval(() => {
-                    //draw screen to canvas
-                    canvas.width = screenVideo.videoWidth;
-                    canvas.height = screenVideo.videoHeight;
-                    canvasContext.drawImage(screenVideo, 0, 0, canvas.width, canvas.height);
+            // Initialize or update the local stream
+            const promise = this.localStream == null
+                ? this.gotStream(canvasStream)
+                : this.updateVideoTrack(canvasStream, streamId, onEndedCallback, null);
 
-                    var cameraWidth = screenVideo.videoWidth * (this.camera_percent / 100);
-                    var cameraHeight = (cameraVideo.videoHeight / cameraVideo.videoWidth) * cameraWidth
+            promise.then(() => {
+                // Initialize the worker
+                const worker = new Worker(new URL("./draw-desktop-with-camera-source-worker.js", import.meta.url));
+
+                // Send the OffscreenCanvas to the worker
+                worker.postMessage({
+                    type: 'init',
+                    data: { canvas: offscreenCanvas, width: screenVideo.videoWidth, height: screenVideo.videoHeight },
+                }, [offscreenCanvas]);
+
+                // Wait for both videos to load
+                Promise.all([setupVideo(screenVideo), setupVideo(cameraVideo)]).then(() => {
+                    const frameInterval = 1000 / 15; // 15 fps
+
+                    // Periodically send frames to the worker
+                    const sendFrames = () => {
+                        if (screenVideo.videoWidth > 0 && cameraVideo.videoWidth > 0) {
+                            createImageBitmap(screenVideo).then(screenBitmap => {
+                                createImageBitmap(cameraVideo).then(cameraBitmap => {
+                                    worker.postMessage({
+                                        type: 'frame',
+                                        data: { screenFrame: screenBitmap, cameraFrame: cameraBitmap },
+                                    }, [screenBitmap, cameraBitmap]);
+                                }).catch(err => {
+                                    console.error("Error creating camera ImageBitmap:", err);
+                                });
+                            }).catch(err => {
+                                console.error("Error creating screen ImageBitmap:", err);
+                            });
+                        } else {
+                            console.warn("Video dimensions are invalid.");
+                        }
+                    };
 
-                    var positionX = (canvas.width - cameraWidth) - this.camera_margin;
-                    var positionY;
+                    const frameTimer = setInterval(sendFrames, frameInterval);
 
-                    if (this.camera_location == "top") {
-                        positionY = this.camera_margin;
-                    } else { //if not top, make it bottom
-                        //draw camera on right bottom corner
-                        positionY = (canvas.height - cameraHeight) - this.camera_margin;
-                    }
-                    canvasContext.drawImage(cameraVideo, positionX, positionY, cameraWidth, cameraHeight);
-                }, 66);
+                    // Cleanup
+                    this.desktopCameraCanvasDrawerTimer = () => {
+                        clearInterval(frameTimer);
+                        worker.terminate();
+                    };
+                });
             });
-        }, true)
+        }, true);
     }
 
     /**