From e2bee8997df34c300d623b3458166303a725eb73 Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Fri, 5 Jun 2026 10:42:16 -0500 Subject: [PATCH 1/3] Add 360 video visualization --- .../core/ui/view/video/FFMPEG360View.js | 14 + .../source/core/ui/view/video/VideoView.js | 13 +- .../source/core/ui/view/video/YUV360Canvas.js | 603 ++++++++++++++++++ src/lib/VisualizationHelpers.ts | 1 + .../visualizations/video/Builder.ts | 1 + .../wizard/customizations/VideoOptions.vue | 19 +- 6 files changed, 649 insertions(+), 2 deletions(-) create mode 100644 lib/osh-js/source/core/ui/view/video/FFMPEG360View.js create mode 100644 lib/osh-js/source/core/ui/view/video/YUV360Canvas.js diff --git a/lib/osh-js/source/core/ui/view/video/FFMPEG360View.js b/lib/osh-js/source/core/ui/view/video/FFMPEG360View.js new file mode 100644 index 00000000..7818c6f3 --- /dev/null +++ b/lib/osh-js/source/core/ui/view/video/FFMPEG360View.js @@ -0,0 +1,14 @@ +import FFMPEGView from './FFMPEGView'; +import YUV360Canvas from './YUV360Canvas'; + +class FFMPEG360View extends FFMPEGView { + constructor(properties) { + super(properties); + } + + createCanvas(width, height, style){ + return new YUV360Canvas({width: width, height: height, contextOptions: {preserveDrawingBuffer: true}}); + } +} + +export default FFMPEG360View; \ No newline at end of file diff --git a/lib/osh-js/source/core/ui/view/video/VideoView.js b/lib/osh-js/source/core/ui/view/video/VideoView.js index a9e527db..79696f16 100644 --- a/lib/osh-js/source/core/ui/view/video/VideoView.js +++ b/lib/osh-js/source/core/ui/view/video/VideoView.js @@ -1,6 +1,7 @@ import MjpegView from "./MjpegView"; import WebCodecView from "./WebCodecView"; import FFMPEGView from "./FFMPEGView"; +import FFMPEG360View from "/lib/osh-js/source/core/ui/view/video/FFMPEG360View"; import View from "../View"; import {isDefined} from "../../../utils/Utils"; @@ -29,6 +30,7 @@ class VideoView extends View { * @param {Boolean} [properties.directPlay=false] - Enable or ignore the framerate play * @param {Boolean} [properties.showTime=false] - Enable or ignore the show timestamp text onto the canvas * @param {Boolean} [properties.showStats=false] - Enable or ignore the display stats (FPS number) onto the canvas + * @param {Boolean} [properties.is360=false] - Enable interactive 360 display * @param {Number} [properties.width=1920] - Set the default canvas width * @param {Number} [properties.height=1080] - Set the default canvas height * @param {Number} [properties.useWebCodecApi=true] - Use experimental WebCodecApi @@ -41,13 +43,22 @@ class VideoView extends View { this.videoView = undefined; this.canvasResolve = undefined; this.useWebCodecApi = true; + if('is360' in properties) { + this.is360 = properties['is360']; + } if('useWebCodecApi' in properties) { this.useWebCodecApi = properties['useWebCodecApi']; } } createVideoView(compression) { - if(compression === 'jpeg') { + if (this.is360 === true) { + this.videoView = new FFMPEG360View({ + ...this.properties, + codec: compression, + layers: [] + }); + } else if(compression === 'jpeg') { // create MJPEG View this.videoView = new MjpegView({ ...this.properties, diff --git a/lib/osh-js/source/core/ui/view/video/YUV360Canvas.js b/lib/osh-js/source/core/ui/view/video/YUV360Canvas.js new file mode 100644 index 00000000..4f0f6673 --- /dev/null +++ b/lib/osh-js/source/core/ui/view/video/YUV360Canvas.js @@ -0,0 +1,603 @@ +import YUVCanvas from './YUVCanvas'; + +class YUV360Canvas extends YUVCanvas { + + constructor(parOptions) { + parOptions = parOptions || {}; + super(parOptions); // calls this.init() via the parent constructor + + // Camera state — set after super() because init() already ran. + // Draw functions capture `this` by reference, so they read the + // current values at call time, not at init time. + this.yaw = parOptions.initialYaw != null ? parOptions.initialYaw : 0; + this.pitch = parOptions.initialPitch != null ? parOptions.initialPitch : 0; + this.fovDeg = parOptions.fov != null ? parOptions.fov : 75; + + this._initCameraControls(); + } + + // ───────────────────────────────────────────────────────────────────────── + // Core init overrides + // ───────────────────────────────────────────────────────────────────────── + + /** + * Override init() so the parent constructor calls our sphere pipeline + * instead of the flat-quad one. Structure mirrors the parent exactly — + * initContextGL → initProgram → initBuffers → initTextures — but the + * draw function assigned at the end targets sphere geometry. + */ + init() { + this.initContextGL(); + + if (this.contextGL) { + this.initProgram(); + this.initBuffers(); + this.initTextures(); + } + + if (this.type === 'yuv420') { + this.drawNextOutputPictureGL = (par) => this._drawYUV420(par); + } else if (this.type === 'yuv422') { + this.drawNextOutputPictureGL = (par) => this._drawYUV422(par); + } + } + + /** + * Override initProgram() with sphere-appropriate shaders. + * + * Vertex shader: transforms sphere vertices with an MVP matrix. + * The sphere's natural UV coordinates already encode longitude/latitude + * correctly for equirectangular textures, so no per-plane offset math + * is needed in the vertex stage. + * + * Fragment shader: identical YUV→RGB conversion to the parent, but + * uses a single shared UV varying (the sphere guarantees U/V planes + * sample correctly regardless of their halved texture dimensions). + */ + initProgram() { + const gl = this.contextGL; + + // ── Vertex shader ───────────────────────────────────────────────────── + const vertSrc = [ + 'attribute vec3 vertexPos;', + 'attribute vec2 texturePos;', + 'uniform mat4 MVP;', + 'varying vec2 vUV;', + 'void main() {', + ' gl_Position = MVP * vec4(vertexPos, 1.0);', + ' vUV = texturePos;', + '}', + ].join('\n'); + + // ── Fragment shader ─────────────────────────────────────────────────── + let fragSrc; + + if (this.type === 'yuv420') { + // Three separate luminance textures (Y full-res, U/V half-res). + // All three are sampled at the same UV; WebGL handles the + // resolution difference automatically via texture dimensions. + fragSrc = [ + 'precision highp float;', + 'varying highp vec2 vUV;', + 'uniform sampler2D ySampler;', + 'uniform sampler2D uSampler;', + 'uniform sampler2D vSampler;', + 'uniform mat4 YUV2RGB;', + 'void main(void) {', + ' float y = texture2D(ySampler, vUV).r;', + ' float u = texture2D(uSampler, vUV).r;', + ' float v = texture2D(vSampler, vUV).r;', + ' gl_FragColor = vec4(y, u, v, 1.0) * YUV2RGB;', + '}', + ].join('\n'); + } else if (this.type === 'yuv422') { + // Luma and chroma are interleaved in a single LUMINANCE texture. + // The unpacking logic is identical to the parent's YUV422 shader. + fragSrc = [ + 'precision highp float;', + 'varying highp vec2 vUV;', + 'uniform sampler2D sampler;', + 'uniform highp vec2 resolution;', + 'uniform mat4 YUV2RGB;', + 'void main(void) {', + ' float texPixX = 1.0 / resolution.x;', + ' float logPixX = 2.0 / resolution.x;', + ' float logHalfPixX = 4.0 / resolution.x;', + ' float steps = floor(vUV.x / logPixX);', + ' float uvSteps = floor(vUV.x / logHalfPixX);', + ' float y = texture2D(sampler, vec2(logPixX * steps + texPixX, vUV.y)).r;', + ' float u = texture2D(sampler, vec2(logHalfPixX * uvSteps, vUV.y)).r;', + ' float v = texture2D(sampler, vec2(logHalfPixX * uvSteps + 2.0 * texPixX, vUV.y)).r;', + ' gl_FragColor = vec4(y, u, v, 1.0) * YUV2RGB;', + '}', + ].join('\n'); + } + + // ── YUV → RGB colour matrix (same values as parent) ─────────────────── + let YUV2RGB; + if (this.conversionType === 'rec709') { + YUV2RGB = [ + 1.16438, 0.00000, 1.79274, -0.97295, + 1.16438, -0.21325, -0.53291, 0.30148, + 1.16438, 2.11240, 0.00000, -1.13340, + 0, 0, 0, 1, + ]; + } else { + // Default: ITU-T Rec. 601 + YUV2RGB = [ + 1.16438, 0.00000, 1.59603, -0.87079, + 1.16438, -0.39176, -0.81297, 0.52959, + 1.16438, 2.01723, 0.00000, -1.08139, + 0, 0, 0, 1, + ]; + } + + // ── Compile, link, activate ─────────────────────────────────────────── + const vert = this._compileShader(gl.VERTEX_SHADER, vertSrc); + const frag = this._compileShader(gl.FRAGMENT_SHADER, fragSrc); + + const program = gl.createProgram(); + gl.attachShader(program, vert); + gl.attachShader(program, frag); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + console.error('EquirectangularSphereCanvas: program link error —', + gl.getProgramInfoLog(program)); + } + + gl.useProgram(program); + + const yuv2rgbLoc = gl.getUniformLocation(program, 'YUV2RGB'); + gl.uniformMatrix4fv(yuv2rgbLoc, false, YUV2RGB); + + // Cache the MVP uniform location for per-frame updates. + this.mvpUniform = gl.getUniformLocation(program, 'MVP'); + this.shaderProgram = program; + } + + /** + * Override initBuffers() to upload a UV sphere mesh instead of a flat quad. + * + * The sphere uses reversed triangle winding so the camera at the origin + * sees texture on the inner surface. Positions and UVs are uploaded to + * STATIC_DRAW buffers; indices go into an ELEMENT_ARRAY_BUFFER. + */ + initBuffers() { + const gl = this.contextGL; + const program = this.shaderProgram; + + const LAT_BANDS = 32; // horizontal rings — increase for smoother poles + const LONG_BANDS = 64; // vertical slices — increase for smoother edges + + const { positions, texCoords, indices } = this._buildSphere(LAT_BANDS, LONG_BANDS); + this.sphereIndexCount = indices.length; + + // ── Position buffer ─────────────────────────────────────────────────── + const posBuf = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, posBuf); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW); + + const posLoc = gl.getAttribLocation(program, 'vertexPos'); + gl.enableVertexAttribArray(posLoc); + gl.vertexAttribPointer(posLoc, 3, gl.FLOAT, false, 0, 0); + + // ── UV buffer ───────────────────────────────────────────────────────── + const uvBuf = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, uvBuf); + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(texCoords), gl.STATIC_DRAW); + + const uvLoc = gl.getAttribLocation(program, 'texturePos'); + gl.enableVertexAttribArray(uvLoc); + gl.vertexAttribPointer(uvLoc, 2, gl.FLOAT, false, 0, 0); + + // ── Index buffer ────────────────────────────────────────────────────── + // 32 × 64 bands → (32+1)×(64+1) = 2145 vertices < 65 535, so Uint16 is fine. + const idxBuf = gl.createBuffer(); + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, idxBuf); + gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, new Uint16Array(indices), gl.STATIC_DRAW); + + this.sphereIndexBuffer = idxBuf; + } + + /** + * Override initTexture() to use LINEAR filtering. + * The parent uses NEAREST, which produces visible blocky seams on a + * curved surface at anything but 1:1 pixel mapping. + */ + initTexture() { + const gl = this.contextGL; + const textureRef = gl.createTexture(); + + gl.bindTexture(gl.TEXTURE_2D, textureRef); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.bindTexture(gl.TEXTURE_2D, null); + + return textureRef; + } + + // ───────────────────────────────────────────────────────────────────────── + // Per-frame draw functions + // ───────────────────────────────────────────────────────────────────────── + + /** + * Upload three YUV planes and render the sphere for a YUV 4:2:0 frame. + * Called automatically via this.drawNextOutputPictureGL each frame. + */ + _drawYUV420(par) { + const gl = this.contextGL; + + const { yData, uData, vData } = par; + const width = this.width; + const height = this.height; + + const yDataPerRow = par.yDataPerRow || width; + const yRowCnt = par.yRowCnt || height; + const uDataPerRow = par.uDataPerRow || (width / 2); + const uRowCnt = par.uRowCnt || (height / 2); + const vDataPerRow = par.vDataPerRow || uDataPerRow; + const vRowCnt = par.vRowCnt || uRowCnt; + + gl.viewport(0, 0, this.canvasElement.width, this.canvasElement.height); + gl.clearColor(0, 0, 0, 1); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.enable(gl.DEPTH_TEST); + // Ensure back-face culling is off — we're rendering the inside of the sphere. + gl.disable(gl.CULL_FACE); + + // Upload Y plane (full resolution, single channel) + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.yTextureRef); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, + yDataPerRow, yRowCnt, 0, + gl.LUMINANCE, gl.UNSIGNED_BYTE, yData); + + // Upload U plane (half resolution) + gl.activeTexture(gl.TEXTURE1); + gl.bindTexture(gl.TEXTURE_2D, this.uTextureRef); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, + uDataPerRow, uRowCnt, 0, + gl.LUMINANCE, gl.UNSIGNED_BYTE, uData); + + // Upload V plane (half resolution) + gl.activeTexture(gl.TEXTURE2); + gl.bindTexture(gl.TEXTURE_2D, this.vTextureRef); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, + vDataPerRow, vRowCnt, 0, + gl.LUMINANCE, gl.UNSIGNED_BYTE, vData); + + gl.uniformMatrix4fv(this.mvpUniform, false, this._computeMVP()); + + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.sphereIndexBuffer); + gl.drawElements(gl.TRIANGLES, this.sphereIndexCount, gl.UNSIGNED_SHORT, 0); + } + + /** + * Upload an interleaved YUV 4:2:2 texture and render the sphere. + */ + _drawYUV422(par) { + const gl = this.contextGL; + + const { data } = par; + const width = this.width; + const height = this.height; + const dataPerRow = par.dataPerRow || (width * 2); + const rowCnt = par.rowCnt || height; + + gl.viewport(0, 0, this.canvasElement.width, this.canvasElement.height); + gl.clearColor(0, 0, 0, 1); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.enable(gl.DEPTH_TEST); + gl.disable(gl.CULL_FACE); + + gl.uniform2f( + gl.getUniformLocation(this.shaderProgram, 'resolution'), + dataPerRow, height + ); + + gl.activeTexture(gl.TEXTURE0); + gl.bindTexture(gl.TEXTURE_2D, this.textureRef); + gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, + dataPerRow, rowCnt, 0, + gl.LUMINANCE, gl.UNSIGNED_BYTE, data); + + gl.uniformMatrix4fv(this.mvpUniform, false, this._computeMVP()); + + gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, this.sphereIndexBuffer); + gl.drawElements(gl.TRIANGLES, this.sphereIndexCount, gl.UNSIGNED_SHORT, 0); + } + + // ───────────────────────────────────────────────────────────────────────── + // Sphere mesh generation + // ───────────────────────────────────────────────────────────────────────── + + /** + * Build a UV sphere with inside-out (reversed) winding. + * + * UV mapping: + * u = 0 → left edge of equirectangular frame (−180° lon) + * u = 1 → right edge (+180° lon) + * v = 0 → top of frame (north pole, +90° lat) + * v = 1 → bottom (south pole, −90° lat) + * + * The horizontal flip (u = 1 − lon/longBands) is necessary because without + * it the image appears mirror-reversed when viewed from inside the sphere. + * + * @param {number} latBands Number of horizontal rings (latitude divisions) + * @param {number} longBands Number of vertical slices (longitude divisions) + * @returns {{ positions: number[], texCoords: number[], indices: number[] }} + */ + _buildSphere(latBands, longBands) { + const positions = []; + const texCoords = []; + const indices = []; + + for (let lat = 0; lat <= latBands; lat++) { + const theta = (lat / latBands) * Math.PI; // 0 (top) → π (bottom) + const sinTheta = Math.sin(theta); + const cosTheta = Math.cos(theta); + + for (let lon = 0; lon <= longBands; lon++) { + const phi = (lon / longBands) * 2 * Math.PI; + const sinPhi = Math.sin(phi); + const cosPhi = Math.cos(phi); + + // Unit-radius sphere centred at the origin. + positions.push( + cosPhi * sinTheta, // x + cosTheta, // y (1 at north pole, −1 at south pole) + sinPhi * sinTheta // z + ); + + // Flip u so the image reads left-to-right from inside the sphere. + texCoords.push( + 1 - lon / longBands, // u + lat / latBands // v + ); + } + } + + // Reversed winding (a, c, b instead of a, b, c) puts front faces + // on the inside surface so the camera sees them from origin. + for (let lat = 0; lat < latBands; lat++) { + for (let lon = 0; lon < longBands; lon++) { + const a = lat * (longBands + 1) + lon; + const b = a + 1; + const c = (lat + 1) * (longBands + 1) + lon; + const d = c + 1; + + indices.push(a, c, b); // reversed winding — upper triangle + indices.push(b, c, d); // reversed winding — lower triangle + } + } + + return { positions, texCoords, indices }; + } + + // ───────────────────────────────────────────────────────────────────────── + // Camera & MVP + // ───────────────────────────────────────────────────────────────────────── + + /** + * Compute the Model-View-Projection matrix for the current camera state. + * + * The model is identity (unit sphere at origin). The view matrix is the + * inverse of the camera's rotation: Rx(−pitch) · Ry(−yaw). The projection + * is a standard perspective frustum. + * + * @returns {Float32Array} Column-major 4×4 MVP matrix ready for uniformMatrix4fv + */ + _computeMVP() { + const fov = ((this.fovDeg != null ? this.fovDeg : 75) * Math.PI) / 180; + const aspect = this.canvasElement.width / (this.canvasElement.height || 1); + const yaw = this.yaw || 0; + const pitch = this.pitch || 0; + + const proj = this._mat4Perspective(fov, aspect, 0.1, 100); + // View = inverse(camera pose) = Rx(−pitch) · Ry(−yaw) + // Order: first rotate world by Ry(−yaw), then tilt by Rx(−pitch). + const view = this._mat4Mul( + this._mat4RotX(-pitch), + this._mat4RotY(-yaw) + ); + + return this._mat4Mul(proj, view); + } + + // ───────────────────────────────────────────────────────────────────────── + // Camera controls + // ───────────────────────────────────────────────────────────────────────── + + /** + * Attach mouse, touch, and wheel listeners to the canvas for interactive + * panning (drag) and FOV zoom (pinch / scroll wheel). + */ + _initCameraControls() { + const canvas = this.canvasElement; + const DRAG_SENS = 0.003; // radians per pixel — adjust to taste + + let dragging = false; + let lastX = 0; + let lastY = 0; + + const onDragStart = (x, y) => { + dragging = true; + lastX = x; + lastY = y; + }; + + const onDragMove = (x, y) => { + if (!dragging) return; + const dx = x - lastX; + const dy = y - lastY; + lastX = x; + lastY = y; + + this.yaw += dx * DRAG_SENS; + // Clamp pitch to ±90° so the view never flips past a pole. + this.pitch = Math.max( + -Math.PI / 2, + Math.min(Math.PI / 2, this.pitch + dy * DRAG_SENS) + ); + }; + + const onDragEnd = () => { dragging = false; }; + + // ── Mouse ───────────────────────────────────────────────────────────── + canvas.addEventListener('mousedown', (e) => onDragStart(e.clientX, e.clientY)); + window.addEventListener('mousemove', (e) => onDragMove(e.clientX, e.clientY)); + window.addEventListener('mouseup', onDragEnd); + + // ── Scroll-wheel FOV zoom ───────────────────────────────────────────── + canvas.addEventListener('wheel', (e) => { + e.preventDefault(); + this.fovDeg = Math.max(30, Math.min(120, this.fovDeg + e.deltaY * 0.05)); + }, { passive: false }); + + // ── Touch drag ──────────────────────────────────────────────────────── + canvas.addEventListener('touchstart', (e) => { + if (e.touches.length === 1) { + e.preventDefault(); + onDragStart(e.touches[0].clientX, e.touches[0].clientY); + } + }, { passive: false }); + + window.addEventListener('touchmove', (e) => { + if (e.touches.length === 1) { + e.preventDefault(); + onDragMove(e.touches[0].clientX, e.touches[0].clientY); + } + }, { passive: false }); + + window.addEventListener('touchend', onDragEnd); + + // ── Pinch-to-zoom FOV ───────────────────────────────────────────────── + let lastPinchDist = null; + + canvas.addEventListener('touchstart', (e) => { + if (e.touches.length === 2) { + lastPinchDist = this._pinchDist(e.touches); + } + }, { passive: true }); + + canvas.addEventListener('touchmove', (e) => { + if (e.touches.length === 2 && lastPinchDist !== null) { + const dist = this._pinchDist(e.touches); + this.fovDeg = Math.max(30, Math.min(120, + this.fovDeg - (dist - lastPinchDist) * 0.1 + )); + lastPinchDist = dist; + } + }, { passive: true }); + + canvas.addEventListener('touchend', () => { lastPinchDist = null; }); + canvas.addEventListener('touchcancel', () => { lastPinchDist = null; }); + } + + _pinchDist(touches) { + const dx = touches[0].clientX - touches[1].clientX; + const dy = touches[0].clientY - touches[1].clientY; + return Math.sqrt(dx * dx + dy * dy); + } + + // ───────────────────────────────────────────────────────────────────────── + // Matrix helpers (column-major, matching WebGL / GLSL convention) + // + // In column-major layout, element at (row, col) lives at index col*4+row. + // Matrix multiplication out = a·b is: out[row,col] = Σ_k a[row,k] · b[k,col] + // ───────────────────────────────────────────────────────────────────────── + + /** + * Standard perspective projection matrix. + * @param {number} fov Vertical field of view in radians + * @param {number} aspect Viewport width / height + * @param {number} near Near clip plane distance + * @param {number} far Far clip plane distance + * @returns {Float32Array} Column-major 4×4 + */ + _mat4Perspective(fov, aspect, near, far) { + const f = 1 / Math.tan(fov / 2); + const nf = 1 / (near - far); + // Columns listed left-to-right; rows listed top-to-bottom within each column. + return new Float32Array([ + f / aspect, 0, 0, 0, // col 0 + 0, f, 0, 0, // col 1 + 0, 0, (far + near) * nf, -1, // col 2 + 0, 0, 2 * far * near * nf, 0, // col 3 + ]); + } + + /** + * Rotation around the Y axis (yaw). + * @param {number} a Angle in radians + * @returns {Float32Array} Column-major 4×4 + */ + _mat4RotY(a) { + const c = Math.cos(a); + const s = Math.sin(a); + return new Float32Array([ + c, 0, -s, 0, // col 0 + 0, 1, 0, 0, // col 1 + s, 0, c, 0, // col 2 + 0, 0, 0, 1, // col 3 + ]); + } + + /** + * Rotation around the X axis (pitch). + * @param {number} a Angle in radians + * @returns {Float32Array} Column-major 4×4 + */ + _mat4RotX(a) { + const c = Math.cos(a); + const s = Math.sin(a); + return new Float32Array([ + 1, 0, 0, 0, // col 0 + 0, c, s, 0, // col 1 + 0, -s, c, 0, // col 2 + 0, 0, 0, 1, // col 3 + ]); + } + + /** + * Multiply two column-major 4×4 matrices. Returns a · b. + * In GL terms: "first apply b, then apply a". + * @param {Float32Array} a + * @param {Float32Array} b + * @returns {Float32Array} Column-major 4×4 + */ + _mat4Mul(a, b) { + const out = new Float32Array(16); + for (let col = 0; col < 4; col++) { + for (let row = 0; row < 4; row++) { + let sum = 0; + for (let k = 0; k < 4; k++) { + sum += a[k * 4 + row] * b[col * 4 + k]; + } + out[col * 4 + row] = sum; + } + } + return out; + } + + // ───────────────────────────────────────────────────────────────────────── + // Shader compile helper + // ───────────────────────────────────────────────────────────────────────── + + _compileShader(type, src) { + const gl = this.contextGL; + const shader = gl.createShader(type); + gl.shaderSource(shader, src); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + const label = type === gl.VERTEX_SHADER ? 'Vertex' : 'Fragment'; + console.error(`EquirectangularSphereCanvas: ${label} shader compile error —`, + gl.getShaderInfoLog(shader)); + } + return shader; + } +} + +export default YUV360Canvas; \ No newline at end of file diff --git a/src/lib/VisualizationHelpers.ts b/src/lib/VisualizationHelpers.ts index 478787b8..8b60be0a 100644 --- a/src/lib/VisualizationHelpers.ts +++ b/src/lib/VisualizationHelpers.ts @@ -97,6 +97,7 @@ export interface IVideoViewProperties extends DataViewProperties { showTime: boolean; showStats: boolean; useWebCodecApi: boolean; + is360: boolean; width: number; height: number; } diff --git a/src/modules/visualization/visualizations/video/Builder.ts b/src/modules/visualization/visualizations/video/Builder.ts index b5379e6f..554a3a31 100644 --- a/src/modules/visualization/visualizations/video/Builder.ts +++ b/src/modules/visualization/visualizations/video/Builder.ts @@ -88,6 +88,7 @@ export function CreateVideoViewProps( useWebCodecApi: true, showTime: visOptions?.time, showStats: visOptions?.stats, + is360: visOptions?.is360, }; for (const [dsId, entry] of Object.entries(datastreams)) { diff --git a/src/modules/visualization/wizard/customizations/VideoOptions.vue b/src/modules/visualization/wizard/customizations/VideoOptions.vue index 2f87b31b..635040b2 100644 --- a/src/modules/visualization/wizard/customizations/VideoOptions.vue +++ b/src/modules/visualization/wizard/customizations/VideoOptions.vue @@ -6,6 +6,7 @@ const vwStore = useVizWizStore(); const stats = ref(false); const time = ref(false); +const is360 = ref(false); watch(stats, (val) => { vwStore.updateVisualizationCustomizationOptions({ stats: val }); @@ -13,7 +14,11 @@ watch(stats, (val) => { watch(time, (val) => { vwStore.updateVisualizationCustomizationOptions({ time: val }); -}); +}) + +watch(is360, (val) => { + vwStore.updateVisualizationCustomizationOptions({ is360: val }); +}) onMounted(() => { if (!vwStore.visualizationCustomizationOptions.stats) { @@ -31,6 +36,14 @@ onMounted(() => { } else { time.value = vwStore.visualizationCustomizationOptions.time; } + + if (!vwStore.visualizationCustomizationOptions.is360) { + vwStore.updateVisualizationCustomizationOptions({ + is360: is360.value, + }); + } else { + is360.value = vwStore.visualizationCustomizationOptions.is360; + } }); @@ -43,4 +56,8 @@ onMounted(() => { v-model="time" label="Show Video Time" /> + From 1c228656b0c07157eb49f018af3b5bc4a0b6110e Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Tue, 16 Jun 2026 12:41:41 -0500 Subject: [PATCH 2/3] Add additional projection types --- .../core/ui/view/video/FFMPEG360View.js | 20 +- .../source/core/ui/view/video/VideoView.js | 8 +- .../source/core/ui/view/video/YUV360Canvas.js | 587 +++++++++--------- src/lib/VisualizationHelpers.ts | 2 +- .../visualizations/video/Builder.ts | 2 +- .../wizard/customizations/VideoOptions.vue | 91 ++- 6 files changed, 395 insertions(+), 315 deletions(-) diff --git a/lib/osh-js/source/core/ui/view/video/FFMPEG360View.js b/lib/osh-js/source/core/ui/view/video/FFMPEG360View.js index 7818c6f3..e4e9bf17 100644 --- a/lib/osh-js/source/core/ui/view/video/FFMPEG360View.js +++ b/lib/osh-js/source/core/ui/view/video/FFMPEG360View.js @@ -4,10 +4,28 @@ import YUV360Canvas from './YUV360Canvas'; class FFMPEG360View extends FFMPEGView { constructor(properties) { super(properties); + this.fisheyeFovDeg = properties?.props360?.fisheyeFovDeg ?? 204; + this.projection = properties?.props360?.projection ?? 'equirectangular'; + this.canvas360.setProjection(this.projection); + this.canvas360.setFisheyeFov(this.fisheyeFovDeg); + } + + get canvas360() { + return /** @type {YUV360Canvas} */ (this.yuvCanvas); } createCanvas(width, height, style){ - return new YUV360Canvas({width: width, height: height, contextOptions: {preserveDrawingBuffer: true}}); + return new YUV360Canvas({width: width, height: height, fisheyeFovDeg: this.fisheyeFovDeg, projection: this.projection, contextOptions: {preserveDrawingBuffer: true}}); + } + + setFisheyeFov(fullFovDeg) { + this.fisheyeFovDeg = fullFovDeg; + if (this.yuvCanvas) this.canvas360.setFisheyeFov(fullFovDeg); + } + + setProjection(projection) { + this.projection = projection; + if (this.yuvCanvas) this.canvas360.setProjection(projection); } } diff --git a/lib/osh-js/source/core/ui/view/video/VideoView.js b/lib/osh-js/source/core/ui/view/video/VideoView.js index 79696f16..975928c5 100644 --- a/lib/osh-js/source/core/ui/view/video/VideoView.js +++ b/lib/osh-js/source/core/ui/view/video/VideoView.js @@ -30,7 +30,7 @@ class VideoView extends View { * @param {Boolean} [properties.directPlay=false] - Enable or ignore the framerate play * @param {Boolean} [properties.showTime=false] - Enable or ignore the show timestamp text onto the canvas * @param {Boolean} [properties.showStats=false] - Enable or ignore the display stats (FPS number) onto the canvas - * @param {Boolean} [properties.is360=false] - Enable interactive 360 display + * @param {Object} [properties.props360={}] - Enable interactive 360 display * @param {Number} [properties.width=1920] - Set the default canvas width * @param {Number} [properties.height=1080] - Set the default canvas height * @param {Number} [properties.useWebCodecApi=true] - Use experimental WebCodecApi @@ -43,8 +43,8 @@ class VideoView extends View { this.videoView = undefined; this.canvasResolve = undefined; this.useWebCodecApi = true; - if('is360' in properties) { - this.is360 = properties['is360']; + if('props360' in properties) { + this.props360 = properties['props360']; } if('useWebCodecApi' in properties) { this.useWebCodecApi = properties['useWebCodecApi']; @@ -52,7 +52,7 @@ class VideoView extends View { } createVideoView(compression) { - if (this.is360 === true) { + if (this.props360 != null) { this.videoView = new FFMPEG360View({ ...this.properties, codec: compression, diff --git a/lib/osh-js/source/core/ui/view/video/YUV360Canvas.js b/lib/osh-js/source/core/ui/view/video/YUV360Canvas.js index 4f0f6673..7b01d6fa 100644 --- a/lib/osh-js/source/core/ui/view/video/YUV360Canvas.js +++ b/lib/osh-js/source/core/ui/view/video/YUV360Canvas.js @@ -1,31 +1,252 @@ import YUVCanvas from './YUVCanvas'; +export const Projection = Object.freeze({ + EQUIRECTANGULAR : 0, + DUAL_FISHEYE : 1, + CUBEMAP_3X2 : 2, + SINGLE_FISHEYE : 3, +}); + +const PROJECTION_MAP = { + 'equirectangular': Projection.EQUIRECTANGULAR, + 'dual_fisheye': Projection.DUAL_FISHEYE, + 'cubemap': Projection.CUBEMAP_3X2, + 'single_fisheye': Projection.SINGLE_FISHEYE, +}; + +const PROJECTION_GLSL = ` +// ── Equirectangular ────────────────────────────────────────────────────────── +vec2 equirectUV(vec3 dir) { + float lon = atan(dir.z, dir.x); + float lat = asin(clamp(dir.y, -1.0, 1.0)); + return vec2( + lon / (2.0 * PI) + 0.5, + 1.0 - (lat / PI + 0.5) + ); +} + +// ── Single fisheye ─────────────────────────────────────────────────────────── +vec2 singleFisheyeUV(vec3 dir, float fovRad) { + float theta = acos(clamp(dir.z, -1.0, 1.0)); + float r = clamp(theta / fovRad, 0.0, 1.0); + float phi = atan(dir.y, dir.x); + return vec2( + cos(phi) * r * 0.5 + 0.5, + 1.0 - (sin(phi) * r * 0.5 + 0.5) + ); +} + +// ── Dual fisheye ───────────────────────────────────────────────────────────── +vec2 dualFisheyeUV(vec3 dir, float fovRad) { + bool front = dir.z < 0.0; + float theta = acos(clamp(front ? -dir.z : dir.z, 0.0, 1.0)); + float r = clamp(theta / fovRad, 0.0, 1.0); + float phi = atan(dir.y, front ? dir.x : -dir.x); + vec2 circle = vec2(cos(phi) * r * 0.25, sin(phi) * r * 0.5); + float xOffset = front ? 0.25 : 0.75; + return vec2(circle.x + xOffset, 1.0 - (circle.y + 0.5)); +} + +// ── Cubemap 3x2 ────────────────────────────────────────────────────────────── +// FFmpeg c3x2 default face order: rludfb +// Row 0: RIGHT(0) LEFT(1) DOWN(2) +// Row 1: UP(3) BACK(4) FRONT(5) +void cubeSelect(vec3 dir, out int face, out float fu, out float fv) { + vec3 a = abs(dir); + + if (a.z >= a.x && a.z >= a.y) { + if (dir.z > 0.0) { + face = 5; // FRONT (+Z) + fu = -dir.x / a.z; + fv = -dir.y / a.z; + } else { + face = 4; // BACK (-Z) + fu = dir.x / a.z; + fv = -dir.y / a.z; + } + } else if (a.x >= a.y) { + if (dir.x > 0.0) { + face = 0; // RIGHT (+X) + fu = dir.z / a.x; + fv = -dir.y / a.x; + } else { + face = 1; // LEFT (-X) + fu = -dir.z / a.x; + fv = -dir.y / a.x; + } + } else { + if (dir.y < 0.0) { + face = 3; // UP + fu = dir.x / a.y; + fv = dir.z / a.y; + } else { + face = 2; // DOWN + fu = dir.x / a.y; + fv = -dir.z / a.y; + } + } +} + +vec2 atlasUV_3x2(int face, float fu, float fv) { + vec2 uv = vec2(fu, fv) * 0.5 + 0.5; + float col = float(face < 3 ? face : face - 3); + float row = float(face < 3 ? 0 : 1); + return vec2((col + uv.x) / 3.0, (row + uv.y) / 2.0); +} + +vec2 cubemapUV(vec3 dir) { + int face; float fu, fv; + cubeSelect(dir, face, fu, fv); + return atlasUV_3x2(face, fu, fv); +} +`; + +function buildFragSrc420() { + return ` +precision highp float; +const float PI = 3.14159265358979323846; + +varying highp vec3 vDir; + +uniform sampler2D ySampler; +uniform sampler2D uSampler; +uniform sampler2D vSampler; +uniform mat4 YUV2RGB; + +uniform int uProjection; +uniform float uFisheyeFovRad; + +${PROJECTION_GLSL} + +void main(void) { + vec3 dir = normalize(vDir); + vec2 uv; + + if (uProjection == 0) uv = equirectUV(dir); + else if (uProjection == 1) uv = dualFisheyeUV(dir, uFisheyeFovRad); + else if (uProjection == 2) uv = cubemapUV(dir); + else if (uProjection == 3) uv = singleFisheyeUV(dir, uFisheyeFovRad); + else uv = equirectUV(dir); + + float y = texture2D(ySampler, uv).r; + float u = texture2D(uSampler, uv).r; + float v = texture2D(vSampler, uv).r; + gl_FragColor = vec4(y, u, v, 1.0) * YUV2RGB; +}`; +} + +function buildFragSrc422() { + return ` +precision highp float; +const float PI = 3.14159265358979323846; + +varying highp vec3 vDir; + +uniform sampler2D sampler; +uniform highp vec2 resolution; +uniform mat4 YUV2RGB; + +uniform int uProjection; +uniform float uFisheyeFovRad; + +${PROJECTION_GLSL} + +void main(void) { + vec3 dir = normalize(vDir); + vec2 uv; + + if (uProjection == 0) uv = equirectUV(dir); + else if (uProjection == 1) uv = dualFisheyeUV(dir, uFisheyeFovRad); + else if (uProjection == 2) uv = cubemapUV(dir); + else if (uProjection == 3) uv = singleFisheyeUV(dir, uFisheyeFovRad); + else uv = equirectUV(dir); + + float texPixX = 1.0 / resolution.x; + float logPixX = 2.0 / resolution.x; + float logHalfPixX = 4.0 / resolution.x; + float steps = floor(uv.x / logPixX); + float uvSteps = floor(uv.x / logHalfPixX); + float y = texture2D(sampler, vec2(logPixX * steps + texPixX, uv.y)).r; + float u = texture2D(sampler, vec2(logHalfPixX * uvSteps, uv.y)).r; + float v = texture2D(sampler, vec2(logHalfPixX * uvSteps + 2.0*texPixX, uv.y)).r; + gl_FragColor = vec4(y, u, v, 1.0) * YUV2RGB; +}`; +} + +const VERT_SRC = ` +attribute vec3 vertexPos; +uniform mat4 MVP; +varying highp vec3 vDir; + +void main() { + gl_Position = MVP * vec4(vertexPos, 1.0); + vDir = vertexPos; +}`; + class YUV360Canvas extends YUVCanvas { + /** + * @param {object} parOptions + * + * Viewing camera: + * @param {number} [parOptions.initialYaw=0] Initial yaw in radians + * @param {number} [parOptions.initialPitch=0] Initial pitch in radians + * @param {number} [parOptions.fov=75] Viewer field of view in degrees + * + * Source projection: + * @param {string} [parOptions.projection='equirectangular'] + * One of: 'equirectangular', 'dual_fisheye', 'cubemap', 'single_fisheye' + * + * Fisheye lens parameters (dual_fisheye and single_fisheye only): + * @param {number} [parOptions.fisheyeFovDeg=204] + * Full field of view of each fisheye lens in degrees. + * 204° is the empirical value for the Insta360 X-series. + * This is the lens FOV, not the viewer FOV. + */ constructor(parOptions) { parOptions = parOptions || {}; - super(parOptions); // calls this.init() via the parent constructor + super(parOptions); - // Camera state — set after super() because init() already ran. - // Draw functions capture `this` by reference, so they read the - // current values at call time, not at init time. this.yaw = parOptions.initialYaw != null ? parOptions.initialYaw : 0; this.pitch = parOptions.initialPitch != null ? parOptions.initialPitch : 0; this.fovDeg = parOptions.fov != null ? parOptions.fov : 75; + this.projection = PROJECTION_MAP[parOptions.projection] ?? Projection.EQUIRECTANGULAR; + + const fullFovDeg = parOptions.fisheyeFovDeg != null ? parOptions.fisheyeFovDeg : 204; + this.fisheyeFovRad = (fullFovDeg / 2) * (Math.PI / 180); + this._initCameraControls(); } // ───────────────────────────────────────────────────────────────────────── - // Core init overrides + // Public API // ───────────────────────────────────────────────────────────────────────── /** - * Override init() so the parent constructor calls our sphere pipeline - * instead of the flat-quad one. Structure mirrors the parent exactly — - * initContextGL → initProgram → initBuffers → initTextures — but the - * draw function assigned at the end targets sphere geometry. + * Switch the source projection at runtime. + * @param {string|number} projection String key or Projection.* constant + */ + setProjection(projection) { + this.projection = typeof projection === 'string' + ? (PROJECTION_MAP[projection] ?? Projection.EQUIRECTANGULAR) + : projection; + this._uploadProjectionUniforms(); + } + + /** + * Update the fisheye lens FOV (full angle in degrees). + * @param {number} fullFovDeg Full lens FOV in degrees (e.g. 204 for Insta360) */ + setFisheyeFov(fullFovDeg) { + this.fisheyeFovRad = (fullFovDeg / 2) * (Math.PI / 180); + this._uploadProjectionUniforms(); + } + + // ───────────────────────────────────────────────────────────────────────── + // Core init overrides + // ───────────────────────────────────────────────────────────────────────── + init() { this.initContextGL(); @@ -42,78 +263,11 @@ class YUV360Canvas extends YUVCanvas { } } - /** - * Override initProgram() with sphere-appropriate shaders. - * - * Vertex shader: transforms sphere vertices with an MVP matrix. - * The sphere's natural UV coordinates already encode longitude/latitude - * correctly for equirectangular textures, so no per-plane offset math - * is needed in the vertex stage. - * - * Fragment shader: identical YUV→RGB conversion to the parent, but - * uses a single shared UV varying (the sphere guarantees U/V planes - * sample correctly regardless of their halved texture dimensions). - */ initProgram() { const gl = this.contextGL; - // ── Vertex shader ───────────────────────────────────────────────────── - const vertSrc = [ - 'attribute vec3 vertexPos;', - 'attribute vec2 texturePos;', - 'uniform mat4 MVP;', - 'varying vec2 vUV;', - 'void main() {', - ' gl_Position = MVP * vec4(vertexPos, 1.0);', - ' vUV = texturePos;', - '}', - ].join('\n'); - - // ── Fragment shader ─────────────────────────────────────────────────── - let fragSrc; - - if (this.type === 'yuv420') { - // Three separate luminance textures (Y full-res, U/V half-res). - // All three are sampled at the same UV; WebGL handles the - // resolution difference automatically via texture dimensions. - fragSrc = [ - 'precision highp float;', - 'varying highp vec2 vUV;', - 'uniform sampler2D ySampler;', - 'uniform sampler2D uSampler;', - 'uniform sampler2D vSampler;', - 'uniform mat4 YUV2RGB;', - 'void main(void) {', - ' float y = texture2D(ySampler, vUV).r;', - ' float u = texture2D(uSampler, vUV).r;', - ' float v = texture2D(vSampler, vUV).r;', - ' gl_FragColor = vec4(y, u, v, 1.0) * YUV2RGB;', - '}', - ].join('\n'); - } else if (this.type === 'yuv422') { - // Luma and chroma are interleaved in a single LUMINANCE texture. - // The unpacking logic is identical to the parent's YUV422 shader. - fragSrc = [ - 'precision highp float;', - 'varying highp vec2 vUV;', - 'uniform sampler2D sampler;', - 'uniform highp vec2 resolution;', - 'uniform mat4 YUV2RGB;', - 'void main(void) {', - ' float texPixX = 1.0 / resolution.x;', - ' float logPixX = 2.0 / resolution.x;', - ' float logHalfPixX = 4.0 / resolution.x;', - ' float steps = floor(vUV.x / logPixX);', - ' float uvSteps = floor(vUV.x / logHalfPixX);', - ' float y = texture2D(sampler, vec2(logPixX * steps + texPixX, vUV.y)).r;', - ' float u = texture2D(sampler, vec2(logHalfPixX * uvSteps, vUV.y)).r;', - ' float v = texture2D(sampler, vec2(logHalfPixX * uvSteps + 2.0 * texPixX, vUV.y)).r;', - ' gl_FragColor = vec4(y, u, v, 1.0) * YUV2RGB;', - '}', - ].join('\n'); - } + const fragSrc = this.type === 'yuv420' ? buildFragSrc420() : buildFragSrc422(); - // ── YUV → RGB colour matrix (same values as parent) ─────────────────── let YUV2RGB; if (this.conversionType === 'rec709') { YUV2RGB = [ @@ -123,7 +277,6 @@ class YUV360Canvas extends YUVCanvas { 0, 0, 0, 1, ]; } else { - // Default: ITU-T Rec. 601 YUV2RGB = [ 1.16438, 0.00000, 1.59603, -0.87079, 1.16438, -0.39176, -0.81297, 0.52959, @@ -132,8 +285,7 @@ class YUV360Canvas extends YUVCanvas { ]; } - // ── Compile, link, activate ─────────────────────────────────────────── - const vert = this._compileShader(gl.VERTEX_SHADER, vertSrc); + const vert = this._compileShader(gl.VERTEX_SHADER, VERT_SRC); const frag = this._compileShader(gl.FRAGMENT_SHADER, fragSrc); const program = gl.createProgram(); @@ -141,94 +293,68 @@ class YUV360Canvas extends YUVCanvas { gl.attachShader(program, frag); gl.linkProgram(program); if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { - console.error('EquirectangularSphereCanvas: program link error —', + console.error('YUV360Canvas: program link error —', gl.getProgramInfoLog(program)); } gl.useProgram(program); - const yuv2rgbLoc = gl.getUniformLocation(program, 'YUV2RGB'); - gl.uniformMatrix4fv(yuv2rgbLoc, false, YUV2RGB); + gl.uniformMatrix4fv( + gl.getUniformLocation(program, 'YUV2RGB'), false, YUV2RGB); - // Cache the MVP uniform location for per-frame updates. - this.mvpUniform = gl.getUniformLocation(program, 'MVP'); - this.shaderProgram = program; + this.mvpUniform = gl.getUniformLocation(program, 'MVP'); + this.projectionUniform = gl.getUniformLocation(program, 'uProjection'); + this.fisheyeFovUniform = gl.getUniformLocation(program, 'uFisheyeFovRad'); + this.shaderProgram = program; + + this._uploadProjectionUniforms(); } - /** - * Override initBuffers() to upload a UV sphere mesh instead of a flat quad. - * - * The sphere uses reversed triangle winding so the camera at the origin - * sees texture on the inner surface. Positions and UVs are uploaded to - * STATIC_DRAW buffers; indices go into an ELEMENT_ARRAY_BUFFER. - */ initBuffers() { const gl = this.contextGL; const program = this.shaderProgram; - const LAT_BANDS = 32; // horizontal rings — increase for smoother poles - const LONG_BANDS = 64; // vertical slices — increase for smoother edges + const LAT_BANDS = 32; + const LONG_BANDS = 64; const { positions, texCoords, indices } = this._buildSphere(LAT_BANDS, LONG_BANDS); this.sphereIndexCount = indices.length; - // ── Position buffer ─────────────────────────────────────────────────── const posBuf = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, posBuf); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(positions), gl.STATIC_DRAW); - const posLoc = gl.getAttribLocation(program, 'vertexPos'); gl.enableVertexAttribArray(posLoc); gl.vertexAttribPointer(posLoc, 3, gl.FLOAT, false, 0, 0); - // ── UV buffer ───────────────────────────────────────────────────────── const uvBuf = gl.createBuffer(); gl.bindBuffer(gl.ARRAY_BUFFER, uvBuf); gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(texCoords), gl.STATIC_DRAW); - const uvLoc = gl.getAttribLocation(program, 'texturePos'); - gl.enableVertexAttribArray(uvLoc); - gl.vertexAttribPointer(uvLoc, 2, gl.FLOAT, false, 0, 0); - - // ── Index buffer ────────────────────────────────────────────────────── - // 32 × 64 bands → (32+1)×(64+1) = 2145 vertices < 65 535, so Uint16 is fine. const idxBuf = gl.createBuffer(); gl.bindBuffer(gl.ELEMENT_ARRAY_BUFFER, idxBuf); gl.bufferData(gl.ELEMENT_ARRAY_BUFFER, new Uint16Array(indices), gl.STATIC_DRAW); - this.sphereIndexBuffer = idxBuf; } - /** - * Override initTexture() to use LINEAR filtering. - * The parent uses NEAREST, which produces visible blocky seams on a - * curved surface at anything but 1:1 pixel mapping. - */ initTexture() { const gl = this.contextGL; const textureRef = gl.createTexture(); - gl.bindTexture(gl.TEXTURE_2D, textureRef); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); gl.bindTexture(gl.TEXTURE_2D, null); - return textureRef; } // ───────────────────────────────────────────────────────────────────────── - // Per-frame draw functions + // Per-frame draw // ───────────────────────────────────────────────────────────────────────── - /** - * Upload three YUV planes and render the sphere for a YUV 4:2:0 frame. - * Called automatically via this.drawNextOutputPictureGL each frame. - */ _drawYUV420(par) { const gl = this.contextGL; - const { yData, uData, vData } = par; const width = this.width; const height = this.height; @@ -244,29 +370,22 @@ class YUV360Canvas extends YUVCanvas { gl.clearColor(0, 0, 0, 1); gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); gl.enable(gl.DEPTH_TEST); - // Ensure back-face culling is off — we're rendering the inside of the sphere. gl.disable(gl.CULL_FACE); - // Upload Y plane (full resolution, single channel) gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, this.yTextureRef); gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, - yDataPerRow, yRowCnt, 0, - gl.LUMINANCE, gl.UNSIGNED_BYTE, yData); + yDataPerRow, yRowCnt, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, yData); - // Upload U plane (half resolution) gl.activeTexture(gl.TEXTURE1); gl.bindTexture(gl.TEXTURE_2D, this.uTextureRef); gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, - uDataPerRow, uRowCnt, 0, - gl.LUMINANCE, gl.UNSIGNED_BYTE, uData); + uDataPerRow, uRowCnt, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, uData); - // Upload V plane (half resolution) gl.activeTexture(gl.TEXTURE2); gl.bindTexture(gl.TEXTURE_2D, this.vTextureRef); gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, - vDataPerRow, vRowCnt, 0, - gl.LUMINANCE, gl.UNSIGNED_BYTE, vData); + vDataPerRow, vRowCnt, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, vData); gl.uniformMatrix4fv(this.mvpUniform, false, this._computeMVP()); @@ -274,13 +393,9 @@ class YUV360Canvas extends YUVCanvas { gl.drawElements(gl.TRIANGLES, this.sphereIndexCount, gl.UNSIGNED_SHORT, 0); } - /** - * Upload an interleaved YUV 4:2:2 texture and render the sphere. - */ _drawYUV422(par) { const gl = this.contextGL; - - const { data } = par; + const { data } = par; const width = this.width; const height = this.height; const dataPerRow = par.dataPerRow || (width * 2); @@ -294,14 +409,12 @@ class YUV360Canvas extends YUVCanvas { gl.uniform2f( gl.getUniformLocation(this.shaderProgram, 'resolution'), - dataPerRow, height - ); + dataPerRow, height); gl.activeTexture(gl.TEXTURE0); gl.bindTexture(gl.TEXTURE_2D, this.textureRef); gl.texImage2D(gl.TEXTURE_2D, 0, gl.LUMINANCE, - dataPerRow, rowCnt, 0, - gl.LUMINANCE, gl.UNSIGNED_BYTE, data); + dataPerRow, rowCnt, 0, gl.LUMINANCE, gl.UNSIGNED_BYTE, data); gl.uniformMatrix4fv(this.mvpUniform, false, this._computeMVP()); @@ -310,32 +423,28 @@ class YUV360Canvas extends YUVCanvas { } // ───────────────────────────────────────────────────────────────────────── - // Sphere mesh generation + // Projection uniforms + // ───────────────────────────────────────────────────────────────────────── + + _uploadProjectionUniforms() { + const gl = this.contextGL; + if (!gl || !this.shaderProgram) return; + gl.useProgram(this.shaderProgram); + gl.uniform1i(this.projectionUniform, this.projection); + gl.uniform1f(this.fisheyeFovUniform, this.fisheyeFovRad); + } + + // ───────────────────────────────────────────────────────────────────────── + // Sphere mesh // ───────────────────────────────────────────────────────────────────────── - /** - * Build a UV sphere with inside-out (reversed) winding. - * - * UV mapping: - * u = 0 → left edge of equirectangular frame (−180° lon) - * u = 1 → right edge (+180° lon) - * v = 0 → top of frame (north pole, +90° lat) - * v = 1 → bottom (south pole, −90° lat) - * - * The horizontal flip (u = 1 − lon/longBands) is necessary because without - * it the image appears mirror-reversed when viewed from inside the sphere. - * - * @param {number} latBands Number of horizontal rings (latitude divisions) - * @param {number} longBands Number of vertical slices (longitude divisions) - * @returns {{ positions: number[], texCoords: number[], indices: number[] }} - */ _buildSphere(latBands, longBands) { const positions = []; const texCoords = []; const indices = []; for (let lat = 0; lat <= latBands; lat++) { - const theta = (lat / latBands) * Math.PI; // 0 (top) → π (bottom) + const theta = (lat / latBands) * Math.PI; const sinTheta = Math.sin(theta); const cosTheta = Math.cos(theta); @@ -344,32 +453,19 @@ class YUV360Canvas extends YUVCanvas { const sinPhi = Math.sin(phi); const cosPhi = Math.cos(phi); - // Unit-radius sphere centred at the origin. - positions.push( - cosPhi * sinTheta, // x - cosTheta, // y (1 at north pole, −1 at south pole) - sinPhi * sinTheta // z - ); - - // Flip u so the image reads left-to-right from inside the sphere. - texCoords.push( - 1 - lon / longBands, // u - lat / latBands // v - ); + positions.push(cosPhi * sinTheta, cosTheta, sinPhi * sinTheta); + texCoords.push(1 - lon / longBands, lat / latBands); } } - // Reversed winding (a, c, b instead of a, b, c) puts front faces - // on the inside surface so the camera sees them from origin. for (let lat = 0; lat < latBands; lat++) { for (let lon = 0; lon < longBands; lon++) { const a = lat * (longBands + 1) + lon; const b = a + 1; const c = (lat + 1) * (longBands + 1) + lon; const d = c + 1; - - indices.push(a, c, b); // reversed winding — upper triangle - indices.push(b, c, d); // reversed winding — lower triangle + indices.push(a, c, b); + indices.push(b, c, d); } } @@ -380,15 +476,6 @@ class YUV360Canvas extends YUVCanvas { // Camera & MVP // ───────────────────────────────────────────────────────────────────────── - /** - * Compute the Model-View-Projection matrix for the current camera state. - * - * The model is identity (unit sphere at origin). The view matrix is the - * inverse of the camera's rotation: Rx(−pitch) · Ry(−yaw). The projection - * is a standard perspective frustum. - * - * @returns {Float32Array} Column-major 4×4 MVP matrix ready for uniformMatrix4fv - */ _computeMVP() { const fov = ((this.fovDeg != null ? this.fovDeg : 75) * Math.PI) / 180; const aspect = this.canvasElement.width / (this.canvasElement.height || 1); @@ -396,13 +483,10 @@ class YUV360Canvas extends YUVCanvas { const pitch = this.pitch || 0; const proj = this._mat4Perspective(fov, aspect, 0.1, 100); - // View = inverse(camera pose) = Rx(−pitch) · Ry(−yaw) - // Order: first rotate world by Ry(−yaw), then tilt by Rx(−pitch). const view = this._mat4Mul( this._mat4RotX(-pitch), this._mat4RotY(-yaw) ); - return this._mat4Mul(proj, view); } @@ -410,89 +494,56 @@ class YUV360Canvas extends YUVCanvas { // Camera controls // ───────────────────────────────────────────────────────────────────────── - /** - * Attach mouse, touch, and wheel listeners to the canvas for interactive - * panning (drag) and FOV zoom (pinch / scroll wheel). - */ _initCameraControls() { const canvas = this.canvasElement; - const DRAG_SENS = 0.003; // radians per pixel — adjust to taste - - let dragging = false; - let lastX = 0; - let lastY = 0; + const DRAG_SENS = 0.003; - const onDragStart = (x, y) => { - dragging = true; - lastX = x; - lastY = y; - }; + let dragging = false; + let lastX = 0; + let lastY = 0; - const onDragMove = (x, y) => { + const onDragStart = (x, y) => { dragging = true; lastX = x; lastY = y; }; + const onDragMove = (x, y) => { if (!dragging) return; const dx = x - lastX; const dy = y - lastY; lastX = x; lastY = y; - - this.yaw += dx * DRAG_SENS; - // Clamp pitch to ±90° so the view never flips past a pole. - this.pitch = Math.max( - -Math.PI / 2, - Math.min(Math.PI / 2, this.pitch + dy * DRAG_SENS) - ); + this.yaw += dx * DRAG_SENS; + this.pitch = Math.max(-Math.PI / 2, + Math.min(Math.PI / 2, this.pitch + dy * DRAG_SENS)); }; - const onDragEnd = () => { dragging = false; }; - // ── Mouse ───────────────────────────────────────────────────────────── canvas.addEventListener('mousedown', (e) => onDragStart(e.clientX, e.clientY)); window.addEventListener('mousemove', (e) => onDragMove(e.clientX, e.clientY)); window.addEventListener('mouseup', onDragEnd); - // ── Scroll-wheel FOV zoom ───────────────────────────────────────────── canvas.addEventListener('wheel', (e) => { e.preventDefault(); this.fovDeg = Math.max(30, Math.min(120, this.fovDeg + e.deltaY * 0.05)); }, { passive: false }); - // ── Touch drag ──────────────────────────────────────────────────────── canvas.addEventListener('touchstart', (e) => { - if (e.touches.length === 1) { - e.preventDefault(); - onDragStart(e.touches[0].clientX, e.touches[0].clientY); - } + if (e.touches.length === 1) { e.preventDefault(); onDragStart(e.touches[0].clientX, e.touches[0].clientY); } }, { passive: false }); - window.addEventListener('touchmove', (e) => { - if (e.touches.length === 1) { - e.preventDefault(); - onDragMove(e.touches[0].clientX, e.touches[0].clientY); - } + if (e.touches.length === 1) { e.preventDefault(); onDragMove(e.touches[0].clientX, e.touches[0].clientY); } }, { passive: false }); - window.addEventListener('touchend', onDragEnd); - // ── Pinch-to-zoom FOV ───────────────────────────────────────────────── let lastPinchDist = null; - canvas.addEventListener('touchstart', (e) => { - if (e.touches.length === 2) { - lastPinchDist = this._pinchDist(e.touches); - } + if (e.touches.length === 2) lastPinchDist = this._pinchDist(e.touches); }, { passive: true }); - canvas.addEventListener('touchmove', (e) => { if (e.touches.length === 2 && lastPinchDist !== null) { const dist = this._pinchDist(e.touches); - this.fovDeg = Math.max(30, Math.min(120, - this.fovDeg - (dist - lastPinchDist) * 0.1 - )); + this.fovDeg = Math.max(30, Math.min(120, this.fovDeg - (dist - lastPinchDist) * 0.1)); lastPinchDist = dist; } }, { passive: true }); - - canvas.addEventListener('touchend', () => { lastPinchDist = null; }); + canvas.addEventListener('touchend', () => { lastPinchDist = null; }); canvas.addEventListener('touchcancel', () => { lastPinchDist = null; }); } @@ -503,89 +554,41 @@ class YUV360Canvas extends YUVCanvas { } // ───────────────────────────────────────────────────────────────────────── - // Matrix helpers (column-major, matching WebGL / GLSL convention) - // - // In column-major layout, element at (row, col) lives at index col*4+row. - // Matrix multiplication out = a·b is: out[row,col] = Σ_k a[row,k] · b[k,col] + // Matrix helpers // ───────────────────────────────────────────────────────────────────────── - /** - * Standard perspective projection matrix. - * @param {number} fov Vertical field of view in radians - * @param {number} aspect Viewport width / height - * @param {number} near Near clip plane distance - * @param {number} far Far clip plane distance - * @returns {Float32Array} Column-major 4×4 - */ _mat4Perspective(fov, aspect, near, far) { const f = 1 / Math.tan(fov / 2); const nf = 1 / (near - far); - // Columns listed left-to-right; rows listed top-to-bottom within each column. return new Float32Array([ - f / aspect, 0, 0, 0, // col 0 - 0, f, 0, 0, // col 1 - 0, 0, (far + near) * nf, -1, // col 2 - 0, 0, 2 * far * near * nf, 0, // col 3 + f / aspect, 0, 0, 0, + 0, f, 0, 0, + 0, 0, (far + near) * nf, -1, + 0, 0, 2*far*near*nf, 0, ]); } - /** - * Rotation around the Y axis (yaw). - * @param {number} a Angle in radians - * @returns {Float32Array} Column-major 4×4 - */ _mat4RotY(a) { - const c = Math.cos(a); - const s = Math.sin(a); - return new Float32Array([ - c, 0, -s, 0, // col 0 - 0, 1, 0, 0, // col 1 - s, 0, c, 0, // col 2 - 0, 0, 0, 1, // col 3 - ]); + const c = Math.cos(a), s = Math.sin(a); + return new Float32Array([ c, 0, -s, 0, 0, 1, 0, 0, s, 0, c, 0, 0, 0, 0, 1 ]); } - /** - * Rotation around the X axis (pitch). - * @param {number} a Angle in radians - * @returns {Float32Array} Column-major 4×4 - */ _mat4RotX(a) { - const c = Math.cos(a); - const s = Math.sin(a); - return new Float32Array([ - 1, 0, 0, 0, // col 0 - 0, c, s, 0, // col 1 - 0, -s, c, 0, // col 2 - 0, 0, 0, 1, // col 3 - ]); + const c = Math.cos(a), s = Math.sin(a); + return new Float32Array([ 1, 0, 0, 0, 0, c, s, 0, 0, -s, c, 0, 0, 0, 0, 1 ]); } - /** - * Multiply two column-major 4×4 matrices. Returns a · b. - * In GL terms: "first apply b, then apply a". - * @param {Float32Array} a - * @param {Float32Array} b - * @returns {Float32Array} Column-major 4×4 - */ _mat4Mul(a, b) { const out = new Float32Array(16); - for (let col = 0; col < 4; col++) { + for (let col = 0; col < 4; col++) for (let row = 0; row < 4; row++) { let sum = 0; - for (let k = 0; k < 4; k++) { - sum += a[k * 4 + row] * b[col * 4 + k]; - } - out[col * 4 + row] = sum; + for (let k = 0; k < 4; k++) sum += a[k*4+row] * b[col*4+k]; + out[col*4+row] = sum; } - } return out; } - // ───────────────────────────────────────────────────────────────────────── - // Shader compile helper - // ───────────────────────────────────────────────────────────────────────── - _compileShader(type, src) { const gl = this.contextGL; const shader = gl.createShader(type); @@ -593,7 +596,7 @@ class YUV360Canvas extends YUVCanvas { gl.compileShader(shader); if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { const label = type === gl.VERTEX_SHADER ? 'Vertex' : 'Fragment'; - console.error(`EquirectangularSphereCanvas: ${label} shader compile error —`, + console.error(`YUV360Canvas: ${label} shader compile error —`, gl.getShaderInfoLog(shader)); } return shader; diff --git a/src/lib/VisualizationHelpers.ts b/src/lib/VisualizationHelpers.ts index 8b60be0a..3d5fca55 100644 --- a/src/lib/VisualizationHelpers.ts +++ b/src/lib/VisualizationHelpers.ts @@ -97,7 +97,7 @@ export interface IVideoViewProperties extends DataViewProperties { showTime: boolean; showStats: boolean; useWebCodecApi: boolean; - is360: boolean; + props360: Object; width: number; height: number; } diff --git a/src/modules/visualization/visualizations/video/Builder.ts b/src/modules/visualization/visualizations/video/Builder.ts index 554a3a31..7b71fd31 100644 --- a/src/modules/visualization/visualizations/video/Builder.ts +++ b/src/modules/visualization/visualizations/video/Builder.ts @@ -88,7 +88,7 @@ export function CreateVideoViewProps( useWebCodecApi: true, showTime: visOptions?.time, showStats: visOptions?.stats, - is360: visOptions?.is360, + props360: visOptions?.props360, }; for (const [dsId, entry] of Object.entries(datastreams)) { diff --git a/src/modules/visualization/wizard/customizations/VideoOptions.vue b/src/modules/visualization/wizard/customizations/VideoOptions.vue index 635040b2..8e91272b 100644 --- a/src/modules/visualization/wizard/customizations/VideoOptions.vue +++ b/src/modules/visualization/wizard/customizations/VideoOptions.vue @@ -1,12 +1,34 @@ @@ -60,4 +103,20 @@ onMounted(() => { v-model="is360" label="Enable 360 View" /> + + From 2723bd1cab85e0438780c6b93d10c4a5b6318b3a Mon Sep 17 00:00:00 2001 From: kyle-fitzp Date: Tue, 16 Jun 2026 13:07:28 -0500 Subject: [PATCH 3/3] Viewer v3 compatibility --- .../visualization/wizard/customizations/VideoOptions.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/modules/visualization/wizard/customizations/VideoOptions.vue b/src/modules/visualization/wizard/customizations/VideoOptions.vue index 79a370c8..fe18756b 100644 --- a/src/modules/visualization/wizard/customizations/VideoOptions.vue +++ b/src/modules/visualization/wizard/customizations/VideoOptions.vue @@ -1,5 +1,5 @@