From 1d0595417c6d7bab5061c2cf80d79983b71b94de Mon Sep 17 00:00:00 2001 From: lukasz karluk Date: Tue, 7 Jul 2026 15:49:44 +1000 Subject: [PATCH 1/2] xr: expose the platform camera image as a per-view RGBA texture (ARKit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds CameraTexturePointer/CameraTextureSize to System::Session::Frame::View so xr backends can surface the platform camera image alongside the color/depth render textures. Null on backends that do not populate it. ARKit implementation: the camera image was already converted YUV->RGB each frame, but only pre-composited into the color texture — which is the scene render target and therefore cannot double as a sampled camera image. Allocate a dedicated BGRA8 texture (recreated on viewport size change, released with the same deleted-texture callback used by the color/depth textures) and render a second camera pass into it, reusing the shared shader via screenPipelineState (no depth/stencil attachments) with the babylonTexture slot left unbound so the shader takes the camera YUV->RGB branch. This is the backend half of WebXR Raw Camera Access support (https://immersive-web.github.io/raw-camera-access/). Co-Authored-By: Claude Fable 5 --- Dependencies/xr/Include/XR.h | 6 ++++ Dependencies/xr/Source/ARKit/XR.mm | 49 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/Dependencies/xr/Include/XR.h b/Dependencies/xr/Include/XR.h index 708d702db..ce7577542 100644 --- a/Dependencies/xr/Include/XR.h +++ b/Dependencies/xr/Include/XR.h @@ -248,6 +248,12 @@ namespace xr void* DepthTexturePointer{}; Size DepthTextureSize; + // WebXR raw camera access: the platform camera image for this + // view, rendered into an RGBA texture each frame. Null when unavailable. + // Currently only populated by the ARKit backend. + void* CameraTexturePointer{}; + Size CameraTextureSize; + float DepthNearZ{}; float DepthFarZ{}; diff --git a/Dependencies/xr/Source/ARKit/XR.mm b/Dependencies/xr/Source/ARKit/XR.mm index 82a8ed6c9..6d5683f19 100644 --- a/Dependencies/xr/Source/ARKit/XR.mm +++ b/Dependencies/xr/Source/ARKit/XR.mm @@ -771,6 +771,12 @@ bool TryInitialize() { ActiveFrameViews[0].DepthTexturePointer = nil; } + if (ActiveFrameViews[0].CameraTexturePointer != nil) { + id oldCameraTexture = (__bridge_transfer id)ActiveFrameViews[0].CameraTexturePointer; + [oldCameraTexture setPurgeableState:MTLPurgeableStateEmpty]; + ActiveFrameViews[0].CameraTexturePointer = nil; + } + Planes.clear(); Meshes.clear(); CleanupAnchor(nil); @@ -885,6 +891,27 @@ void UpdateXRView(MTKView* activeXRView) { ActiveFrameViews[0].DepthTextureFormat = TextureFormat::D24S8; ActiveFrameViews[0].DepthTextureSize = {width, height}; } + + // WebXR raw camera access: allocate the standalone camera texture. + // The camera image is also pre-composited into the color texture above, but that + // texture is the scene render target, so it cannot double as a sampled camera + // image. This dedicated texture receives its own YUV->RGB pass below. + { + if (ActiveFrameViews[0].CameraTexturePointer != nil) { + id oldCameraTexture = (__bridge_transfer id)ActiveFrameViews[0].CameraTexturePointer; + deletedTextureAsyncCallback(ActiveFrameViews[0].CameraTexturePointer).then(arcana::inline_scheduler, arcana::cancellation::none(), [oldCameraTexture]() { + [oldCameraTexture setPurgeableState:MTLPurgeableStateEmpty]; + }); + ActiveFrameViews[0].CameraTexturePointer = nil; + } + + MTLTextureDescriptor *textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:width height:height mipmapped:NO]; + textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + id texture = [metalDevice newTextureWithDescriptor:textureDescriptor]; + + ActiveFrameViews[0].CameraTexturePointer = (__bridge_retained void*)texture; + ActiveFrameViews[0].CameraTextureSize = {width, height}; + } } // Draw the camera texture to the color texture and clear the depth texture before handing them off to Babylon. @@ -945,6 +972,28 @@ void UpdateXRView(MTKView* activeXRView) { }]; } + // WebXR raw camera access: render the camera image into its own + // sampled texture. Reuses the shared shader via screenPipelineState (which has + // no depth/stencil attachments); the babylonTexture slot (0) is intentionally + // left unbound so the shader takes the camera YUV->RGB branch. + if (ActiveFrameViews[0].CameraTexturePointer != nil && cameraTextureY != nil && cameraTextureCbCr != nil) { + MTLRenderPassDescriptor *cameraPassDescriptor = [MTLRenderPassDescriptor renderPassDescriptor]; + if (cameraPassDescriptor != nil) { + cameraPassDescriptor.colorAttachments[0].texture = (__bridge id)ActiveFrameViews[0].CameraTexturePointer; + cameraPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionDontCare; + cameraPassDescriptor.colorAttachments[0].storeAction = MTLStoreActionStore; + + id cameraEncoder = [currentCommandBuffer renderCommandEncoderWithDescriptor:cameraPassDescriptor]; + cameraEncoder.label = @"XRRawCameraAccessEncoder"; + [cameraEncoder setRenderPipelineState:screenPipelineState]; + [cameraEncoder setVertexBytes:vertices length:sizeof(vertices) atIndex:0]; + [cameraEncoder setFragmentTexture:cameraTextureY atIndex:1]; + [cameraEncoder setFragmentTexture:cameraTextureCbCr atIndex:2]; + [cameraEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; + [cameraEncoder endEncoding]; + } + } + // Finalize rendering here & push the command buffer to the GPU. [currentCommandBuffer commit]; From 67384ae8919ab34cd71ba3c1b8305099a1a6bfb9 Mon Sep 17 00:00:00 2001 From: lukasz karluk Date: Tue, 7 Jul 2026 15:49:44 +1000 Subject: [PATCH 2/2] NativeXr: implement WebXR Raw Camera Access (XRView.camera + getCameraImage) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the JS surface of the WebXR Raw Camera Access module (https://immersive-web.github.io/raw-camera-access/) on top of the per-view camera texture provided by the xr backend: - XRCamera (new): exposes width/height and wraps the platform camera texture (MTLTexture on ARKit) for bgfx using the same createTexture2D + overrideInternal-on-render-thread retry pattern as NativeCamera's CameraDevice. The bgfx handle is owned by a Graphics::Texture whose JS lifetime is a Napi::Pointer external. - XRView.camera: returns the per-view XRCamera (undefined when the backend provides no camera texture). XRViewerPose threads the camera texture pointer/size from the frame views into XRView::Update. - XRWebGLBinding.getCameraImage(camera): returns the camera image as a Napi::Pointer — the object type NativeEngine uses as a hardware texture's underlyingResource — so Babylon.js's WebXRRawCameraAccess feature can wrap and sample it without modification. Verified on device (iPhone 15 Pro Max, iOS 26.5): Babylon.js 9.9.1's RAW_CAMERA_ACCESS feature attaches, onTexturesUpdatedObservable fires, and the live ARKit camera image renders on scene geometry inside an immersive-ar session at 60 FPS. Co-Authored-By: Claude Fable 5 --- Plugins/NativeXr/Source/NativeXr.cpp | 2 + Plugins/NativeXr/Source/XRCamera.h | 146 +++++++++++++++++++++++ Plugins/NativeXr/Source/XRView.h | 31 ++++- Plugins/NativeXr/Source/XRViewerPose.h | 3 +- Plugins/NativeXr/Source/XRWebGLBinding.h | 26 +++- 5 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 Plugins/NativeXr/Source/XRCamera.h diff --git a/Plugins/NativeXr/Source/NativeXr.cpp b/Plugins/NativeXr/Source/NativeXr.cpp index cacb5c00b..f928b517f 100644 --- a/Plugins/NativeXr/Source/NativeXr.cpp +++ b/Plugins/NativeXr/Source/NativeXr.cpp @@ -11,6 +11,7 @@ #include "Constants.h" #include "PointerEvent.h" +#include "XRCamera.h" #include "XRWebGLBinding.h" #include "XRWebGLLayer.h" #include "XRRigidTransform.h" @@ -52,6 +53,7 @@ namespace Babylon PointerEvent::Initialize(env); + XRCamera::Initialize(env); XRWebGLBinding::Initialize(env); XRWebGLLayer::Initialize(env); XRRigidTransform::Initialize(env); diff --git a/Plugins/NativeXr/Source/XRCamera.h b/Plugins/NativeXr/Source/XRCamera.h new file mode 100644 index 000000000..dc1c56cc7 --- /dev/null +++ b/Plugins/NativeXr/Source/XRCamera.h @@ -0,0 +1,146 @@ +#pragma once + +// WebXR raw camera access: JS XRCamera object, per the WebXR Raw Camera +// Access module (https://immersive-web.github.io/raw-camera-access/). Exposed via +// XRView.camera; the camera image is retrieved with XRWebGLBinding.getCameraImage, +// which returns a Napi::Pointer — the same object type +// NativeEngine uses as an InternalTexture's underlyingResource, so Babylon.js can +// bind it as a regular sampler. +// +// The texture pointer originates from the xr backend (View::CameraTexturePointer, an +// MTLTexture on ARKit) and is wired to bgfx with the same createTexture2D + +// overrideInternal pattern used by NativeCamera's CameraDevice. + +#include +#include +#include +#include +#include + +#include +#include + +namespace Babylon +{ + class XRCamera : public Napi::ObjectWrap + { + static constexpr auto JS_CLASS_NAME = "XRCamera"; + + public: + static void Initialize(Napi::Env env) + { + Napi::HandleScope scope{env}; + + Napi::Function func = DefineClass( + env, + JS_CLASS_NAME, + { + InstanceAccessor("width", &XRCamera::GetWidth, nullptr), + InstanceAccessor("height", &XRCamera::GetHeight, nullptr), + }); + + env.Global().Set(JS_CLASS_NAME, func); + } + + static Napi::Object New(const Napi::CallbackInfo& info) + { + return info.Env().Global().Get(JS_CLASS_NAME).As().New({}); + } + + XRCamera(const Napi::CallbackInfo& info) + : Napi::ObjectWrap{info} + , m_deviceContext{&Graphics::DeviceContext::GetFromJavaScript(info.Env())} + { + } + + // Called per getViewerPose from XRView::Update with the current platform camera + // texture (MTLTexture on ARKit). Creates/refreshes the bgfx-wrapped texture when + // the platform texture identity or size changes. + void Update(Napi::Env env, void* texturePointer, size_t width, size_t height) + { + if (texturePointer == nullptr || width == 0 || height == 0) + { + m_hasTexture = false; + return; + } + + if (texturePointer != m_texturePointer || width != m_width || height != m_height) + { + m_texturePointer = texturePointer; + m_width = width; + m_height = height; + + // Create a bgfx texture to be overridden with the platform texture. The + // Graphics::Texture (which owns the bgfx handle) is owned by the JS pointer + // object below; replacing the persistent ref releases the old one via GC. + const auto bgfxHandle = bgfx::createTexture2D( + static_cast(width), static_cast(height), false, 1, + bgfx::TextureFormat::BGRA8, BGFX_TEXTURE_NONE | BGFX_SAMPLER_NONE); + + auto* texture = new Graphics::Texture(*m_deviceContext); + texture->Attach(bgfxHandle, true, static_cast(width), static_cast(height), + false, 1, bgfx::TextureFormat::BGRA8, BGFX_TEXTURE_NONE | BGFX_SAMPLER_NONE); + + m_textureJs = Napi::Persistent(Napi::Pointer::Create(env, texture, Napi::NapiPointerDeleter(texture))); + + // New handle needs a (re)override. + m_overrideState = std::make_shared(); + m_bgfxHandle = bgfxHandle; + } + + // bgfx::overrideInternal only succeeds once the handle has been initialized by a + // bgfx frame, so run it on the render thread after rendering and retry until it + // sticks (same approach as NativeCamera's CameraDevice::UpdateCameraTexture). + if (!m_overrideState->done.load() && !m_overrideState->pending.exchange(true)) + { + arcana::make_task(m_deviceContext->AfterRenderScheduler(), arcana::cancellation::none(), + [state = m_overrideState, handle = m_bgfxHandle, pointer = m_texturePointer]() { + state->done.store(bgfx::overrideInternal(handle, reinterpret_cast(pointer)) != 0); + state->pending.store(false); + }); + } + + m_hasTexture = true; + } + + bool HasTexture() const + { + return m_hasTexture && !m_textureJs.IsEmpty(); + } + + Napi::Value GetCameraImageValue(Napi::Env env) + { + if (!HasTexture()) + { + return env.Undefined(); + } + return m_textureJs.Value(); + } + + private: + struct OverrideState + { + std::atomic done{false}; + std::atomic pending{false}; + }; + + Graphics::DeviceContext* m_deviceContext{}; + void* m_texturePointer{}; + size_t m_width{}; + size_t m_height{}; + bool m_hasTexture{}; + bgfx::TextureHandle m_bgfxHandle{bgfx::kInvalidHandle}; + std::shared_ptr m_overrideState{std::make_shared()}; + Napi::Reference m_textureJs{}; + + Napi::Value GetWidth(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), static_cast(m_width)); + } + + Napi::Value GetHeight(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), static_cast(m_height)); + } + }; +} // Babylon diff --git a/Plugins/NativeXr/Source/XRView.h b/Plugins/NativeXr/Source/XRView.h index 50a653b5d..0783ac5cc 100644 --- a/Plugins/NativeXr/Source/XRView.h +++ b/Plugins/NativeXr/Source/XRView.h @@ -1,5 +1,7 @@ #pragma once +#include "XRCamera.h" + namespace Babylon { class XRView : public Napi::ObjectWrap @@ -20,6 +22,7 @@ namespace Babylon InstanceAccessor("projectionMatrix", &XRView::GetProjectionMatrix, nullptr), InstanceAccessor("transform", &XRView::GetTransform, nullptr), InstanceAccessor("isFirstPersonObserver", &XRView::IsFirstPersonObserver, nullptr), + InstanceAccessor("camera", &XRView::GetCamera, nullptr), }); env.Global().Set(JS_CLASS_NAME, func); @@ -40,7 +43,8 @@ namespace Babylon { } - void Update(size_t eyeIdx, gsl::span projectionMatrix, const xr::Space& space, bool isFirstPersonObserver) + void Update(const Napi::CallbackInfo& info, size_t eyeIdx, gsl::span projectionMatrix, const xr::Space& space, bool isFirstPersonObserver, + void* cameraTexturePointer, size_t cameraTextureWidth, size_t cameraTextureHeight) { if (eyeIdx != m_eyeIdx) { @@ -53,6 +57,21 @@ namespace Babylon XRRigidTransform::Unwrap(m_rigidTransform.Value())->Update(space, false); m_isFirstPersonObserver = isFirstPersonObserver; + + // WebXR raw camera access: keep the per-view XRCamera in sync with the + // platform camera texture (null on platforms/backends that don't provide one). + if (cameraTexturePointer != nullptr) + { + if (m_camera.IsEmpty()) + { + m_camera = Napi::Persistent(XRCamera::New(info)); + } + XRCamera::Unwrap(m_camera.Value())->Update(info.Env(), cameraTexturePointer, cameraTextureWidth, cameraTextureHeight); + } + else if (!m_camera.IsEmpty()) + { + XRCamera::Unwrap(m_camera.Value())->Update(info.Env(), nullptr, 0, 0); + } } private: @@ -61,6 +80,7 @@ namespace Babylon Napi::Reference m_projectionMatrix{}; Napi::ObjectReference m_rigidTransform{}; bool m_isFirstPersonObserver{}; + Napi::ObjectReference m_camera{}; Napi::Value GetEye(const Napi::CallbackInfo& info) { @@ -81,5 +101,14 @@ namespace Babylon { return Napi::Boolean::From(info.Env(), m_isFirstPersonObserver); } + + Napi::Value GetCamera(const Napi::CallbackInfo& info) + { + if (m_camera.IsEmpty() || !XRCamera::Unwrap(m_camera.Value())->HasTexture()) + { + return info.Env().Undefined(); + } + return m_camera.Value(); + } }; } // Babylon diff --git a/Plugins/NativeXr/Source/XRViewerPose.h b/Plugins/NativeXr/Source/XRViewerPose.h index 9cbceecf6..51ad92be8 100644 --- a/Plugins/NativeXr/Source/XRViewerPose.h +++ b/Plugins/NativeXr/Source/XRViewerPose.h @@ -77,7 +77,8 @@ namespace Babylon for (uint32_t idx = 0; idx < static_cast(frame.Views.size()); ++idx) { const auto& view = frame.Views[idx]; - m_views[idx]->Update(idx, view.ProjectionMatrix, view.Space, view.IsFirstPersonObserver); + m_views[idx]->Update(info, idx, view.ProjectionMatrix, view.Space, view.IsFirstPersonObserver, + view.CameraTexturePointer, view.CameraTextureSize.Width, view.CameraTextureSize.Height); } // Check the frame to see if it has valid tracking, if it does not then the position should diff --git a/Plugins/NativeXr/Source/XRWebGLBinding.h b/Plugins/NativeXr/Source/XRWebGLBinding.h index 48286f7b8..69fe61b71 100644 --- a/Plugins/NativeXr/Source/XRWebGLBinding.h +++ b/Plugins/NativeXr/Source/XRWebGLBinding.h @@ -1,5 +1,7 @@ #pragma once +#include "XRCamera.h" + namespace Babylon { class XRWebGLBinding : public Napi::ObjectWrap @@ -14,7 +16,9 @@ namespace Babylon Napi::Function func = DefineClass( env, JS_CLASS_NAME, - {}); + { + InstanceMethod("getCameraImage", &XRWebGLBinding::GetCameraImage), + }); env.Global().Set(JS_CLASS_NAME, func); } @@ -28,5 +32,25 @@ namespace Babylon : Napi::ObjectWrap{info} { } + + private: + // WebXR raw camera access: returns the camera image for the given + // XRCamera as a Napi::Pointer — the object type NativeEngine + // uses as a hardware texture's underlyingResource, so Babylon.js can sample it. + Napi::Value GetCameraImage(const Napi::CallbackInfo& info) + { + if (info.Length() < 1 || !info[0].IsObject()) + { + return info.Env().Undefined(); + } + + XRCamera* camera = XRCamera::Unwrap(info[0].As()); + if (camera == nullptr) + { + return info.Env().Undefined(); + } + + return camera->GetCameraImageValue(info.Env()); + } }; } // Babylon