Implement WebXR Raw Camera Access for the ARKit backend

Dependencies/xr/Include/XR.h

@@ -248,6 +248,12 @@ namespace xr
                     void* DepthTexturePointer{};
                     Size DepthTextureSize;
 
+                    // WebXR raw camera access: the platform camera image for this
+                    // view, rendered into an RGBA texture each frame. Null when unavailable.
+                    // Currently only populated by the ARKit backend.
+                    void* CameraTexturePointer{};
+                    Size CameraTextureSize;
+
                     float DepthNearZ{};
                     float DepthFarZ{};
 

Dependencies/xr/Source/ARKit/XR.mm

@@ -771,6 +771,12 @@ bool TryInitialize() {
                 ActiveFrameViews[0].DepthTexturePointer = nil;
             }
 
+            if (ActiveFrameViews[0].CameraTexturePointer != nil) {
+                id<MTLTexture> oldCameraTexture = (__bridge_transfer id<MTLTexture>)ActiveFrameViews[0].CameraTexturePointer;
+                [oldCameraTexture setPurgeableState:MTLPurgeableStateEmpty];
+                ActiveFrameViews[0].CameraTexturePointer = nil;
+            }
+
             Planes.clear();
             Meshes.clear();
             CleanupAnchor(nil);
@@ -885,6 +891,27 @@ void UpdateXRView(MTKView* activeXRView) {
                     ActiveFrameViews[0].DepthTextureFormat = TextureFormat::D24S8;
                     ActiveFrameViews[0].DepthTextureSize = {width, height};
                 }
+
+                // WebXR raw camera access: allocate the standalone camera texture.
+                // The camera image is also pre-composited into the color texture above, but that
+                // texture is the scene render target, so it cannot double as a sampled camera
+                // image. This dedicated texture receives its own YUV->RGB pass below.
+                {
+                    if (ActiveFrameViews[0].CameraTexturePointer != nil) {
+                        id<MTLTexture> oldCameraTexture = (__bridge_transfer id<MTLTexture>)ActiveFrameViews[0].CameraTexturePointer;
+                        deletedTextureAsyncCallback(ActiveFrameViews[0].CameraTexturePointer).then(arcana::inline_scheduler, arcana::cancellation::none(), [oldCameraTexture]() {
+                            [oldCameraTexture setPurgeableState:MTLPurgeableStateEmpty];
+                        });
+                        ActiveFrameViews[0].CameraTexturePointer = nil;
+                    }
+
+                    MTLTextureDescriptor *textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:width height:height mipmapped:NO];
+                    textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
+                    id<MTLTexture> texture = [metalDevice newTextureWithDescriptor:textureDescriptor];
+
+                    ActiveFrameViews[0].CameraTexturePointer = (__bridge_retained void*)texture;
+                    ActiveFrameViews[0].CameraTextureSize = {width, height};
+                }
             }
 
             // Draw the camera texture to the color texture and clear the depth texture before handing them off to Babylon.
@@ -945,6 +972,28 @@ void UpdateXRView(MTKView* activeXRView) {
                 }];
             }
 
+            // WebXR raw camera access: render the camera image into its own
+            // sampled texture. Reuses the shared shader via screenPipelineState (which has
+            // no depth/stencil attachments); the babylonTexture slot (0) is intentionally
+            // left unbound so the shader takes the camera YUV->RGB branch.
+            if (ActiveFrameViews[0].CameraTexturePointer != nil && cameraTextureY != nil && cameraTextureCbCr != nil) {
+                MTLRenderPassDescriptor *cameraPassDescriptor = [MTLRenderPassDescriptor renderPassDescriptor];
+                if (cameraPassDescriptor != nil) {
+                    cameraPassDescriptor.colorAttachments[0].texture = (__bridge id<MTLTexture>)ActiveFrameViews[0].CameraTexturePointer;
+                    cameraPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionDontCare;
+                    cameraPassDescriptor.colorAttachments[0].storeAction = MTLStoreActionStore;
+
+                    id<MTLRenderCommandEncoder> cameraEncoder = [currentCommandBuffer renderCommandEncoderWithDescriptor:cameraPassDescriptor];
+                    cameraEncoder.label = @"XRRawCameraAccessEncoder";
+                    [cameraEncoder setRenderPipelineState:screenPipelineState];
+                    [cameraEncoder setVertexBytes:vertices length:sizeof(vertices) atIndex:0];
+                    [cameraEncoder setFragmentTexture:cameraTextureY atIndex:1];
+                    [cameraEncoder setFragmentTexture:cameraTextureCbCr atIndex:2];
+                    [cameraEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
+                    [cameraEncoder endEncoding];
+                }
+            }
+
             // Finalize rendering here & push the command buffer to the GPU.
             [currentCommandBuffer commit];
 

Plugins/NativeXr/Source/NativeXr.cpp

@@ -11,6 +11,7 @@
 
 #include "Constants.h"
 #include "PointerEvent.h"
+#include "XRCamera.h"
 #include "XRWebGLBinding.h"
 #include "XRWebGLLayer.h"
 #include "XRRigidTransform.h"
@@ -52,6 +53,7 @@ namespace Babylon
 
             PointerEvent::Initialize(env);
 
+            XRCamera::Initialize(env);
             XRWebGLBinding::Initialize(env);
             XRWebGLLayer::Initialize(env);
             XRRigidTransform::Initialize(env);

Plugins/NativeXr/Source/XRCamera.h

@@ -0,0 +1,146 @@
+#pragma once
+
+// WebXR raw camera access: JS XRCamera object, per the WebXR Raw Camera
+// Access module (https://immersive-web.github.io/raw-camera-access/). Exposed via
+// XRView.camera; the camera image is retrieved with XRWebGLBinding.getCameraImage,
+// which returns a Napi::Pointer<Graphics::Texture> — the same object type
+// NativeEngine uses as an InternalTexture's underlyingResource, so Babylon.js can
+// bind it as a regular sampler.
+//
+// The texture pointer originates from the xr backend (View::CameraTexturePointer, an
+// MTLTexture on ARKit) and is wired to bgfx with the same createTexture2D +
+// overrideInternal pattern used by NativeCamera's CameraDevice.
+
+#include <Babylon/Graphics/DeviceContext.h>
+#include <Babylon/Graphics/Texture.h>
+#include <napi/pointer.h>
+#include <arcana/threading/task.h>
+#include <bgfx/bgfx.h>
+
+#include <atomic>
+#include <memory>
+
+namespace Babylon
+{
+    class XRCamera : public Napi::ObjectWrap<XRCamera>
+    {
+        static constexpr auto JS_CLASS_NAME = "XRCamera";
+
+    public:
+        static void Initialize(Napi::Env env)
+        {
+            Napi::HandleScope scope{env};
+
+            Napi::Function func = DefineClass(
+                env,
+                JS_CLASS_NAME,
+                {
+                    InstanceAccessor("width", &XRCamera::GetWidth, nullptr),
+                    InstanceAccessor("height", &XRCamera::GetHeight, nullptr),
+                });
+
+            env.Global().Set(JS_CLASS_NAME, func);
+        }
+
+        static Napi::Object New(const Napi::CallbackInfo& info)
+        {
+            return info.Env().Global().Get(JS_CLASS_NAME).As<Napi::Function>().New({});
+        }
+
+        XRCamera(const Napi::CallbackInfo& info)
+            : Napi::ObjectWrap<XRCamera>{info}
+            , m_deviceContext{&Graphics::DeviceContext::GetFromJavaScript(info.Env())}
+        {
+        }
+
+        // Called per getViewerPose from XRView::Update with the current platform camera
+        // texture (MTLTexture on ARKit). Creates/refreshes the bgfx-wrapped texture when
+        // the platform texture identity or size changes.
+        void Update(Napi::Env env, void* texturePointer, size_t width, size_t height)
+        {
+            if (texturePointer == nullptr || width == 0 || height == 0)
+            {
+                m_hasTexture = false;
+                return;
+            }
+
+            if (texturePointer != m_texturePointer || width != m_width || height != m_height)
+            {
+                m_texturePointer = texturePointer;
+                m_width = width;
+                m_height = height;
+
+                // Create a bgfx texture to be overridden with the platform texture. The
+                // Graphics::Texture (which owns the bgfx handle) is owned by the JS pointer
+                // object below; replacing the persistent ref releases the old one via GC.
+                const auto bgfxHandle = bgfx::createTexture2D(
+                    static_cast<uint16_t>(width), static_cast<uint16_t>(height), false, 1,
+                    bgfx::TextureFormat::BGRA8, BGFX_TEXTURE_NONE | BGFX_SAMPLER_NONE);
+
+                auto* texture = new Graphics::Texture(*m_deviceContext);
+                texture->Attach(bgfxHandle, true, static_cast<uint16_t>(width), static_cast<uint16_t>(height),
+                    false, 1, bgfx::TextureFormat::BGRA8, BGFX_TEXTURE_NONE | BGFX_SAMPLER_NONE);
+
+                m_textureJs = Napi::Persistent(Napi::Pointer<Graphics::Texture>::Create(env, texture, Napi::NapiPointerDeleter(texture)));
+
+                // New handle needs a (re)override.
+                m_overrideState = std::make_shared<OverrideState>();
+                m_bgfxHandle = bgfxHandle;
+            }
+
+            // bgfx::overrideInternal only succeeds once the handle has been initialized by a
+            // bgfx frame, so run it on the render thread after rendering and retry until it
+            // sticks (same approach as NativeCamera's CameraDevice::UpdateCameraTexture).
+            if (!m_overrideState->done.load() && !m_overrideState->pending.exchange(true))
+            {
+                arcana::make_task(m_deviceContext->AfterRenderScheduler(), arcana::cancellation::none(),
+                    [state = m_overrideState, handle = m_bgfxHandle, pointer = m_texturePointer]() {
+                        state->done.store(bgfx::overrideInternal(handle, reinterpret_cast<uintptr_t>(pointer)) != 0);
+                        state->pending.store(false);
+                    });
+            }
+
+            m_hasTexture = true;
+        }
+
+        bool HasTexture() const
+        {
+            return m_hasTexture && !m_textureJs.IsEmpty();
+        }
+
+        Napi::Value GetCameraImageValue(Napi::Env env)
+        {
+            if (!HasTexture())
+            {
+                return env.Undefined();
+            }
+            return m_textureJs.Value();
+        }
+
+    private:
+        struct OverrideState
+        {
+            std::atomic<bool> done{false};
+            std::atomic<bool> pending{false};
+        };
+
+        Graphics::DeviceContext* m_deviceContext{};
+        void* m_texturePointer{};
+        size_t m_width{};
+        size_t m_height{};
+        bool m_hasTexture{};
+        bgfx::TextureHandle m_bgfxHandle{bgfx::kInvalidHandle};
+        std::shared_ptr<OverrideState> m_overrideState{std::make_shared<OverrideState>()};
+        Napi::Reference<Napi::Value> m_textureJs{};
+
+        Napi::Value GetWidth(const Napi::CallbackInfo& info)
+        {
+            return Napi::Value::From(info.Env(), static_cast<uint32_t>(m_width));
+        }
+
+        Napi::Value GetHeight(const Napi::CallbackInfo& info)
+        {
+            return Napi::Value::From(info.Env(), static_cast<uint32_t>(m_height));
+        }
+    };
+} // Babylon

Plugins/NativeXr/Source/XRView.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "XRCamera.h"
+
 namespace Babylon
 {
     class XRView : public Napi::ObjectWrap<XRView>
@@ -20,6 +22,7 @@ namespace Babylon
                     InstanceAccessor("projectionMatrix", &XRView::GetProjectionMatrix, nullptr),
                     InstanceAccessor("transform", &XRView::GetTransform, nullptr),
                     InstanceAccessor("isFirstPersonObserver", &XRView::IsFirstPersonObserver, nullptr),
+                    InstanceAccessor("camera", &XRView::GetCamera, nullptr),
                 });
 
             env.Global().Set(JS_CLASS_NAME, func);
@@ -40,7 +43,8 @@ namespace Babylon
         {
         }
 
-        void Update(size_t eyeIdx, gsl::span<const float, 16> projectionMatrix, const xr::Space& space, bool isFirstPersonObserver)
+        void Update(const Napi::CallbackInfo& info, size_t eyeIdx, gsl::span<const float, 16> projectionMatrix, const xr::Space& space, bool isFirstPersonObserver,
+            void* cameraTexturePointer, size_t cameraTextureWidth, size_t cameraTextureHeight)
         {
             if (eyeIdx != m_eyeIdx)
             {
@@ -53,6 +57,21 @@ namespace Babylon
             XRRigidTransform::Unwrap(m_rigidTransform.Value())->Update(space, false);
 
             m_isFirstPersonObserver = isFirstPersonObserver;
+
+            // WebXR raw camera access: keep the per-view XRCamera in sync with the
+            // platform camera texture (null on platforms/backends that don't provide one).
+            if (cameraTexturePointer != nullptr)
+            {
+                if (m_camera.IsEmpty())
+                {
+                    m_camera = Napi::Persistent(XRCamera::New(info));
+                }
+                XRCamera::Unwrap(m_camera.Value())->Update(info.Env(), cameraTexturePointer, cameraTextureWidth, cameraTextureHeight);
+            }
+            else if (!m_camera.IsEmpty())
+            {
+                XRCamera::Unwrap(m_camera.Value())->Update(info.Env(), nullptr, 0, 0);
+            }
         }
 
     private:
@@ -61,6 +80,7 @@ namespace Babylon
         Napi::Reference<Napi::Float32Array> m_projectionMatrix{};
         Napi::ObjectReference m_rigidTransform{};
         bool m_isFirstPersonObserver{};
+        Napi::ObjectReference m_camera{};
 
         Napi::Value GetEye(const Napi::CallbackInfo& info)
         {
@@ -81,5 +101,14 @@ namespace Babylon
         {
             return Napi::Boolean::From(info.Env(), m_isFirstPersonObserver);
         }
+
+        Napi::Value GetCamera(const Napi::CallbackInfo& info)
+        {
+            if (m_camera.IsEmpty() || !XRCamera::Unwrap(m_camera.Value())->HasTexture())
+            {
+                return info.Env().Undefined();
+            }
+            return m_camera.Value();
+        }
     };
 } // Babylon

Plugins/NativeXr/Source/XRViewerPose.h

@@ -77,7 +77,8 @@ namespace Babylon
             for (uint32_t idx = 0; idx < static_cast<uint32_t>(frame.Views.size()); ++idx)
             {
                 const auto& view = frame.Views[idx];
-                m_views[idx]->Update(idx, view.ProjectionMatrix, view.Space, view.IsFirstPersonObserver);
+                m_views[idx]->Update(info, idx, view.ProjectionMatrix, view.Space, view.IsFirstPersonObserver,
+                    view.CameraTexturePointer, view.CameraTextureSize.Width, view.CameraTextureSize.Height);
             }
 
             // Check the frame to see if it has valid tracking, if it does not then the position should

Plugins/NativeXr/Source/XRWebGLBinding.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "XRCamera.h"
+
 namespace Babylon
 {
     class XRWebGLBinding : public Napi::ObjectWrap<XRWebGLBinding>
@@ -14,7 +16,9 @@ namespace Babylon
             Napi::Function func = DefineClass(
                 env,
                 JS_CLASS_NAME,
-                {});
+                {
+                    InstanceMethod("getCameraImage", &XRWebGLBinding::GetCameraImage),
+                });
 
             env.Global().Set(JS_CLASS_NAME, func);
         }
@@ -28,5 +32,25 @@ namespace Babylon
             : Napi::ObjectWrap<XRWebGLBinding>{info}
         {
         }
+
+    private:
+        // WebXR raw camera access: returns the camera image for the given
+        // XRCamera as a Napi::Pointer<Graphics::Texture> — the object type NativeEngine
+        // uses as a hardware texture's underlyingResource, so Babylon.js can sample it.
+        Napi::Value GetCameraImage(const Napi::CallbackInfo& info)
+        {
+            if (info.Length() < 1 || !info[0].IsObject())
+            {
+                return info.Env().Undefined();
+            }
+
+            XRCamera* camera = XRCamera::Unwrap(info[0].As<Napi::Object>());
+            if (camera == nullptr)
+            {
+                return info.Env().Undefined();
+            }
+
+            return camera->GetCameraImageValue(info.Env());
+        }
     };
 } // Babylon