diff --git a/Dependencies/xr/Include/XR.h b/Dependencies/xr/Include/XR.h index 708d702db..ce7577542 100644 --- a/Dependencies/xr/Include/XR.h +++ b/Dependencies/xr/Include/XR.h @@ -248,6 +248,12 @@ namespace xr void* DepthTexturePointer{}; Size DepthTextureSize; + // WebXR raw camera access: the platform camera image for this + // view, rendered into an RGBA texture each frame. Null when unavailable. + // Currently only populated by the ARKit backend. + void* CameraTexturePointer{}; + Size CameraTextureSize; + float DepthNearZ{}; float DepthFarZ{}; diff --git a/Dependencies/xr/Source/ARKit/XR.mm b/Dependencies/xr/Source/ARKit/XR.mm index 82a8ed6c9..6d5683f19 100644 --- a/Dependencies/xr/Source/ARKit/XR.mm +++ b/Dependencies/xr/Source/ARKit/XR.mm @@ -771,6 +771,12 @@ bool TryInitialize() { ActiveFrameViews[0].DepthTexturePointer = nil; } + if (ActiveFrameViews[0].CameraTexturePointer != nil) { + id oldCameraTexture = (__bridge_transfer id)ActiveFrameViews[0].CameraTexturePointer; + [oldCameraTexture setPurgeableState:MTLPurgeableStateEmpty]; + ActiveFrameViews[0].CameraTexturePointer = nil; + } + Planes.clear(); Meshes.clear(); CleanupAnchor(nil); @@ -885,6 +891,27 @@ void UpdateXRView(MTKView* activeXRView) { ActiveFrameViews[0].DepthTextureFormat = TextureFormat::D24S8; ActiveFrameViews[0].DepthTextureSize = {width, height}; } + + // WebXR raw camera access: allocate the standalone camera texture. + // The camera image is also pre-composited into the color texture above, but that + // texture is the scene render target, so it cannot double as a sampled camera + // image. This dedicated texture receives its own YUV->RGB pass below. + { + if (ActiveFrameViews[0].CameraTexturePointer != nil) { + id oldCameraTexture = (__bridge_transfer id)ActiveFrameViews[0].CameraTexturePointer; + deletedTextureAsyncCallback(ActiveFrameViews[0].CameraTexturePointer).then(arcana::inline_scheduler, arcana::cancellation::none(), [oldCameraTexture]() { + [oldCameraTexture setPurgeableState:MTLPurgeableStateEmpty]; + }); + ActiveFrameViews[0].CameraTexturePointer = nil; + } + + MTLTextureDescriptor *textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:width height:height mipmapped:NO]; + textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; + id texture = [metalDevice newTextureWithDescriptor:textureDescriptor]; + + ActiveFrameViews[0].CameraTexturePointer = (__bridge_retained void*)texture; + ActiveFrameViews[0].CameraTextureSize = {width, height}; + } } // Draw the camera texture to the color texture and clear the depth texture before handing them off to Babylon. @@ -945,6 +972,28 @@ void UpdateXRView(MTKView* activeXRView) { }]; } + // WebXR raw camera access: render the camera image into its own + // sampled texture. Reuses the shared shader via screenPipelineState (which has + // no depth/stencil attachments); the babylonTexture slot (0) is intentionally + // left unbound so the shader takes the camera YUV->RGB branch. + if (ActiveFrameViews[0].CameraTexturePointer != nil && cameraTextureY != nil && cameraTextureCbCr != nil) { + MTLRenderPassDescriptor *cameraPassDescriptor = [MTLRenderPassDescriptor renderPassDescriptor]; + if (cameraPassDescriptor != nil) { + cameraPassDescriptor.colorAttachments[0].texture = (__bridge id)ActiveFrameViews[0].CameraTexturePointer; + cameraPassDescriptor.colorAttachments[0].loadAction = MTLLoadActionDontCare; + cameraPassDescriptor.colorAttachments[0].storeAction = MTLStoreActionStore; + + id cameraEncoder = [currentCommandBuffer renderCommandEncoderWithDescriptor:cameraPassDescriptor]; + cameraEncoder.label = @"XRRawCameraAccessEncoder"; + [cameraEncoder setRenderPipelineState:screenPipelineState]; + [cameraEncoder setVertexBytes:vertices length:sizeof(vertices) atIndex:0]; + [cameraEncoder setFragmentTexture:cameraTextureY atIndex:1]; + [cameraEncoder setFragmentTexture:cameraTextureCbCr atIndex:2]; + [cameraEncoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4]; + [cameraEncoder endEncoding]; + } + } + // Finalize rendering here & push the command buffer to the GPU. [currentCommandBuffer commit]; diff --git a/Plugins/NativeXr/Source/NativeXr.cpp b/Plugins/NativeXr/Source/NativeXr.cpp index cacb5c00b..f928b517f 100644 --- a/Plugins/NativeXr/Source/NativeXr.cpp +++ b/Plugins/NativeXr/Source/NativeXr.cpp @@ -11,6 +11,7 @@ #include "Constants.h" #include "PointerEvent.h" +#include "XRCamera.h" #include "XRWebGLBinding.h" #include "XRWebGLLayer.h" #include "XRRigidTransform.h" @@ -52,6 +53,7 @@ namespace Babylon PointerEvent::Initialize(env); + XRCamera::Initialize(env); XRWebGLBinding::Initialize(env); XRWebGLLayer::Initialize(env); XRRigidTransform::Initialize(env); diff --git a/Plugins/NativeXr/Source/XRCamera.h b/Plugins/NativeXr/Source/XRCamera.h new file mode 100644 index 000000000..dc1c56cc7 --- /dev/null +++ b/Plugins/NativeXr/Source/XRCamera.h @@ -0,0 +1,146 @@ +#pragma once + +// WebXR raw camera access: JS XRCamera object, per the WebXR Raw Camera +// Access module (https://immersive-web.github.io/raw-camera-access/). Exposed via +// XRView.camera; the camera image is retrieved with XRWebGLBinding.getCameraImage, +// which returns a Napi::Pointer — the same object type +// NativeEngine uses as an InternalTexture's underlyingResource, so Babylon.js can +// bind it as a regular sampler. +// +// The texture pointer originates from the xr backend (View::CameraTexturePointer, an +// MTLTexture on ARKit) and is wired to bgfx with the same createTexture2D + +// overrideInternal pattern used by NativeCamera's CameraDevice. + +#include +#include +#include +#include +#include + +#include +#include + +namespace Babylon +{ + class XRCamera : public Napi::ObjectWrap + { + static constexpr auto JS_CLASS_NAME = "XRCamera"; + + public: + static void Initialize(Napi::Env env) + { + Napi::HandleScope scope{env}; + + Napi::Function func = DefineClass( + env, + JS_CLASS_NAME, + { + InstanceAccessor("width", &XRCamera::GetWidth, nullptr), + InstanceAccessor("height", &XRCamera::GetHeight, nullptr), + }); + + env.Global().Set(JS_CLASS_NAME, func); + } + + static Napi::Object New(const Napi::CallbackInfo& info) + { + return info.Env().Global().Get(JS_CLASS_NAME).As().New({}); + } + + XRCamera(const Napi::CallbackInfo& info) + : Napi::ObjectWrap{info} + , m_deviceContext{&Graphics::DeviceContext::GetFromJavaScript(info.Env())} + { + } + + // Called per getViewerPose from XRView::Update with the current platform camera + // texture (MTLTexture on ARKit). Creates/refreshes the bgfx-wrapped texture when + // the platform texture identity or size changes. + void Update(Napi::Env env, void* texturePointer, size_t width, size_t height) + { + if (texturePointer == nullptr || width == 0 || height == 0) + { + m_hasTexture = false; + return; + } + + if (texturePointer != m_texturePointer || width != m_width || height != m_height) + { + m_texturePointer = texturePointer; + m_width = width; + m_height = height; + + // Create a bgfx texture to be overridden with the platform texture. The + // Graphics::Texture (which owns the bgfx handle) is owned by the JS pointer + // object below; replacing the persistent ref releases the old one via GC. + const auto bgfxHandle = bgfx::createTexture2D( + static_cast(width), static_cast(height), false, 1, + bgfx::TextureFormat::BGRA8, BGFX_TEXTURE_NONE | BGFX_SAMPLER_NONE); + + auto* texture = new Graphics::Texture(*m_deviceContext); + texture->Attach(bgfxHandle, true, static_cast(width), static_cast(height), + false, 1, bgfx::TextureFormat::BGRA8, BGFX_TEXTURE_NONE | BGFX_SAMPLER_NONE); + + m_textureJs = Napi::Persistent(Napi::Pointer::Create(env, texture, Napi::NapiPointerDeleter(texture))); + + // New handle needs a (re)override. + m_overrideState = std::make_shared(); + m_bgfxHandle = bgfxHandle; + } + + // bgfx::overrideInternal only succeeds once the handle has been initialized by a + // bgfx frame, so run it on the render thread after rendering and retry until it + // sticks (same approach as NativeCamera's CameraDevice::UpdateCameraTexture). + if (!m_overrideState->done.load() && !m_overrideState->pending.exchange(true)) + { + arcana::make_task(m_deviceContext->AfterRenderScheduler(), arcana::cancellation::none(), + [state = m_overrideState, handle = m_bgfxHandle, pointer = m_texturePointer]() { + state->done.store(bgfx::overrideInternal(handle, reinterpret_cast(pointer)) != 0); + state->pending.store(false); + }); + } + + m_hasTexture = true; + } + + bool HasTexture() const + { + return m_hasTexture && !m_textureJs.IsEmpty(); + } + + Napi::Value GetCameraImageValue(Napi::Env env) + { + if (!HasTexture()) + { + return env.Undefined(); + } + return m_textureJs.Value(); + } + + private: + struct OverrideState + { + std::atomic done{false}; + std::atomic pending{false}; + }; + + Graphics::DeviceContext* m_deviceContext{}; + void* m_texturePointer{}; + size_t m_width{}; + size_t m_height{}; + bool m_hasTexture{}; + bgfx::TextureHandle m_bgfxHandle{bgfx::kInvalidHandle}; + std::shared_ptr m_overrideState{std::make_shared()}; + Napi::Reference m_textureJs{}; + + Napi::Value GetWidth(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), static_cast(m_width)); + } + + Napi::Value GetHeight(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), static_cast(m_height)); + } + }; +} // Babylon diff --git a/Plugins/NativeXr/Source/XRView.h b/Plugins/NativeXr/Source/XRView.h index 50a653b5d..0783ac5cc 100644 --- a/Plugins/NativeXr/Source/XRView.h +++ b/Plugins/NativeXr/Source/XRView.h @@ -1,5 +1,7 @@ #pragma once +#include "XRCamera.h" + namespace Babylon { class XRView : public Napi::ObjectWrap @@ -20,6 +22,7 @@ namespace Babylon InstanceAccessor("projectionMatrix", &XRView::GetProjectionMatrix, nullptr), InstanceAccessor("transform", &XRView::GetTransform, nullptr), InstanceAccessor("isFirstPersonObserver", &XRView::IsFirstPersonObserver, nullptr), + InstanceAccessor("camera", &XRView::GetCamera, nullptr), }); env.Global().Set(JS_CLASS_NAME, func); @@ -40,7 +43,8 @@ namespace Babylon { } - void Update(size_t eyeIdx, gsl::span projectionMatrix, const xr::Space& space, bool isFirstPersonObserver) + void Update(const Napi::CallbackInfo& info, size_t eyeIdx, gsl::span projectionMatrix, const xr::Space& space, bool isFirstPersonObserver, + void* cameraTexturePointer, size_t cameraTextureWidth, size_t cameraTextureHeight) { if (eyeIdx != m_eyeIdx) { @@ -53,6 +57,21 @@ namespace Babylon XRRigidTransform::Unwrap(m_rigidTransform.Value())->Update(space, false); m_isFirstPersonObserver = isFirstPersonObserver; + + // WebXR raw camera access: keep the per-view XRCamera in sync with the + // platform camera texture (null on platforms/backends that don't provide one). + if (cameraTexturePointer != nullptr) + { + if (m_camera.IsEmpty()) + { + m_camera = Napi::Persistent(XRCamera::New(info)); + } + XRCamera::Unwrap(m_camera.Value())->Update(info.Env(), cameraTexturePointer, cameraTextureWidth, cameraTextureHeight); + } + else if (!m_camera.IsEmpty()) + { + XRCamera::Unwrap(m_camera.Value())->Update(info.Env(), nullptr, 0, 0); + } } private: @@ -61,6 +80,7 @@ namespace Babylon Napi::Reference m_projectionMatrix{}; Napi::ObjectReference m_rigidTransform{}; bool m_isFirstPersonObserver{}; + Napi::ObjectReference m_camera{}; Napi::Value GetEye(const Napi::CallbackInfo& info) { @@ -81,5 +101,14 @@ namespace Babylon { return Napi::Boolean::From(info.Env(), m_isFirstPersonObserver); } + + Napi::Value GetCamera(const Napi::CallbackInfo& info) + { + if (m_camera.IsEmpty() || !XRCamera::Unwrap(m_camera.Value())->HasTexture()) + { + return info.Env().Undefined(); + } + return m_camera.Value(); + } }; } // Babylon diff --git a/Plugins/NativeXr/Source/XRViewerPose.h b/Plugins/NativeXr/Source/XRViewerPose.h index 9cbceecf6..51ad92be8 100644 --- a/Plugins/NativeXr/Source/XRViewerPose.h +++ b/Plugins/NativeXr/Source/XRViewerPose.h @@ -77,7 +77,8 @@ namespace Babylon for (uint32_t idx = 0; idx < static_cast(frame.Views.size()); ++idx) { const auto& view = frame.Views[idx]; - m_views[idx]->Update(idx, view.ProjectionMatrix, view.Space, view.IsFirstPersonObserver); + m_views[idx]->Update(info, idx, view.ProjectionMatrix, view.Space, view.IsFirstPersonObserver, + view.CameraTexturePointer, view.CameraTextureSize.Width, view.CameraTextureSize.Height); } // Check the frame to see if it has valid tracking, if it does not then the position should diff --git a/Plugins/NativeXr/Source/XRWebGLBinding.h b/Plugins/NativeXr/Source/XRWebGLBinding.h index 48286f7b8..69fe61b71 100644 --- a/Plugins/NativeXr/Source/XRWebGLBinding.h +++ b/Plugins/NativeXr/Source/XRWebGLBinding.h @@ -1,5 +1,7 @@ #pragma once +#include "XRCamera.h" + namespace Babylon { class XRWebGLBinding : public Napi::ObjectWrap @@ -14,7 +16,9 @@ namespace Babylon Napi::Function func = DefineClass( env, JS_CLASS_NAME, - {}); + { + InstanceMethod("getCameraImage", &XRWebGLBinding::GetCameraImage), + }); env.Global().Set(JS_CLASS_NAME, func); } @@ -28,5 +32,25 @@ namespace Babylon : Napi::ObjectWrap{info} { } + + private: + // WebXR raw camera access: returns the camera image for the given + // XRCamera as a Napi::Pointer — the object type NativeEngine + // uses as a hardware texture's underlyingResource, so Babylon.js can sample it. + Napi::Value GetCameraImage(const Napi::CallbackInfo& info) + { + if (info.Length() < 1 || !info[0].IsObject()) + { + return info.Env().Undefined(); + } + + XRCamera* camera = XRCamera::Unwrap(info[0].As()); + if (camera == nullptr) + { + return info.Env().Undefined(); + } + + return camera->GetCameraImageValue(info.Env()); + } }; } // Babylon