From 59e24d1bf397568296c20460229f0afe3937ac3b Mon Sep 17 00:00:00 2001 From: lukasz karluk Date: Tue, 7 Jul 2026 18:31:50 +1000 Subject: [PATCH 1/2] NativeCamera: add VideoPlayer backend for video-file decoding (AVPlayer) Adds a platform VideoPlayer abstraction (VideoPlayer.h) plus an Apple implementation backed by AVPlayer + AVPlayerItemVideoOutput. Frames are requested as BGRA (no YUV conversion needed), viewed zero-copy through a CVMetalTextureCache and blitted into a persistent MTLTexture, which is wired to Babylon's InternalTexture with the same bgfx::overrideInternal-on-render- thread retry pattern as CameraDevice::UpdateCameraTexture. Each player instance owns its own decoder and texture, so any number of videos can play simultaneously. URLs may be app:///, absolute file paths, or http(s). Playback control (play/pause/loop/muted/volume/seek) and metadata (duration/currentTime/dimensions) are exposed for the JS layer; end-of-stream loops via seek-to-zero or reports Ended. Event callbacks may fire on any thread and are documented as caller-marshalled. This is the decode/GPU half of HTMLVideoElement.src support. Co-Authored-By: Claude Opus 4.8 --- Plugins/NativeCamera/CMakeLists.txt | 6 + .../NativeCamera/Source/Apple/VideoPlayer.mm | 391 ++++++++++++++++++ Plugins/NativeCamera/Source/VideoPlayer.h | 61 +++ 3 files changed, 458 insertions(+) create mode 100644 Plugins/NativeCamera/Source/Apple/VideoPlayer.mm create mode 100644 Plugins/NativeCamera/Source/VideoPlayer.h diff --git a/Plugins/NativeCamera/CMakeLists.txt b/Plugins/NativeCamera/CMakeLists.txt index c369f2f3d..72583db0e 100644 --- a/Plugins/NativeCamera/CMakeLists.txt +++ b/Plugins/NativeCamera/CMakeLists.txt @@ -19,10 +19,16 @@ set(SOURCES "Source/NativeCamera.cpp" "Source/NativeVideo.cpp" "Source/NativeVideo.h" + "Source/VideoPlayer.h" "Source/CameraDevice.h" "Source/CameraDeviceSharedPImpl.h" "Source/${PLATFORM}/CameraDevice.${BABYLON_NATIVE_PLATFORM_IMPL_EXT}") +if(APPLE) + # Video-file playback (HTMLVideoElement.src) backend - AVPlayer based. + list(APPEND SOURCES "Source/Apple/VideoPlayer.mm") +endif() + add_library(NativeCamera ${SOURCES}) warnings_as_errors(NativeCamera) diff --git a/Plugins/NativeCamera/Source/Apple/VideoPlayer.mm b/Plugins/NativeCamera/Source/Apple/VideoPlayer.mm new file mode 100644 index 000000000..d4417c14b --- /dev/null +++ b/Plugins/NativeCamera/Source/Apple/VideoPlayer.mm @@ -0,0 +1,391 @@ +#if ! __has_feature(objc_arc) +#error "ARC is off" +#endif + +// Apple implementation of VideoPlayer (see ../VideoPlayer.h) backed by AVPlayer + +// AVPlayerItemVideoOutput. Decoded frames arrive as BGRA pixel buffers (no YUV pass +// needed) and are blitted into a persistent MTLTexture, which is wired to Babylon's +// InternalTexture with the same bgfx::overrideInternal retry pattern used by +// CameraDevice::UpdateCameraTexture. + +#import +#import +#import +#import + +#include +#include +#include +#include +#include + +#include "../VideoPlayer.h" + +#include +#include + +@class VideoPlayerItemObserver; + +namespace Babylon::Plugins +{ + namespace + { + NSURL* ResolveUrl(const std::string& url) + { + NSString* str = [NSString stringWithUTF8String:url.c_str()]; + if ([str hasPrefix:@"app://"]) + { + // Match the app:/// scheme used by ScriptLoader: resolve against the bundle. + NSURL* parsed = [NSURL URLWithString:str]; + NSString* relativePath = parsed.path; // e.g. "/Scripts/video.mp4" + NSString* fullPath = [[[NSBundle mainBundle] resourcePath] stringByAppendingString:relativePath]; + return [NSURL fileURLWithPath:fullPath]; + } + if ([str hasPrefix:@"/"]) + { + return [NSURL fileURLWithPath:str]; + } + return [NSURL URLWithString:str]; + } + } + + class VideoPlayerApple final : public VideoPlayer + { + public: + // Shared with render-thread tasks and ObjC observers so in-flight work stays + // valid after the VideoPlayer is destroyed. + struct State + { + Graphics::DeviceContext* deviceContext{}; + EventCallbackT eventCallback{}; + + AVPlayer* player{}; + AVPlayerItem* item{}; + AVPlayerItemVideoOutput* videoOutput{}; + VideoPlayerItemObserver* observer{}; + id endObserverToken{}; + + // Metal resources (initialized lazily on the render thread). + id metalDevice{}; + id commandQueue{}; + CVMetalTextureCacheRef textureCache{}; + id textureRGBA{}; + + bgfx::TextureHandle bgfxHandle{bgfx::kInvalidHandle}; + bool refreshBgfxHandle{true}; + + std::atomic ready{false}; + std::atomic loop{false}; + std::atomic invalid{false}; + std::atomic width{0}; + std::atomic height{0}; + + ~State() + { + if (textureCache) + { + CVMetalTextureCacheFlush(textureCache, 0); + CFRelease(textureCache); + } + } + }; + + VideoPlayerApple(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback); + ~VideoPlayerApple() override; + + void Play() override + { + [m_state->player play]; + } + + void Pause() override + { + [m_state->player pause]; + } + + void SetLoop(bool loop) override + { + m_state->loop.store(loop); + } + + void SetMuted(bool muted) override + { + m_state->player.muted = muted; + } + + void SetVolume(double volume) override + { + m_state->player.volume = static_cast(volume); + } + + void Seek(double timeInSeconds) override + { + auto state = m_state; + [m_state->player seekToTime:CMTimeMakeWithSeconds(timeInSeconds, 600) + toleranceBefore:kCMTimeZero + toleranceAfter:kCMTimeZero + completionHandler:^(BOOL) { + if (!state->invalid.load() && state->eventCallback) + { + state->eventCallback(Event::Seeked); + } + }]; + } + + bool IsReady() const override + { + return m_state->ready.load(); + } + + double GetDuration() const override + { + if (!m_state->ready.load()) + { + return 0; + } + const CMTime duration = m_state->item.duration; + return CMTIME_IS_NUMERIC(duration) ? CMTimeGetSeconds(duration) : 0; + } + + double GetCurrentTime() const override + { + const CMTime time = m_state->player.currentTime; + return CMTIME_IS_NUMERIC(time) ? CMTimeGetSeconds(time) : 0; + } + + uint32_t GetWidth() const override + { + return m_state->width.load(); + } + + uint32_t GetHeight() const override + { + return m_state->height.load(); + } + + void UpdateTexture(bgfx::TextureHandle textureHandle) override; + + private: + std::shared_ptr m_state{}; + }; +} + +// Observes AVPlayerItem.status so we can report readiness (metadata/dimensions). +@interface VideoPlayerItemObserver : NSObject +- (id)initWithState:(std::shared_ptr)state; +- (void)stopObserving; +@end + +@implementation VideoPlayerItemObserver +{ + std::shared_ptr _state; + BOOL _observing; +} + +- (id)initWithState:(std::shared_ptr)state +{ + if (self = [super init]) + { + _state = std::move(state); + [_state->item addObserver:self forKeyPath:@"status" options:NSKeyValueObservingOptionNew context:nil]; + _observing = YES; + } + return self; +} + +- (void)stopObserving +{ + if (_observing) + { + [_state->item removeObserver:self forKeyPath:@"status"]; + _observing = NO; + } +} + +- (void)observeValueForKeyPath:(NSString*)keyPath ofObject:(id)__unused object change:(NSDictionary*)__unused change context:(void*)__unused context +{ + if ([keyPath isEqualToString:@"status"] && _state->item.status == AVPlayerItemStatusReadyToPlay && !_state->ready.load()) + { + const CGSize size = _state->item.presentationSize; + _state->width.store(static_cast(size.width)); + _state->height.store(static_cast(size.height)); + _state->ready.store(true); + if (!_state->invalid.load() && _state->eventCallback) + { + _state->eventCallback(Babylon::Plugins::VideoPlayer::Event::MetadataLoaded); + } + } +} +@end + +namespace Babylon::Plugins +{ + std::unique_ptr VideoPlayer::Create(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback) + { + return std::make_unique(deviceContext, url, std::move(eventCallback)); + } + + VideoPlayerApple::VideoPlayerApple(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback) + : m_state{std::make_shared()} + { + m_state->deviceContext = &deviceContext; + m_state->eventCallback = std::move(eventCallback); + + NSURL* nsUrl = ResolveUrl(url); + m_state->item = [AVPlayerItem playerItemWithURL:nsUrl]; + m_state->player = [AVPlayer playerWithPlayerItem:m_state->item]; + m_state->player.actionAtItemEnd = AVPlayerActionAtItemEndNone; + + // Decoded frames as BGRA so they can be blitted straight into the sampled texture. + NSDictionary* outputAttributes = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (id)kCVPixelBufferMetalCompatibilityKey: @YES, + }; + m_state->videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:outputAttributes]; + [m_state->item addOutput:m_state->videoOutput]; + + m_state->observer = [[VideoPlayerItemObserver alloc] initWithState:m_state]; + + // End-of-playback: loop by seeking back to zero, otherwise report Ended. + auto state = m_state; + m_state->endObserverToken = [[NSNotificationCenter defaultCenter] addObserverForName:AVPlayerItemDidPlayToEndTimeNotification + object:m_state->item + queue:nil + usingBlock:^(NSNotification*) { + if (state->invalid.load()) + { + return; + } + if (state->loop.load()) + { + [state->player seekToTime:kCMTimeZero toleranceBefore:kCMTimeZero toleranceAfter:kCMTimeZero completionHandler:^(BOOL) {}]; + } + else + { + [state->player pause]; + if (state->eventCallback) + { + state->eventCallback(Event::Ended); + } + } + }]; + } + + VideoPlayerApple::~VideoPlayerApple() + { + m_state->invalid.store(true); + [m_state->observer stopObserving]; + if (m_state->endObserverToken) + { + [[NSNotificationCenter defaultCenter] removeObserver:m_state->endObserverToken]; + m_state->endObserverToken = nil; + } + [m_state->player pause]; + // In-flight render-thread tasks hold the shared State and finish safely. + } + + void VideoPlayerApple::UpdateTexture(bgfx::TextureHandle textureHandle) + { + if (!m_state->ready.load()) + { + return; + } + + // Hook into AfterRender (like CameraDevice::UpdateCameraTexture) so the bgfx + // handle has been initialized by a frame before we override it, and so Metal + // work doesn't race the renderer. + arcana::make_task(m_state->deviceContext->AfterRenderScheduler(), arcana::cancellation::none(), [state = m_state, textureHandle] { + if (state->invalid.load()) + { + return; + } + + // Lazily bind to bgfx's Metal device/queue (render thread only). + if (state->metalDevice == nil) + { + state->metalDevice = (__bridge id)bgfx::getInternalData()->context; + state->commandQueue = (__bridge id)bgfx::getInternalData()->commandQueue; + CVMetalTextureCacheCreate(nullptr, nullptr, state->metalDevice, nullptr, &state->textureCache); + } + + if (state->bgfxHandle.idx != textureHandle.idx) + { + state->bgfxHandle = textureHandle; + state->refreshBgfxHandle = true; + } + + const CMTime itemTime = [state->videoOutput itemTimeForHostTime:CACurrentMediaTime()]; + if ([state->videoOutput hasNewPixelBufferForItemTime:itemTime]) + { + CVPixelBufferRef pixelBuffer = [state->videoOutput copyPixelBufferForItemTime:itemTime itemTimeForDisplay:nil]; + if (pixelBuffer != nullptr) + { + const size_t width = CVPixelBufferGetWidth(pixelBuffer); + const size_t height = CVPixelBufferGetHeight(pixelBuffer); + state->width.store(static_cast(width)); + state->height.store(static_cast(height)); + + // (Re)create the persistent sampled texture on size change. + if (state->textureRGBA == nil || state->textureRGBA.width != width || state->textureRGBA.height != height) + { + MTLTextureDescriptor* descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm + width:width + height:height + mipmapped:NO]; + descriptor.usage = MTLTextureUsageShaderRead; + state->textureRGBA = [state->metalDevice newTextureWithDescriptor:descriptor]; + state->refreshBgfxHandle = true; + } + + // Zero-copy Metal view of the decoded BGRA pixel buffer. + CVMetalTextureRef sourceTextureRef{}; + CVMetalTextureCacheCreateTextureFromImage(nullptr, state->textureCache, pixelBuffer, nullptr, + MTLPixelFormatBGRA8Unorm, width, height, 0, &sourceTextureRef); + + if (sourceTextureRef != nullptr) + { + id sourceTexture = CVMetalTextureGetTexture(sourceTextureRef); + if (sourceTexture != nil) + { + id commandBuffer = [state->commandQueue commandBuffer]; + commandBuffer.label = @"NativeVideoPlayerBlit"; + id blit = [commandBuffer blitCommandEncoder]; + [blit copyFromTexture:sourceTexture + sourceSlice:0 + sourceLevel:0 + sourceOrigin:MTLOriginMake(0, 0, 0) + sourceSize:MTLSizeMake(width, height, 1) + toTexture:state->textureRGBA + destinationSlice:0 + destinationLevel:0 + destinationOrigin:MTLOriginMake(0, 0, 0)]; + [blit endEncoding]; + [commandBuffer addCompletedHandler:^(id) { + // Keep the pixel buffer + texture view alive until the GPU is done. + CFRelease(sourceTextureRef); + CVPixelBufferRelease(pixelBuffer); + }]; + [commandBuffer commit]; + } + else + { + CFRelease(sourceTextureRef); + CVPixelBufferRelease(pixelBuffer); + } + } + else + { + CVPixelBufferRelease(pixelBuffer); + } + } + } + + // Wire (or re-wire) Babylon's texture handle to our persistent texture. May + // fail until the handle has been initialized by bgfx; retried next frame. + if (state->textureRGBA != nil && state->refreshBgfxHandle && bgfx::isValid(state->bgfxHandle)) + { + state->refreshBgfxHandle = bgfx::overrideInternal(state->bgfxHandle, reinterpret_cast(state->textureRGBA)) == 0; + } + }); + } +} diff --git a/Plugins/NativeCamera/Source/VideoPlayer.h b/Plugins/NativeCamera/Source/VideoPlayer.h new file mode 100644 index 000000000..395fc0964 --- /dev/null +++ b/Plugins/NativeCamera/Source/VideoPlayer.h @@ -0,0 +1,61 @@ +#pragma once + +// Video-file playback backend for NativeVideo (HTMLVideoElement.src support). +// +// Each instance owns a platform media player (AVPlayer on Apple) and a persistent +// GPU texture that receives decoded frames. The texture is wired to Babylon.js's +// VideoTexture with the same bgfx createTexture2D-less overrideInternal pattern +// used by CameraDevice: Babylon creates the InternalTexture (createDynamicTexture) +// and NativeVideo::UpdateTexture hands its bgfx handle here every frame. +// +// Event callbacks may fire on ANY thread — callers must marshal to the JS thread. + +#include +#include +#include +#include + +namespace Babylon::Graphics +{ + class DeviceContext; +} + +namespace Babylon::Plugins +{ + class VideoPlayer + { + public: + enum class Event + { + MetadataLoaded, // duration/dimensions known; playback can start + Ended, // reached the end (not fired when looping) + Seeked, // a Seek() completed + }; + + using EventCallbackT = std::function; + + // Returns nullptr on platforms without an implementation. + // Supported url forms: app:///, absolute file paths, http(s) URLs. + static std::unique_ptr Create(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback); + + virtual ~VideoPlayer() = default; + + virtual void Play() = 0; + virtual void Pause() = 0; + virtual void SetLoop(bool loop) = 0; + virtual void SetMuted(bool muted) = 0; + virtual void SetVolume(double volume) = 0; + virtual void Seek(double timeInSeconds) = 0; + + virtual bool IsReady() const = 0; + virtual double GetDuration() const = 0; + virtual double GetCurrentTime() const = 0; + virtual uint32_t GetWidth() const = 0; + virtual uint32_t GetHeight() const = 0; + + // Pull the latest decoded frame into the persistent texture and (re)wire it to + // the given bgfx texture handle. Called on the JS thread once per render frame; + // the GPU work runs on the render thread. Cheap no-op when no new frame exists. + virtual void UpdateTexture(bgfx::TextureHandle textureHandle) = 0; + }; +} From 9f3665284f45ec7a93baee5fabce8de5bca3be16 Mon Sep 17 00:00:00 2001 From: lukasz karluk Date: Tue, 7 Jul 2026 18:31:50 +1000 Subject: [PATCH 2/2] NativeCamera: support HTMLVideoElement.src (web-style video playback API) Extends the HTMLVideoElement polyfill so a video file/URL can be played and its frames rendered into the Babylon scene, alongside the existing srcObject (camera MediaStream) source. Assigning src creates a VideoPlayer; the two sources are mutually exclusive, matching the web API. Implements the HTMLVideoElement surface Babylon.js's VideoTexture and typical app code use: src, currentTime (get/seek), duration, loop, muted, paused, ended, videoWidth/videoHeight, plus events (loadedmetadata, loadeddata, canplay, playing, pause, seeked, ended, resize). isNative is reported so VideoTexture accepts the element directly. VideoPlayer event callbacks are marshalled onto the JS thread via JsRuntime::Dispatch; the element holds a self-reference while it owns a source so it stays alive during playback. UpdateTexture pulls decoded frames each render frame and raises resize on dimension changes. Babylon.js is unmodified: new BABYLON.VideoTexture(name, videoElement, scene) with videoElement.src set works as on the web, including multiple simultaneous videos. Verified on device (iPhone 15 Pro Max, iOS 26.5) with two independent looping videos plus an out-of-phase seek at 60 FPS. Co-Authored-By: Claude Opus 4.8 --- Plugins/NativeCamera/Source/NativeVideo.cpp | 195 +++++++++++++++++++- Plugins/NativeCamera/Source/NativeVideo.h | 36 +++- 2 files changed, 228 insertions(+), 3 deletions(-) diff --git a/Plugins/NativeCamera/Source/NativeVideo.cpp b/Plugins/NativeCamera/Source/NativeVideo.cpp index 5198f357d..587264fec 100644 --- a/Plugins/NativeCamera/Source/NativeVideo.cpp +++ b/Plugins/NativeCamera/Source/NativeVideo.cpp @@ -22,6 +22,14 @@ namespace Babylon::Plugins InstanceAccessor("readyState", &NativeVideo::GetReadyState, nullptr), InstanceAccessor("HAVE_CURRENT_DATA", &NativeVideo::GetHaveCurrentData, nullptr), InstanceAccessor("srcObject", &NativeVideo::GetSrcObject, &NativeVideo::SetSrcObject), + InstanceAccessor("src", &NativeVideo::GetSrc, &NativeVideo::SetSrc), + InstanceAccessor("isNative", &NativeVideo::GetIsNative, nullptr), + InstanceAccessor("currentTime", &NativeVideo::GetCurrentTime, &NativeVideo::SetCurrentTime), + InstanceAccessor("duration", &NativeVideo::GetDuration, nullptr), + InstanceAccessor("loop", &NativeVideo::GetLoop, &NativeVideo::SetLoop), + InstanceAccessor("muted", &NativeVideo::GetMuted, &NativeVideo::SetMuted), + InstanceAccessor("paused", &NativeVideo::GetPaused, nullptr), + InstanceAccessor("ended", &NativeVideo::GetEnded, nullptr), }); env.Global().Set(JS_CLASS_NAME, func); @@ -34,11 +42,17 @@ namespace Babylon::Plugins NativeVideo::NativeVideo(const Napi::CallbackInfo& info) : Napi::ObjectWrap{info} + , m_runtime{&JsRuntime::GetFromJavaScript(info.Env())} { } Napi::Value NativeVideo::GetVideoWidth(const Napi::CallbackInfo& /*info*/) { + if (m_videoPlayer != nullptr) + { + return Napi::Value::From(Env(), m_videoPlayer->GetWidth()); + } + if (!m_streamObject.Value().IsNull() && !m_streamObject.Value().IsUndefined()) { return Napi::Value::From(Env(), MediaStream::Unwrap(m_streamObject.Value())->Width); @@ -49,6 +63,11 @@ namespace Babylon::Plugins Napi::Value NativeVideo::GetVideoHeight(const Napi::CallbackInfo& /*info*/) { + if (m_videoPlayer != nullptr) + { + return Napi::Value::From(Env(), m_videoPlayer->GetHeight()); + } + if (!m_streamObject.Value().IsNull() && !m_streamObject.Value().IsUndefined()) { return Napi::Value::From(Env(), MediaStream::Unwrap(m_streamObject.Value())->Height); @@ -88,6 +107,21 @@ namespace Babylon::Plugins RaiseEvent("resize"); } } + // Video-file playback (src). Not gated on m_IsPlaying so a seek while paused + // still refreshes the current frame; the player is a cheap no-op when idle. + else if (m_videoPlayer != nullptr && m_videoPlayer->IsReady()) + { + m_videoPlayer->UpdateTexture(textureHandle); + + const uint32_t width = m_videoPlayer->GetWidth(); + const uint32_t height = m_videoPlayer->GetHeight(); + if (width != m_lastVideoWidth || height != m_lastVideoHeight) + { + m_lastVideoWidth = width; + m_lastVideoHeight = height; + RaiseEvent("resize"); + } + } } void NativeVideo::AddEventListener(const Napi::CallbackInfo& info) @@ -149,6 +183,14 @@ namespace Babylon::Plugins m_IsPlaying = true; RaiseEvent("playing"); } + else if (!m_IsPlaying && m_videoPlayer != nullptr) + { + m_IsPlaying = true; + m_ended = false; + // Safe to call before the player is ready; playback starts once buffered. + m_videoPlayer->Play(); + RaiseEvent("playing"); + } deferred.Resolve(env.Undefined()); @@ -157,7 +199,18 @@ namespace Babylon::Plugins void NativeVideo::Pause(const Napi::CallbackInfo& /*info*/) { + const bool wasPlaying = m_IsPlaying; m_IsPlaying = false; + if (m_videoPlayer != nullptr) + { + m_videoPlayer->Pause(); + if (wasPlaying) + { + RaiseEvent("pause"); + // Babylon.js's VideoTexture listens for "paused" (its own convention). + RaiseEvent("paused"); + } + } } void NativeVideo::SetSrcObject(const Napi::CallbackInfo& info, const Napi::Value& value) @@ -172,7 +225,10 @@ namespace Babylon::Plugins return; } - // We've received a MediaStream object + // We've received a MediaStream object. A media element has a single source: + // assigning a stream replaces any file player. + m_videoPlayer.reset(); + m_src.clear(); m_streamObject = Napi::Persistent(value.As()); this->m_isReady = true; @@ -185,4 +241,141 @@ namespace Babylon::Plugins // If the streamObject has not yet been defined return Null instead to match the web API return m_streamObject.IsEmpty() ? info.Env().Null() : m_streamObject.Value(); } + + Napi::Value NativeVideo::GetSrc(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_src); + } + + void NativeVideo::SetSrc(const Napi::CallbackInfo& info, const Napi::Value& value) + { + m_videoPlayer.reset(); + m_isReady = false; + m_IsPlaying = false; + m_ended = false; + m_lastVideoWidth = 0; + m_lastVideoHeight = 0; + m_src.clear(); + + if (value.IsNull() || value.IsUndefined() || !value.IsString()) + { + RaiseEvent("emptied"); + return; + } + + m_src = value.As().Utf8Value(); + if (m_src.empty()) + { + RaiseEvent("emptied"); + return; + } + + // A media element with an active source is kept alive (and makes the + // cross-thread event dispatches below safe). + m_selfRef = Napi::Persistent(info.This().As()); + + // Assigning a file source replaces any stream source. + m_streamObject = Napi::ObjectReference(); + +#ifdef __APPLE__ + auto& deviceContext = Graphics::DeviceContext::GetFromJavaScript(info.Env()); + m_videoPlayer = VideoPlayer::Create(deviceContext, m_src, [this](VideoPlayer::Event event) { + // May be called on any thread — marshal to the JS thread. m_selfRef keeps + // `this` alive for the lifetime of the dispatch. + m_runtime->Dispatch([this, event](Napi::Env) { + switch (event) + { + case VideoPlayer::Event::MetadataLoaded: + m_isReady = true; + RaiseEvent("loadedmetadata"); + RaiseEvent("loadeddata"); + RaiseEvent("canplay"); + // play() may have been called before the media was ready. + if (m_IsPlaying && m_videoPlayer != nullptr) + { + m_videoPlayer->Play(); + } + break; + case VideoPlayer::Event::Ended: + m_IsPlaying = false; + m_ended = true; + RaiseEvent("ended"); + break; + case VideoPlayer::Event::Seeked: + RaiseEvent("seeked"); + break; + } + }); + }); + if (m_videoPlayer != nullptr) + { + m_videoPlayer->SetLoop(m_loop); + m_videoPlayer->SetMuted(m_muted); + } +#endif + } + + Napi::Value NativeVideo::GetIsNative(const Napi::CallbackInfo& info) + { + // Lets Babylon.js's VideoTexture accept this element directly (its _getVideo + // returns native elements as-is instead of calling document.createElement). + return Napi::Value::From(info.Env(), true); + } + + Napi::Value NativeVideo::GetCurrentTime(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_videoPlayer != nullptr ? m_videoPlayer->GetCurrentTime() : 0.0); + } + + void NativeVideo::SetCurrentTime(const Napi::CallbackInfo& /*info*/, const Napi::Value& value) + { + if (m_videoPlayer != nullptr && value.IsNumber()) + { + m_ended = false; + m_videoPlayer->Seek(value.As().DoubleValue()); + } + } + + Napi::Value NativeVideo::GetDuration(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_videoPlayer != nullptr ? m_videoPlayer->GetDuration() : 0.0); + } + + Napi::Value NativeVideo::GetLoop(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_loop); + } + + void NativeVideo::SetLoop(const Napi::CallbackInfo& /*info*/, const Napi::Value& value) + { + m_loop = value.ToBoolean(); + if (m_videoPlayer != nullptr) + { + m_videoPlayer->SetLoop(m_loop); + } + } + + Napi::Value NativeVideo::GetMuted(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_muted); + } + + void NativeVideo::SetMuted(const Napi::CallbackInfo& /*info*/, const Napi::Value& value) + { + m_muted = value.ToBoolean(); + if (m_videoPlayer != nullptr) + { + m_videoPlayer->SetMuted(m_muted); + } + } + + Napi::Value NativeVideo::GetPaused(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), !m_IsPlaying); + } + + Napi::Value NativeVideo::GetEnded(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_ended); + } } diff --git a/Plugins/NativeCamera/Source/NativeVideo.h b/Plugins/NativeCamera/Source/NativeVideo.h index ab8e27ca8..3d2546c5e 100644 --- a/Plugins/NativeCamera/Source/NativeVideo.h +++ b/Plugins/NativeCamera/Source/NativeVideo.h @@ -2,6 +2,7 @@ #include #include "CameraDevice.h" #include "MediaStream.h" +#include "VideoPlayer.h" #include #include #include @@ -10,8 +11,11 @@ namespace Babylon::Plugins { - // NativeVideo provides a polyfill for the HTMLVideoElement which is used in conjunction with a MediaStream object to - // pull frames out of a camera and render them into the Babylon scene. + // NativeVideo provides a polyfill for the HTMLVideoElement. It supports two media + // sources, matching the web API: + // - srcObject: a MediaStream (camera frames via CameraDevice) + // - src: a video file/URL, decoded by a per-instance VideoPlayer (AVPlayer on Apple) + // In both cases frames are pulled into the Babylon scene through UpdateTexture. class NativeVideo : public Napi::ObjectWrap { public: @@ -37,6 +41,20 @@ namespace Babylon::Plugins Napi::Value GetSrcObject(const Napi::CallbackInfo& info); void SetSrcObject(const Napi::CallbackInfo& info, const Napi::Value& value); + // Video-file (src) support. + Napi::Value GetSrc(const Napi::CallbackInfo& info); + void SetSrc(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetIsNative(const Napi::CallbackInfo& info); + Napi::Value GetCurrentTime(const Napi::CallbackInfo& info); + void SetCurrentTime(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetDuration(const Napi::CallbackInfo& info); + Napi::Value GetLoop(const Napi::CallbackInfo& info); + void SetLoop(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetMuted(const Napi::CallbackInfo& info); + void SetMuted(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetPaused(const Napi::CallbackInfo& info); + Napi::Value GetEnded(const Napi::CallbackInfo& info); + std::unordered_map> m_eventHandlerRefs{}; bool m_isReady{false}; @@ -44,5 +62,19 @@ namespace Babylon::Plugins bool m_IsPlaying{}; Napi::ObjectReference m_streamObject{}; + + // Video-file (src) support. + JsRuntime* m_runtime{}; + std::unique_ptr m_videoPlayer{}; + std::string m_src{}; + bool m_loop{}; + bool m_muted{}; + bool m_ended{}; + uint32_t m_lastVideoWidth{}; + uint32_t m_lastVideoHeight{}; + // Keeps the element alive while it owns a media player (media elements with an + // active source aren't collected mid-playback) and makes the cross-thread event + // dispatches safe. + Napi::ObjectReference m_selfRef{}; }; }