diff --git a/Plugins/NativeCamera/CMakeLists.txt b/Plugins/NativeCamera/CMakeLists.txt index c369f2f3d..72583db0e 100644 --- a/Plugins/NativeCamera/CMakeLists.txt +++ b/Plugins/NativeCamera/CMakeLists.txt @@ -19,10 +19,16 @@ set(SOURCES "Source/NativeCamera.cpp" "Source/NativeVideo.cpp" "Source/NativeVideo.h" + "Source/VideoPlayer.h" "Source/CameraDevice.h" "Source/CameraDeviceSharedPImpl.h" "Source/${PLATFORM}/CameraDevice.${BABYLON_NATIVE_PLATFORM_IMPL_EXT}") +if(APPLE) + # Video-file playback (HTMLVideoElement.src) backend - AVPlayer based. + list(APPEND SOURCES "Source/Apple/VideoPlayer.mm") +endif() + add_library(NativeCamera ${SOURCES}) warnings_as_errors(NativeCamera) diff --git a/Plugins/NativeCamera/Source/Apple/VideoPlayer.mm b/Plugins/NativeCamera/Source/Apple/VideoPlayer.mm new file mode 100644 index 000000000..d4417c14b --- /dev/null +++ b/Plugins/NativeCamera/Source/Apple/VideoPlayer.mm @@ -0,0 +1,391 @@ +#if ! __has_feature(objc_arc) +#error "ARC is off" +#endif + +// Apple implementation of VideoPlayer (see ../VideoPlayer.h) backed by AVPlayer + +// AVPlayerItemVideoOutput. Decoded frames arrive as BGRA pixel buffers (no YUV pass +// needed) and are blitted into a persistent MTLTexture, which is wired to Babylon's +// InternalTexture with the same bgfx::overrideInternal retry pattern used by +// CameraDevice::UpdateCameraTexture. + +#import +#import +#import +#import + +#include +#include +#include +#include +#include + +#include "../VideoPlayer.h" + +#include +#include + +@class VideoPlayerItemObserver; + +namespace Babylon::Plugins +{ + namespace + { + NSURL* ResolveUrl(const std::string& url) + { + NSString* str = [NSString stringWithUTF8String:url.c_str()]; + if ([str hasPrefix:@"app://"]) + { + // Match the app:/// scheme used by ScriptLoader: resolve against the bundle. + NSURL* parsed = [NSURL URLWithString:str]; + NSString* relativePath = parsed.path; // e.g. "/Scripts/video.mp4" + NSString* fullPath = [[[NSBundle mainBundle] resourcePath] stringByAppendingString:relativePath]; + return [NSURL fileURLWithPath:fullPath]; + } + if ([str hasPrefix:@"/"]) + { + return [NSURL fileURLWithPath:str]; + } + return [NSURL URLWithString:str]; + } + } + + class VideoPlayerApple final : public VideoPlayer + { + public: + // Shared with render-thread tasks and ObjC observers so in-flight work stays + // valid after the VideoPlayer is destroyed. + struct State + { + Graphics::DeviceContext* deviceContext{}; + EventCallbackT eventCallback{}; + + AVPlayer* player{}; + AVPlayerItem* item{}; + AVPlayerItemVideoOutput* videoOutput{}; + VideoPlayerItemObserver* observer{}; + id endObserverToken{}; + + // Metal resources (initialized lazily on the render thread). + id metalDevice{}; + id commandQueue{}; + CVMetalTextureCacheRef textureCache{}; + id textureRGBA{}; + + bgfx::TextureHandle bgfxHandle{bgfx::kInvalidHandle}; + bool refreshBgfxHandle{true}; + + std::atomic ready{false}; + std::atomic loop{false}; + std::atomic invalid{false}; + std::atomic width{0}; + std::atomic height{0}; + + ~State() + { + if (textureCache) + { + CVMetalTextureCacheFlush(textureCache, 0); + CFRelease(textureCache); + } + } + }; + + VideoPlayerApple(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback); + ~VideoPlayerApple() override; + + void Play() override + { + [m_state->player play]; + } + + void Pause() override + { + [m_state->player pause]; + } + + void SetLoop(bool loop) override + { + m_state->loop.store(loop); + } + + void SetMuted(bool muted) override + { + m_state->player.muted = muted; + } + + void SetVolume(double volume) override + { + m_state->player.volume = static_cast(volume); + } + + void Seek(double timeInSeconds) override + { + auto state = m_state; + [m_state->player seekToTime:CMTimeMakeWithSeconds(timeInSeconds, 600) + toleranceBefore:kCMTimeZero + toleranceAfter:kCMTimeZero + completionHandler:^(BOOL) { + if (!state->invalid.load() && state->eventCallback) + { + state->eventCallback(Event::Seeked); + } + }]; + } + + bool IsReady() const override + { + return m_state->ready.load(); + } + + double GetDuration() const override + { + if (!m_state->ready.load()) + { + return 0; + } + const CMTime duration = m_state->item.duration; + return CMTIME_IS_NUMERIC(duration) ? CMTimeGetSeconds(duration) : 0; + } + + double GetCurrentTime() const override + { + const CMTime time = m_state->player.currentTime; + return CMTIME_IS_NUMERIC(time) ? CMTimeGetSeconds(time) : 0; + } + + uint32_t GetWidth() const override + { + return m_state->width.load(); + } + + uint32_t GetHeight() const override + { + return m_state->height.load(); + } + + void UpdateTexture(bgfx::TextureHandle textureHandle) override; + + private: + std::shared_ptr m_state{}; + }; +} + +// Observes AVPlayerItem.status so we can report readiness (metadata/dimensions). +@interface VideoPlayerItemObserver : NSObject +- (id)initWithState:(std::shared_ptr)state; +- (void)stopObserving; +@end + +@implementation VideoPlayerItemObserver +{ + std::shared_ptr _state; + BOOL _observing; +} + +- (id)initWithState:(std::shared_ptr)state +{ + if (self = [super init]) + { + _state = std::move(state); + [_state->item addObserver:self forKeyPath:@"status" options:NSKeyValueObservingOptionNew context:nil]; + _observing = YES; + } + return self; +} + +- (void)stopObserving +{ + if (_observing) + { + [_state->item removeObserver:self forKeyPath:@"status"]; + _observing = NO; + } +} + +- (void)observeValueForKeyPath:(NSString*)keyPath ofObject:(id)__unused object change:(NSDictionary*)__unused change context:(void*)__unused context +{ + if ([keyPath isEqualToString:@"status"] && _state->item.status == AVPlayerItemStatusReadyToPlay && !_state->ready.load()) + { + const CGSize size = _state->item.presentationSize; + _state->width.store(static_cast(size.width)); + _state->height.store(static_cast(size.height)); + _state->ready.store(true); + if (!_state->invalid.load() && _state->eventCallback) + { + _state->eventCallback(Babylon::Plugins::VideoPlayer::Event::MetadataLoaded); + } + } +} +@end + +namespace Babylon::Plugins +{ + std::unique_ptr VideoPlayer::Create(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback) + { + return std::make_unique(deviceContext, url, std::move(eventCallback)); + } + + VideoPlayerApple::VideoPlayerApple(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback) + : m_state{std::make_shared()} + { + m_state->deviceContext = &deviceContext; + m_state->eventCallback = std::move(eventCallback); + + NSURL* nsUrl = ResolveUrl(url); + m_state->item = [AVPlayerItem playerItemWithURL:nsUrl]; + m_state->player = [AVPlayer playerWithPlayerItem:m_state->item]; + m_state->player.actionAtItemEnd = AVPlayerActionAtItemEndNone; + + // Decoded frames as BGRA so they can be blitted straight into the sampled texture. + NSDictionary* outputAttributes = @{ + (id)kCVPixelBufferPixelFormatTypeKey: @(kCVPixelFormatType_32BGRA), + (id)kCVPixelBufferMetalCompatibilityKey: @YES, + }; + m_state->videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:outputAttributes]; + [m_state->item addOutput:m_state->videoOutput]; + + m_state->observer = [[VideoPlayerItemObserver alloc] initWithState:m_state]; + + // End-of-playback: loop by seeking back to zero, otherwise report Ended. + auto state = m_state; + m_state->endObserverToken = [[NSNotificationCenter defaultCenter] addObserverForName:AVPlayerItemDidPlayToEndTimeNotification + object:m_state->item + queue:nil + usingBlock:^(NSNotification*) { + if (state->invalid.load()) + { + return; + } + if (state->loop.load()) + { + [state->player seekToTime:kCMTimeZero toleranceBefore:kCMTimeZero toleranceAfter:kCMTimeZero completionHandler:^(BOOL) {}]; + } + else + { + [state->player pause]; + if (state->eventCallback) + { + state->eventCallback(Event::Ended); + } + } + }]; + } + + VideoPlayerApple::~VideoPlayerApple() + { + m_state->invalid.store(true); + [m_state->observer stopObserving]; + if (m_state->endObserverToken) + { + [[NSNotificationCenter defaultCenter] removeObserver:m_state->endObserverToken]; + m_state->endObserverToken = nil; + } + [m_state->player pause]; + // In-flight render-thread tasks hold the shared State and finish safely. + } + + void VideoPlayerApple::UpdateTexture(bgfx::TextureHandle textureHandle) + { + if (!m_state->ready.load()) + { + return; + } + + // Hook into AfterRender (like CameraDevice::UpdateCameraTexture) so the bgfx + // handle has been initialized by a frame before we override it, and so Metal + // work doesn't race the renderer. + arcana::make_task(m_state->deviceContext->AfterRenderScheduler(), arcana::cancellation::none(), [state = m_state, textureHandle] { + if (state->invalid.load()) + { + return; + } + + // Lazily bind to bgfx's Metal device/queue (render thread only). + if (state->metalDevice == nil) + { + state->metalDevice = (__bridge id)bgfx::getInternalData()->context; + state->commandQueue = (__bridge id)bgfx::getInternalData()->commandQueue; + CVMetalTextureCacheCreate(nullptr, nullptr, state->metalDevice, nullptr, &state->textureCache); + } + + if (state->bgfxHandle.idx != textureHandle.idx) + { + state->bgfxHandle = textureHandle; + state->refreshBgfxHandle = true; + } + + const CMTime itemTime = [state->videoOutput itemTimeForHostTime:CACurrentMediaTime()]; + if ([state->videoOutput hasNewPixelBufferForItemTime:itemTime]) + { + CVPixelBufferRef pixelBuffer = [state->videoOutput copyPixelBufferForItemTime:itemTime itemTimeForDisplay:nil]; + if (pixelBuffer != nullptr) + { + const size_t width = CVPixelBufferGetWidth(pixelBuffer); + const size_t height = CVPixelBufferGetHeight(pixelBuffer); + state->width.store(static_cast(width)); + state->height.store(static_cast(height)); + + // (Re)create the persistent sampled texture on size change. + if (state->textureRGBA == nil || state->textureRGBA.width != width || state->textureRGBA.height != height) + { + MTLTextureDescriptor* descriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm + width:width + height:height + mipmapped:NO]; + descriptor.usage = MTLTextureUsageShaderRead; + state->textureRGBA = [state->metalDevice newTextureWithDescriptor:descriptor]; + state->refreshBgfxHandle = true; + } + + // Zero-copy Metal view of the decoded BGRA pixel buffer. + CVMetalTextureRef sourceTextureRef{}; + CVMetalTextureCacheCreateTextureFromImage(nullptr, state->textureCache, pixelBuffer, nullptr, + MTLPixelFormatBGRA8Unorm, width, height, 0, &sourceTextureRef); + + if (sourceTextureRef != nullptr) + { + id sourceTexture = CVMetalTextureGetTexture(sourceTextureRef); + if (sourceTexture != nil) + { + id commandBuffer = [state->commandQueue commandBuffer]; + commandBuffer.label = @"NativeVideoPlayerBlit"; + id blit = [commandBuffer blitCommandEncoder]; + [blit copyFromTexture:sourceTexture + sourceSlice:0 + sourceLevel:0 + sourceOrigin:MTLOriginMake(0, 0, 0) + sourceSize:MTLSizeMake(width, height, 1) + toTexture:state->textureRGBA + destinationSlice:0 + destinationLevel:0 + destinationOrigin:MTLOriginMake(0, 0, 0)]; + [blit endEncoding]; + [commandBuffer addCompletedHandler:^(id) { + // Keep the pixel buffer + texture view alive until the GPU is done. + CFRelease(sourceTextureRef); + CVPixelBufferRelease(pixelBuffer); + }]; + [commandBuffer commit]; + } + else + { + CFRelease(sourceTextureRef); + CVPixelBufferRelease(pixelBuffer); + } + } + else + { + CVPixelBufferRelease(pixelBuffer); + } + } + } + + // Wire (or re-wire) Babylon's texture handle to our persistent texture. May + // fail until the handle has been initialized by bgfx; retried next frame. + if (state->textureRGBA != nil && state->refreshBgfxHandle && bgfx::isValid(state->bgfxHandle)) + { + state->refreshBgfxHandle = bgfx::overrideInternal(state->bgfxHandle, reinterpret_cast(state->textureRGBA)) == 0; + } + }); + } +} diff --git a/Plugins/NativeCamera/Source/NativeVideo.cpp b/Plugins/NativeCamera/Source/NativeVideo.cpp index 5198f357d..587264fec 100644 --- a/Plugins/NativeCamera/Source/NativeVideo.cpp +++ b/Plugins/NativeCamera/Source/NativeVideo.cpp @@ -22,6 +22,14 @@ namespace Babylon::Plugins InstanceAccessor("readyState", &NativeVideo::GetReadyState, nullptr), InstanceAccessor("HAVE_CURRENT_DATA", &NativeVideo::GetHaveCurrentData, nullptr), InstanceAccessor("srcObject", &NativeVideo::GetSrcObject, &NativeVideo::SetSrcObject), + InstanceAccessor("src", &NativeVideo::GetSrc, &NativeVideo::SetSrc), + InstanceAccessor("isNative", &NativeVideo::GetIsNative, nullptr), + InstanceAccessor("currentTime", &NativeVideo::GetCurrentTime, &NativeVideo::SetCurrentTime), + InstanceAccessor("duration", &NativeVideo::GetDuration, nullptr), + InstanceAccessor("loop", &NativeVideo::GetLoop, &NativeVideo::SetLoop), + InstanceAccessor("muted", &NativeVideo::GetMuted, &NativeVideo::SetMuted), + InstanceAccessor("paused", &NativeVideo::GetPaused, nullptr), + InstanceAccessor("ended", &NativeVideo::GetEnded, nullptr), }); env.Global().Set(JS_CLASS_NAME, func); @@ -34,11 +42,17 @@ namespace Babylon::Plugins NativeVideo::NativeVideo(const Napi::CallbackInfo& info) : Napi::ObjectWrap{info} + , m_runtime{&JsRuntime::GetFromJavaScript(info.Env())} { } Napi::Value NativeVideo::GetVideoWidth(const Napi::CallbackInfo& /*info*/) { + if (m_videoPlayer != nullptr) + { + return Napi::Value::From(Env(), m_videoPlayer->GetWidth()); + } + if (!m_streamObject.Value().IsNull() && !m_streamObject.Value().IsUndefined()) { return Napi::Value::From(Env(), MediaStream::Unwrap(m_streamObject.Value())->Width); @@ -49,6 +63,11 @@ namespace Babylon::Plugins Napi::Value NativeVideo::GetVideoHeight(const Napi::CallbackInfo& /*info*/) { + if (m_videoPlayer != nullptr) + { + return Napi::Value::From(Env(), m_videoPlayer->GetHeight()); + } + if (!m_streamObject.Value().IsNull() && !m_streamObject.Value().IsUndefined()) { return Napi::Value::From(Env(), MediaStream::Unwrap(m_streamObject.Value())->Height); @@ -88,6 +107,21 @@ namespace Babylon::Plugins RaiseEvent("resize"); } } + // Video-file playback (src). Not gated on m_IsPlaying so a seek while paused + // still refreshes the current frame; the player is a cheap no-op when idle. + else if (m_videoPlayer != nullptr && m_videoPlayer->IsReady()) + { + m_videoPlayer->UpdateTexture(textureHandle); + + const uint32_t width = m_videoPlayer->GetWidth(); + const uint32_t height = m_videoPlayer->GetHeight(); + if (width != m_lastVideoWidth || height != m_lastVideoHeight) + { + m_lastVideoWidth = width; + m_lastVideoHeight = height; + RaiseEvent("resize"); + } + } } void NativeVideo::AddEventListener(const Napi::CallbackInfo& info) @@ -149,6 +183,14 @@ namespace Babylon::Plugins m_IsPlaying = true; RaiseEvent("playing"); } + else if (!m_IsPlaying && m_videoPlayer != nullptr) + { + m_IsPlaying = true; + m_ended = false; + // Safe to call before the player is ready; playback starts once buffered. + m_videoPlayer->Play(); + RaiseEvent("playing"); + } deferred.Resolve(env.Undefined()); @@ -157,7 +199,18 @@ namespace Babylon::Plugins void NativeVideo::Pause(const Napi::CallbackInfo& /*info*/) { + const bool wasPlaying = m_IsPlaying; m_IsPlaying = false; + if (m_videoPlayer != nullptr) + { + m_videoPlayer->Pause(); + if (wasPlaying) + { + RaiseEvent("pause"); + // Babylon.js's VideoTexture listens for "paused" (its own convention). + RaiseEvent("paused"); + } + } } void NativeVideo::SetSrcObject(const Napi::CallbackInfo& info, const Napi::Value& value) @@ -172,7 +225,10 @@ namespace Babylon::Plugins return; } - // We've received a MediaStream object + // We've received a MediaStream object. A media element has a single source: + // assigning a stream replaces any file player. + m_videoPlayer.reset(); + m_src.clear(); m_streamObject = Napi::Persistent(value.As()); this->m_isReady = true; @@ -185,4 +241,141 @@ namespace Babylon::Plugins // If the streamObject has not yet been defined return Null instead to match the web API return m_streamObject.IsEmpty() ? info.Env().Null() : m_streamObject.Value(); } + + Napi::Value NativeVideo::GetSrc(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_src); + } + + void NativeVideo::SetSrc(const Napi::CallbackInfo& info, const Napi::Value& value) + { + m_videoPlayer.reset(); + m_isReady = false; + m_IsPlaying = false; + m_ended = false; + m_lastVideoWidth = 0; + m_lastVideoHeight = 0; + m_src.clear(); + + if (value.IsNull() || value.IsUndefined() || !value.IsString()) + { + RaiseEvent("emptied"); + return; + } + + m_src = value.As().Utf8Value(); + if (m_src.empty()) + { + RaiseEvent("emptied"); + return; + } + + // A media element with an active source is kept alive (and makes the + // cross-thread event dispatches below safe). + m_selfRef = Napi::Persistent(info.This().As()); + + // Assigning a file source replaces any stream source. + m_streamObject = Napi::ObjectReference(); + +#ifdef __APPLE__ + auto& deviceContext = Graphics::DeviceContext::GetFromJavaScript(info.Env()); + m_videoPlayer = VideoPlayer::Create(deviceContext, m_src, [this](VideoPlayer::Event event) { + // May be called on any thread — marshal to the JS thread. m_selfRef keeps + // `this` alive for the lifetime of the dispatch. + m_runtime->Dispatch([this, event](Napi::Env) { + switch (event) + { + case VideoPlayer::Event::MetadataLoaded: + m_isReady = true; + RaiseEvent("loadedmetadata"); + RaiseEvent("loadeddata"); + RaiseEvent("canplay"); + // play() may have been called before the media was ready. + if (m_IsPlaying && m_videoPlayer != nullptr) + { + m_videoPlayer->Play(); + } + break; + case VideoPlayer::Event::Ended: + m_IsPlaying = false; + m_ended = true; + RaiseEvent("ended"); + break; + case VideoPlayer::Event::Seeked: + RaiseEvent("seeked"); + break; + } + }); + }); + if (m_videoPlayer != nullptr) + { + m_videoPlayer->SetLoop(m_loop); + m_videoPlayer->SetMuted(m_muted); + } +#endif + } + + Napi::Value NativeVideo::GetIsNative(const Napi::CallbackInfo& info) + { + // Lets Babylon.js's VideoTexture accept this element directly (its _getVideo + // returns native elements as-is instead of calling document.createElement). + return Napi::Value::From(info.Env(), true); + } + + Napi::Value NativeVideo::GetCurrentTime(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_videoPlayer != nullptr ? m_videoPlayer->GetCurrentTime() : 0.0); + } + + void NativeVideo::SetCurrentTime(const Napi::CallbackInfo& /*info*/, const Napi::Value& value) + { + if (m_videoPlayer != nullptr && value.IsNumber()) + { + m_ended = false; + m_videoPlayer->Seek(value.As().DoubleValue()); + } + } + + Napi::Value NativeVideo::GetDuration(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_videoPlayer != nullptr ? m_videoPlayer->GetDuration() : 0.0); + } + + Napi::Value NativeVideo::GetLoop(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_loop); + } + + void NativeVideo::SetLoop(const Napi::CallbackInfo& /*info*/, const Napi::Value& value) + { + m_loop = value.ToBoolean(); + if (m_videoPlayer != nullptr) + { + m_videoPlayer->SetLoop(m_loop); + } + } + + Napi::Value NativeVideo::GetMuted(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_muted); + } + + void NativeVideo::SetMuted(const Napi::CallbackInfo& /*info*/, const Napi::Value& value) + { + m_muted = value.ToBoolean(); + if (m_videoPlayer != nullptr) + { + m_videoPlayer->SetMuted(m_muted); + } + } + + Napi::Value NativeVideo::GetPaused(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), !m_IsPlaying); + } + + Napi::Value NativeVideo::GetEnded(const Napi::CallbackInfo& info) + { + return Napi::Value::From(info.Env(), m_ended); + } } diff --git a/Plugins/NativeCamera/Source/NativeVideo.h b/Plugins/NativeCamera/Source/NativeVideo.h index ab8e27ca8..3d2546c5e 100644 --- a/Plugins/NativeCamera/Source/NativeVideo.h +++ b/Plugins/NativeCamera/Source/NativeVideo.h @@ -2,6 +2,7 @@ #include #include "CameraDevice.h" #include "MediaStream.h" +#include "VideoPlayer.h" #include #include #include @@ -10,8 +11,11 @@ namespace Babylon::Plugins { - // NativeVideo provides a polyfill for the HTMLVideoElement which is used in conjunction with a MediaStream object to - // pull frames out of a camera and render them into the Babylon scene. + // NativeVideo provides a polyfill for the HTMLVideoElement. It supports two media + // sources, matching the web API: + // - srcObject: a MediaStream (camera frames via CameraDevice) + // - src: a video file/URL, decoded by a per-instance VideoPlayer (AVPlayer on Apple) + // In both cases frames are pulled into the Babylon scene through UpdateTexture. class NativeVideo : public Napi::ObjectWrap { public: @@ -37,6 +41,20 @@ namespace Babylon::Plugins Napi::Value GetSrcObject(const Napi::CallbackInfo& info); void SetSrcObject(const Napi::CallbackInfo& info, const Napi::Value& value); + // Video-file (src) support. + Napi::Value GetSrc(const Napi::CallbackInfo& info); + void SetSrc(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetIsNative(const Napi::CallbackInfo& info); + Napi::Value GetCurrentTime(const Napi::CallbackInfo& info); + void SetCurrentTime(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetDuration(const Napi::CallbackInfo& info); + Napi::Value GetLoop(const Napi::CallbackInfo& info); + void SetLoop(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetMuted(const Napi::CallbackInfo& info); + void SetMuted(const Napi::CallbackInfo& info, const Napi::Value& value); + Napi::Value GetPaused(const Napi::CallbackInfo& info); + Napi::Value GetEnded(const Napi::CallbackInfo& info); + std::unordered_map> m_eventHandlerRefs{}; bool m_isReady{false}; @@ -44,5 +62,19 @@ namespace Babylon::Plugins bool m_IsPlaying{}; Napi::ObjectReference m_streamObject{}; + + // Video-file (src) support. + JsRuntime* m_runtime{}; + std::unique_ptr m_videoPlayer{}; + std::string m_src{}; + bool m_loop{}; + bool m_muted{}; + bool m_ended{}; + uint32_t m_lastVideoWidth{}; + uint32_t m_lastVideoHeight{}; + // Keeps the element alive while it owns a media player (media elements with an + // active source aren't collected mid-playback) and makes the cross-thread event + // dispatches safe. + Napi::ObjectReference m_selfRef{}; }; } diff --git a/Plugins/NativeCamera/Source/VideoPlayer.h b/Plugins/NativeCamera/Source/VideoPlayer.h new file mode 100644 index 000000000..395fc0964 --- /dev/null +++ b/Plugins/NativeCamera/Source/VideoPlayer.h @@ -0,0 +1,61 @@ +#pragma once + +// Video-file playback backend for NativeVideo (HTMLVideoElement.src support). +// +// Each instance owns a platform media player (AVPlayer on Apple) and a persistent +// GPU texture that receives decoded frames. The texture is wired to Babylon.js's +// VideoTexture with the same bgfx createTexture2D-less overrideInternal pattern +// used by CameraDevice: Babylon creates the InternalTexture (createDynamicTexture) +// and NativeVideo::UpdateTexture hands its bgfx handle here every frame. +// +// Event callbacks may fire on ANY thread — callers must marshal to the JS thread. + +#include +#include +#include +#include + +namespace Babylon::Graphics +{ + class DeviceContext; +} + +namespace Babylon::Plugins +{ + class VideoPlayer + { + public: + enum class Event + { + MetadataLoaded, // duration/dimensions known; playback can start + Ended, // reached the end (not fired when looping) + Seeked, // a Seek() completed + }; + + using EventCallbackT = std::function; + + // Returns nullptr on platforms without an implementation. + // Supported url forms: app:///, absolute file paths, http(s) URLs. + static std::unique_ptr Create(Graphics::DeviceContext& deviceContext, const std::string& url, EventCallbackT eventCallback); + + virtual ~VideoPlayer() = default; + + virtual void Play() = 0; + virtual void Pause() = 0; + virtual void SetLoop(bool loop) = 0; + virtual void SetMuted(bool muted) = 0; + virtual void SetVolume(double volume) = 0; + virtual void Seek(double timeInSeconds) = 0; + + virtual bool IsReady() const = 0; + virtual double GetDuration() const = 0; + virtual double GetCurrentTime() const = 0; + virtual uint32_t GetWidth() const = 0; + virtual uint32_t GetHeight() const = 0; + + // Pull the latest decoded frame into the persistent texture and (re)wire it to + // the given bgfx texture handle. Called on the JS thread once per render frame; + // the GPU work runs on the render thread. Cheap no-op when no new frame exists. + virtual void UpdateTexture(bgfx::TextureHandle textureHandle) = 0; + }; +}