Fix pixel density in AdaptiveStream

.changes/adaptive-stream-pixel-density

@@ -0,0 +1 @@
+patch type="fixed" "Fix adaptive stream dimensions on high-density displays"

lib/src/publication/remote.dart

@@ -162,19 +162,30 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
 
     final videoTrack = track as VideoTrack;
 
-    // filter visible build contexts
-    final viewSizes = videoTrack.viewKeys
-        .map((e) => e.currentContext)
+    // Filter visible build contexts and scale each view's logical size by its
+    // own pixel density, so the server is asked for physical-pixel dimensions
+    // (retina-aware). Each view's density is configured on its VideoTrackRenderer
+    // and resolved per-view; with AdaptiveStreamPixelDensity.auto the actual
+    // device pixel ratio is read from that view via MediaQuery. The largest
+    // resulting size across all of the track's views is requested.
+    final viewSizes = videoTrack.viewRegistrations
+        .map((registration) {
+          final context = registration.key.currentContext;
+          if (context == null) return null;
+          final renderBox = context.findRenderObject() as RenderBox?;
+          if (renderBox == null || !renderBox.hasSize) return null;
+          final density = registration.pixelDensity.resolve(
+            MediaQuery.maybeDevicePixelRatioOf(context) ?? 1.0,
+          );
+          return renderBox.size * density;
+        })
         .nonNulls
-        .map((e) => e.findRenderObject() as RenderBox?)
-        .nonNulls
-        .where((e) => e.hasSize)
-        .map((e) => e.size);
+        .toList();
 
     logger.finer('[Visibility] ${track?.sid} watching ${viewSizes.length} views...');
 
     if (viewSizes.isNotEmpty) {
-      final largestSize = viewSizes.reduce((value, element) => maxOfSizes(value, element));
+      final largestSize = viewSizes.reduce(maxOfSizes);
       _adaptiveStreamDimensions = VideoDimensions(
         largestSize.width.ceil(),
         largestSize.height.ceil(),
@@ -238,7 +249,7 @@ class RemoteTrackPublication<T extends RemoteTrack> extends TrackPublication<T>
           (_) => _computeVideoViewVisibility(),
         );
 
-        newValue.onVideoViewBuild = (_) {
+        newValue.onVideoViewBuild = () {
           logger.finer('[Visibility] VideoView did build');
           if (_lastSentTrackSettings?.disabled == true) {
             // quick enable

lib/src/track/local/local.dart

@@ -35,27 +35,32 @@ import '../processor_native.dart' if (dart.library.js_interop) '../processor_web
 import '../remote/audio.dart';
 import '../remote/video.dart';
 import '../track.dart';
+import '../video_track_view_registration.dart';
 import 'audio.dart';
 import 'video.dart';
 
 /// Used to group [LocalVideoTrack] and [RemoteVideoTrack].
 mixin VideoTrack on Track {
+  /// The views attached to this track. Set by [VideoTrackRenderer] and read by
+  /// the visibility observer to compute adaptive-stream dimensions.
   @internal
-  final List<GlobalKey> viewKeys = [];
+  final List<VideoTrackViewRegistration> viewRegistrations = [];
 
   @internal
-  Function(Key)? onVideoViewBuild;
+  VoidCallback? onVideoViewBuild;
 
   @internal
-  GlobalKey addViewKey() {
-    final key = GlobalKey();
-    viewKeys.add(key);
-    return key;
+  VideoTrackViewRegistration addViewRegistration({
+    AdaptiveStreamPixelDensity pixelDensity = AdaptiveStreamPixelDensity.auto,
+  }) {
+    final registration = VideoTrackViewRegistration(pixelDensity: pixelDensity);
+    viewRegistrations.add(registration);
+    return registration;
   }
 
   @internal
-  void removeViewKey(GlobalKey key) {
-    viewKeys.remove(key);
+  void removeViewRegistration(VideoTrackViewRegistration registration) {
+    viewRegistrations.remove(registration);
   }
 }
 

lib/src/track/video_track_view_registration.dart

@@ -0,0 +1,33 @@
+// Copyright 2024 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'package:flutter/material.dart';
+
+import 'package:meta/meta.dart';
+
+import '../types/other.dart';
+
+@internal
+class VideoTrackViewRegistration {
+  /// The widget key used by adaptive stream to find this view's render context.
+  final GlobalKey key = GlobalKey();
+
+  /// The pixel density used to convert this view's logical size to physical
+  /// pixels when computing adaptive-stream dimensions.
+  AdaptiveStreamPixelDensity pixelDensity;
+
+  VideoTrackViewRegistration({
+    this.pixelDensity = AdaptiveStreamPixelDensity.auto,
+  });
+}

lib/src/types/other.dart

@@ -233,3 +233,50 @@ class ParticipantTrackPermission {
     this.allowedTrackSids,
   );
 }
+
+/// Controls how a video view's logical size is scaled to physical pixels when
+/// computing adaptive-stream dimensions. Mirrors the JS SDK's `pixelDensity`
+/// option (`number | 'screen'`).
+///
+/// Server layers are sized in physical pixels, so on high-density (retina)
+/// displays the logical view size under-represents the pixels needed. Set on a
+/// view via [VideoTrackRenderer]; the largest result is requested across all
+/// views attached to the track.
+class AdaptiveStreamPixelDensity {
+  /// Upper bound applied to the resolved density to keep bandwidth in check.
+  static const maxDensity = 3.0;
+
+  /// Fixed multiplier, or `null` to use the view's device pixel ratio ([auto]).
+  final double? value;
+
+  const AdaptiveStreamPixelDensity._(this.value);
+
+  /// Use the view's actual device pixel ratio, read via `MediaQuery`.
+  /// Equivalent to the JS SDK's `'screen'` setting. Capped at [maxDensity].
+  static const auto = AdaptiveStreamPixelDensity._(null);
+
+  /// A positive fixed pixel-density multiplier (fractional allowed, e.g. `1.5`,
+  /// `2.0`, `2.75`). The effective value is capped at [maxDensity] (3x) when
+  /// resolved.
+  const AdaptiveStreamPixelDensity.fixed(double density)
+      : assert(density > 0, 'density must be positive'),
+        value = density;
+
+  /// Resolves the effective multiplier, capped at [maxDensity]. For [auto],
+  /// falls back to the supplied [devicePixelRatio].
+  double resolve(double devicePixelRatio) {
+    final density = value ?? devicePixelRatio;
+    if (density.isNaN || density <= 0) return 1.0;
+    return density > maxDensity ? maxDensity : density;
+  }
+
+  @override
+  bool operator ==(Object other) =>
+      identical(this, other) || (other is AdaptiveStreamPixelDensity && other.value == value);
+
+  @override
+  int get hashCode => value.hashCode;
+
+  @override
+  String toString() => value == null ? 'AdaptiveStreamPixelDensity.auto' : 'AdaptiveStreamPixelDensity.fixed($value)';
+}

lib/src/widgets/video_track_renderer.dart

@@ -29,6 +29,7 @@ import '../support/platform.dart';
 import '../track/local/local.dart';
 import '../track/local/video.dart';
 import '../track/options.dart';
+import '../track/video_track_view_registration.dart';
 import '../types/other.dart';
 
 enum VideoViewMirrorMode {
@@ -69,6 +70,12 @@ class VideoTrackRenderer extends StatefulWidget {
   /// wrap the video view in a Center widget (if [fit] is [VideoViewFit.contain])
   final bool autoCenter;
 
+  /// Controls how this view's logical size is converted to the physical-pixel
+  /// dimensions requested from the server when adaptive stream is enabled.
+  /// Defaults to [AdaptiveStreamPixelDensity.auto] (the view's own device pixel
+  /// ratio), avoiding an under-sized layer on retina / high-density displays.
+  final AdaptiveStreamPixelDensity adaptiveStreamPixelDensity;
+
   const VideoTrackRenderer(
     this.track, {
     this.fit = VideoViewFit.contain,
@@ -77,6 +84,7 @@ class VideoTrackRenderer extends StatefulWidget {
     this.autoDisposeRenderer = true,
     this.cachedRenderer,
     this.autoCenter = true,
+    this.adaptiveStreamPixelDensity = AdaptiveStreamPixelDensity.auto,
     Key? key,
   }) : super(key: key);
 
@@ -91,7 +99,7 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
   double? _aspectRatio;
   EventsListener<TrackEvent>? _listener;
   // Used to compute visibility information
-  late GlobalKey _internalKey;
+  late VideoTrackViewRegistration _viewRegistration;
 
   Future<rtc.VideoRenderer> _initializeRenderer() async {
     if (lkPlatformIs(PlatformType.iOS) && widget.renderMode == VideoRenderMode.platformView) {
@@ -142,7 +150,7 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
     if (widget.cachedRenderer != null) {
       _renderer = widget.cachedRenderer;
     }
-    _internalKey = widget.track.addViewKey();
+    _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity);
     if (kIsWeb) {
       unawaited(() async {
         await _initializeRenderer();
@@ -154,7 +162,7 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
 
   @override
   void dispose() {
-    widget.track.removeViewKey(_internalKey);
+    widget.track.removeViewRegistration(_viewRegistration);
     unawaited(_listener?.dispose());
     if (widget.autoDisposeRenderer) {
       disposeRenderer();
@@ -188,11 +196,13 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
   void didUpdateWidget(covariant VideoTrackRenderer oldWidget) {
     super.didUpdateWidget(oldWidget);
     if (widget.track != oldWidget.track) {
-      oldWidget.track.removeViewKey(_internalKey);
-      _internalKey = widget.track.addViewKey();
+      oldWidget.track.removeViewRegistration(_viewRegistration);
+      _viewRegistration = widget.track.addViewRegistration(pixelDensity: widget.adaptiveStreamPixelDensity);
       unawaited(() async {
         await _attach();
       }());
+    } else if (widget.adaptiveStreamPixelDensity != oldWidget.adaptiveStreamPixelDensity) {
+      _viewRegistration.pixelDensity = widget.adaptiveStreamPixelDensity;
     }
 
     if ([BrowserType.safari, BrowserType.firefox].contains(lkBrowser()) && oldWidget.key != widget.key) {
@@ -203,11 +213,11 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
   Widget _videoViewForWeb() => !_rendererReadyForWeb
       ? Container()
       : Builder(
-          key: _internalKey,
+          key: _viewRegistration.key,
           builder: (ctx) {
             // let it render before notifying build
             WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) {
-              widget.track.onVideoViewBuild?.call(_internalKey);
+              widget.track.onVideoViewBuild?.call();
             });
             return rtc.RTCVideoView(
               _renderer! as rtc.RTCVideoRenderer,
@@ -244,11 +254,11 @@ class _VideoTrackRendererState extends State<VideoTrackRenderer> {
         if ((snapshot.hasData && _renderer != null) ||
             (lkPlatformIs(PlatformType.iOS) && widget.renderMode == VideoRenderMode.platformView)) {
           return Builder(
-            key: _internalKey,
+            key: _viewRegistration.key,
             builder: (ctx) {
               // let it render before notifying build
               WidgetsBindingCompatible.instance?.addPostFrameCallback((timeStamp) {
-                widget.track.onVideoViewBuild?.call(_internalKey);
+                widget.track.onVideoViewBuild?.call();
               });
 
               if (!lkPlatformIsMobile() || widget.track is! LocalVideoTrack) {

test/options/adaptive_stream_pixel_density_test.dart

@@ -0,0 +1,76 @@
+// Copyright 2025 LiveKit, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import 'package:flutter_test/flutter_test.dart';
+
+import 'package:livekit_client/src/types/other.dart';
+
+void main() {
+  group('AdaptiveStreamPixelDensity.resolve', () {
+    test('fixed densities ignore the device pixel ratio', () {
+      expect(const AdaptiveStreamPixelDensity.fixed(1.0).resolve(3.0), 1.0);
+      expect(const AdaptiveStreamPixelDensity.fixed(2.0).resolve(1.0), 2.0);
+    });
+
+    test('fractional fixed densities are supported', () {
+      expect(const AdaptiveStreamPixelDensity.fixed(1.5).resolve(3.0), 1.5);
+      expect(const AdaptiveStreamPixelDensity.fixed(2.75).resolve(1.0), 2.75);
+    });
+
+    test('auto falls back to the supplied device pixel ratio', () {
+      expect(AdaptiveStreamPixelDensity.auto.resolve(1.0), 1.0);
+      expect(AdaptiveStreamPixelDensity.auto.resolve(2.0), 2.0);
+      expect(AdaptiveStreamPixelDensity.auto.resolve(2.625), 2.625);
+    });
+
+    test('caps at 3x for both fixed and auto', () {
+      expect(const AdaptiveStreamPixelDensity.fixed(4.0).resolve(1.0), 3.0);
+      expect(AdaptiveStreamPixelDensity.auto.resolve(4.0), 3.0);
+      expect(AdaptiveStreamPixelDensity.maxDensity, 3.0);
+    });
+
+    test('falls back for invalid auto device pixel ratios', () {
+      expect(AdaptiveStreamPixelDensity.auto.resolve(0), 1.0);
+      expect(AdaptiveStreamPixelDensity.auto.resolve(-2.0), 1.0);
+      expect(AdaptiveStreamPixelDensity.auto.resolve(double.nan), 1.0);
+    });
+
+    test('fixed densities must be positive', () {
+      expect(
+        () => AdaptiveStreamPixelDensity.fixed(0),
+        throwsA(isA<AssertionError>()),
+      );
+      expect(
+        () => AdaptiveStreamPixelDensity.fixed(-1.0),
+        throwsA(isA<AssertionError>()),
+      );
+    });
+
+    test('value is null only for auto', () {
+      expect(AdaptiveStreamPixelDensity.auto.value, isNull);
+      expect(const AdaptiveStreamPixelDensity.fixed(1.5).value, 1.5);
+    });
+
+    test('equality is by value', () {
+      expect(
+        const AdaptiveStreamPixelDensity.fixed(2.0),
+        const AdaptiveStreamPixelDensity.fixed(2.0),
+      );
+      expect(
+        const AdaptiveStreamPixelDensity.fixed(2.0) == AdaptiveStreamPixelDensity.auto,
+        isFalse,
+      );
+    });
+  });
+}