From c59522b3ca58da3f696879c1879707b4c08b2c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Thu, 5 Jun 2025 23:12:54 +0200 Subject: [PATCH 1/8] server: crop using display projection --- .../scrcpy/video/ScreenCapture.java | 80 +++++++++++++------ 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index 5f4e1803f9..e9bb3c03e0 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -43,8 +43,10 @@ public class ScreenCapture extends SurfaceCapture { private IBinder display; private VirtualDisplay virtualDisplay; + private AffineMatrix glTransform; private AffineMatrix transform; private OpenGLRunner glRunner; + private Rect cropInDevice; public ScreenCapture(VirtualDisplayListener vdListener, Options options) { this.vdListener = vdListener; @@ -85,19 +87,38 @@ public void prepare() throws ConfigurationException { captureOrientation = Orientation.fromRotation(displayInfo.getRotation()); } - VideoFilter filter = new VideoFilter(displaySize); + boolean transposed = (displayInfo.getRotation() % 2) != 0; - if (crop != null) { - boolean transposed = (displayInfo.getRotation() % 2) != 0; - filter.addCrop(crop, transposed); + Rect cropRect = crop; + if (cropRect != null && transposed) { + cropRect = new Rect(cropRect.top, cropRect.left, cropRect.bottom, cropRect.right); } + Size baseSize = displaySize; + if (cropRect != null) { + baseSize = new Size(cropRect.width(), cropRect.height()); + } + + // Filter used for the OpenGL transform (orientation/angle, without crop) + VideoFilter glFilter = new VideoFilter(baseSize); + boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; - filter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - filter.addAngle(angle); + glFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + glFilter.addAngle(angle); + + glTransform = glFilter.getInverseTransform(); + videoSize = glFilter.getOutputSize().limit(maxSize).round8(); + + // Filter used for event positions (includes crop) + VideoFilter pmFilter = new VideoFilter(displaySize); + if (cropRect != null) { + pmFilter.addCrop(cropRect, false); + } + pmFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + pmFilter.addAngle(angle); - transform = filter.getInverseTransform(); - videoSize = filter.getOutputSize().limit(maxSize).round8(); + transform = pmFilter.getInverseTransform(); + cropInDevice = cropRect; } @Override @@ -112,33 +133,44 @@ public void start(Surface surface) throws IOException { } Size inputSize; - if (transform != null) { - // If there is a filter, it must receive the full display content - inputSize = displayInfo.getSize(); + if (glTransform != null) { + inputSize = cropInDevice != null ? new Size(cropInDevice.width(), cropInDevice.height()) : displayInfo.getSize(); assert glRunner == null; - OpenGLFilter glFilter = new AffineOpenGLFilter(transform); + OpenGLFilter glFilter = new AffineOpenGLFilter(glTransform); glRunner = new OpenGLRunner(glFilter); surface = glRunner.start(inputSize, videoSize, surface); } else { - // If there is no filter, the display must be rendered at target video size directly - inputSize = videoSize; + inputSize = cropInDevice != null ? new Size(cropInDevice.width(), cropInDevice.height()) : videoSize; } - try { - virtualDisplay = ServiceManager.getDisplayManager() - .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); - Ln.d("Display: using DisplayManager API"); - } catch (Exception displayManagerException) { + if (cropInDevice == null) { + try { + virtualDisplay = ServiceManager.getDisplayManager() + .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); + Ln.d("Display: using DisplayManager API"); + } catch (Exception displayManagerException) { + try { + display = createDisplay(); + + Size deviceSize = displayInfo.getSize(); + int layerStack = displayInfo.getLayerStack(); + setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); + Ln.d("Display: using SurfaceControl API"); + } catch (Exception surfaceControlException) { + Ln.e("Could not create display using DisplayManager", displayManagerException); + Ln.e("Could not create display using SurfaceControl", surfaceControlException); + throw new AssertionError("Could not create display"); + } + } + } else { try { display = createDisplay(); - Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); - Ln.d("Display: using SurfaceControl API"); + setDisplaySurface(display, surface, cropInDevice, inputSize.toRect(), layerStack); + Ln.d("Display: using SurfaceControl API (crop)"); } catch (Exception surfaceControlException) { - Ln.e("Could not create display using DisplayManager", displayManagerException); - Ln.e("Could not create display using SurfaceControl", surfaceControlException); + Ln.e("Could not create cropped display", surfaceControlException); throw new AssertionError("Could not create display"); } } From 9c491c18e9c173cf0d20fd80a624eb429e25f85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Thu, 5 Jun 2025 23:36:50 +0200 Subject: [PATCH 2/8] fix quest cropping with proper projection rotation --- .../java/com/genymobile/scrcpy/video/ScreenCapture.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index e9bb3c03e0..aa2a33a4a3 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -154,7 +154,7 @@ public void start(Surface surface) throws IOException { Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); + setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack, 0); Ln.d("Display: using SurfaceControl API"); } catch (Exception surfaceControlException) { Ln.e("Could not create display using DisplayManager", displayManagerException); @@ -167,7 +167,7 @@ public void start(Surface surface) throws IOException { display = createDisplay(); int layerStack = displayInfo.getLayerStack(); - setDisplaySurface(display, surface, cropInDevice, inputSize.toRect(), layerStack); + setDisplaySurface(display, surface, cropInDevice, inputSize.toRect(), layerStack, displayInfo.getRotation()); Ln.d("Display: using SurfaceControl API (crop)"); } catch (Exception surfaceControlException) { Ln.e("Could not create cropped display", surfaceControlException); @@ -233,11 +233,11 @@ private static IBinder createDisplay() throws Exception { return SurfaceControl.createDisplay("scrcpy", secure); } - private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack) { + private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack, int orientation) { SurfaceControl.openTransaction(); try { SurfaceControl.setDisplaySurface(display, surface); - SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect); + SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); SurfaceControl.setDisplayLayerStack(display, layerStack); } finally { SurfaceControl.closeTransaction(); From 0ba9e547e06eb51886c4a6ef9b99230e6c6e3a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Thu, 5 Jun 2025 23:55:26 +0200 Subject: [PATCH 3/8] Reimplement cropping using OpenGL --- .../scrcpy/video/ScreenCapture.java | 90 +++++++------------ 1 file changed, 31 insertions(+), 59 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index aa2a33a4a3..f558ad6bc8 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -43,10 +43,8 @@ public class ScreenCapture extends SurfaceCapture { private IBinder display; private VirtualDisplay virtualDisplay; - private AffineMatrix glTransform; private AffineMatrix transform; private OpenGLRunner glRunner; - private Rect cropInDevice; public ScreenCapture(VirtualDisplayListener vdListener, Options options) { this.vdListener = vdListener; @@ -87,38 +85,23 @@ public void prepare() throws ConfigurationException { captureOrientation = Orientation.fromRotation(displayInfo.getRotation()); } - boolean transposed = (displayInfo.getRotation() % 2) != 0; - - Rect cropRect = crop; - if (cropRect != null && transposed) { - cropRect = new Rect(cropRect.top, cropRect.left, cropRect.bottom, cropRect.right); - } - - Size baseSize = displaySize; - if (cropRect != null) { - baseSize = new Size(cropRect.width(), cropRect.height()); - } - - // Filter used for the OpenGL transform (orientation/angle, without crop) - VideoFilter glFilter = new VideoFilter(baseSize); + VideoFilter filter = new VideoFilter(displaySize); boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; - glFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - glFilter.addAngle(angle); - - glTransform = glFilter.getInverseTransform(); - videoSize = glFilter.getOutputSize().limit(maxSize).round8(); - - // Filter used for event positions (includes crop) - VideoFilter pmFilter = new VideoFilter(displaySize); - if (cropRect != null) { - pmFilter.addCrop(cropRect, false); + filter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + filter.addAngle(angle); + + if (crop != null) { + boolean transposed = (displayInfo.getRotation() % 2) != 0; + Rect cropRect = crop; + if (transposed) { + cropRect = new Rect(crop.top, crop.left, crop.bottom, crop.right); + } + filter.addCrop(cropRect, false); } - pmFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - pmFilter.addAngle(angle); - transform = pmFilter.getInverseTransform(); - cropInDevice = cropRect; + transform = filter.getInverseTransform(); + videoSize = filter.getOutputSize().limit(maxSize).round8(); } @Override @@ -133,44 +116,33 @@ public void start(Surface surface) throws IOException { } Size inputSize; - if (glTransform != null) { - inputSize = cropInDevice != null ? new Size(cropInDevice.width(), cropInDevice.height()) : displayInfo.getSize(); + if (transform != null) { + // If there is a filter, it must receive the full display content + inputSize = displayInfo.getSize(); assert glRunner == null; - OpenGLFilter glFilter = new AffineOpenGLFilter(glTransform); + OpenGLFilter glFilter = new AffineOpenGLFilter(transform); glRunner = new OpenGLRunner(glFilter); surface = glRunner.start(inputSize, videoSize, surface); } else { - inputSize = cropInDevice != null ? new Size(cropInDevice.width(), cropInDevice.height()) : videoSize; + // If there is no filter, the display must be rendered at target video size directly + inputSize = videoSize; } - if (cropInDevice == null) { - try { - virtualDisplay = ServiceManager.getDisplayManager() - .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); - Ln.d("Display: using DisplayManager API"); - } catch (Exception displayManagerException) { - try { - display = createDisplay(); - - Size deviceSize = displayInfo.getSize(); - int layerStack = displayInfo.getLayerStack(); - setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack, 0); - Ln.d("Display: using SurfaceControl API"); - } catch (Exception surfaceControlException) { - Ln.e("Could not create display using DisplayManager", displayManagerException); - Ln.e("Could not create display using SurfaceControl", surfaceControlException); - throw new AssertionError("Could not create display"); - } - } - } else { + try { + virtualDisplay = ServiceManager.getDisplayManager() + .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); + Ln.d("Display: using DisplayManager API"); + } catch (Exception displayManagerException) { try { display = createDisplay(); + Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - setDisplaySurface(display, surface, cropInDevice, inputSize.toRect(), layerStack, displayInfo.getRotation()); - Ln.d("Display: using SurfaceControl API (crop)"); + setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); + Ln.d("Display: using SurfaceControl API"); } catch (Exception surfaceControlException) { - Ln.e("Could not create cropped display", surfaceControlException); + Ln.e("Could not create display using DisplayManager", displayManagerException); + Ln.e("Could not create display using SurfaceControl", surfaceControlException); throw new AssertionError("Could not create display"); } } @@ -233,11 +205,11 @@ private static IBinder createDisplay() throws Exception { return SurfaceControl.createDisplay("scrcpy", secure); } - private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack, int orientation) { + private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack) { SurfaceControl.openTransaction(); try { SurfaceControl.setDisplaySurface(display, surface); - SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); + SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect); SurfaceControl.setDisplayLayerStack(display, layerStack); } finally { SurfaceControl.closeTransaction(); From 62527b15e92812ab4479f5b2607786da45e8c5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Fri, 6 Jun 2025 00:02:10 +0200 Subject: [PATCH 4/8] Fix crop pipeline and projection orientation --- .../scrcpy/video/ScreenCapture.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index f558ad6bc8..9d14951e43 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -87,19 +87,15 @@ public void prepare() throws ConfigurationException { VideoFilter filter = new VideoFilter(displaySize); - boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; - filter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - filter.addAngle(angle); - if (crop != null) { boolean transposed = (displayInfo.getRotation() % 2) != 0; - Rect cropRect = crop; - if (transposed) { - cropRect = new Rect(crop.top, crop.left, crop.bottom, crop.right); - } - filter.addCrop(cropRect, false); + filter.addCrop(crop, transposed); } + boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; + filter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + filter.addAngle(angle); + transform = filter.getInverseTransform(); videoSize = filter.getOutputSize().limit(maxSize).round8(); } @@ -138,7 +134,8 @@ public void start(Surface surface) throws IOException { Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); + int orientation = crop != null ? displayInfo.getRotation() : 0; + setDisplaySurface(display, surface, orientation, deviceSize.toRect(), inputSize.toRect(), layerStack); Ln.d("Display: using SurfaceControl API"); } catch (Exception surfaceControlException) { Ln.e("Could not create display using DisplayManager", displayManagerException); @@ -205,11 +202,11 @@ private static IBinder createDisplay() throws Exception { return SurfaceControl.createDisplay("scrcpy", secure); } - private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack) { + private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) { SurfaceControl.openTransaction(); try { SurfaceControl.setDisplaySurface(display, surface); - SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect); + SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); SurfaceControl.setDisplayLayerStack(display, layerStack); } finally { SurfaceControl.closeTransaction(); From 60e048af2db333913d0a4205d21b95fb304c4d46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Fri, 6 Jun 2025 00:02:14 +0200 Subject: [PATCH 5/8] Refactor cropping pipeline --- .../scrcpy/video/ScreenCapture.java | 82 +++++++++++++------ 1 file changed, 59 insertions(+), 23 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index 9d14951e43..ae4b061748 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -43,7 +43,9 @@ public class ScreenCapture extends SurfaceCapture { private IBinder display; private VirtualDisplay virtualDisplay; - private AffineMatrix transform; + private AffineMatrix transform; // OpenGL transform (orientation/angle) + private AffineMatrix eventTransform; // full transform used for input events + private Rect orientedCrop; private OpenGLRunner glRunner; public ScreenCapture(VirtualDisplayListener vdListener, Options options) { @@ -85,19 +87,32 @@ public void prepare() throws ConfigurationException { captureOrientation = Orientation.fromRotation(displayInfo.getRotation()); } - VideoFilter filter = new VideoFilter(displaySize); + boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; + // Compute the oriented crop to apply on the display + orientedCrop = null; if (crop != null) { boolean transposed = (displayInfo.getRotation() % 2) != 0; - filter.addCrop(crop, transposed); + orientedCrop = transposed ? new Rect(crop.top, crop.left, crop.bottom, crop.right) + : new Rect(crop); } - boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; - filter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - filter.addAngle(angle); + // Compute the transform for input events (includes crop) + VideoFilter eventFilter = new VideoFilter(displaySize); + if (orientedCrop != null) { + eventFilter.addCrop(orientedCrop, false); + } + eventFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + eventFilter.addAngle(angle); + + eventTransform = eventFilter.getInverseTransform(); + videoSize = eventFilter.getOutputSize().limit(maxSize).round8(); - transform = filter.getInverseTransform(); - videoSize = filter.getOutputSize().limit(maxSize).round8(); + // Compute the transform for the OpenGL pipeline (no crop) + VideoFilter glFilter = new VideoFilter(displaySize); + glFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + glFilter.addAngle(angle); + transform = glFilter.getInverseTransform(); } @Override @@ -113,29 +128,44 @@ public void start(Surface surface) throws IOException { Size inputSize; if (transform != null) { - // If there is a filter, it must receive the full display content - inputSize = displayInfo.getSize(); + // If there is a filter, it must receive the content in the display orientation + if (orientedCrop != null) { + inputSize = new Size(orientedCrop.width(), orientedCrop.height()); + } else { + inputSize = displayInfo.getSize(); + } assert glRunner == null; OpenGLFilter glFilter = new AffineOpenGLFilter(transform); glRunner = new OpenGLRunner(glFilter); surface = glRunner.start(inputSize, videoSize, surface); } else { - // If there is no filter, the display must be rendered at target video size directly - inputSize = videoSize; + if (orientedCrop != null) { + inputSize = new Size(orientedCrop.width(), orientedCrop.height()); + } else { + inputSize = videoSize; + } } - try { - virtualDisplay = ServiceManager.getDisplayManager() - .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); - Ln.d("Display: using DisplayManager API"); - } catch (Exception displayManagerException) { + boolean useDisplayManager = orientedCrop == null; + Exception displayManagerException = null; + if (useDisplayManager) { + try { + virtualDisplay = ServiceManager.getDisplayManager() + .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); + Ln.d("Display: using DisplayManager API"); + } catch (Exception e) { + displayManagerException = e; + } + } + + if (virtualDisplay == null) { try { display = createDisplay(); Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - int orientation = crop != null ? displayInfo.getRotation() : 0; - setDisplaySurface(display, surface, orientation, deviceSize.toRect(), inputSize.toRect(), layerStack); + Rect layerRect = orientedCrop != null ? orientedCrop : deviceSize.toRect(); + setDisplaySurface(display, surface, layerRect, inputSize.toRect(), layerStack); Ln.d("Display: using SurfaceControl API"); } catch (Exception surfaceControlException) { Ln.e("Could not create display using DisplayManager", displayManagerException); @@ -150,11 +180,11 @@ public void start(Surface surface) throws IOException { if (virtualDisplay == null || displayId == 0) { // Surface control or main display: send all events to the original display, relative to the device size Size deviceSize = displayInfo.getSize(); - positionMapper = PositionMapper.create(videoSize, transform, deviceSize); + positionMapper = PositionMapper.create(videoSize, eventTransform, deviceSize); virtualDisplayId = displayId; } else { // The positions are relative to the virtual display, not the original display (so use inputSize, not deviceSize!) - positionMapper = PositionMapper.create(videoSize, transform, inputSize); + positionMapper = PositionMapper.create(videoSize, eventTransform, inputSize); virtualDisplayId = virtualDisplay.getDisplay().getDisplayId(); } vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper); @@ -202,11 +232,17 @@ private static IBinder createDisplay() throws Exception { return SurfaceControl.createDisplay("scrcpy", secure); } - private static void setDisplaySurface(IBinder display, Surface surface, int orientation, Rect deviceRect, Rect displayRect, int layerStack) { + /** + * Attach a surface to a display and optionally crop it. + * + * @param deviceRect the portion of the device to display (in the current orientation) + * @param displayRect the area of the surface where to render {@code deviceRect} + */ + private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack) { SurfaceControl.openTransaction(); try { SurfaceControl.setDisplaySurface(display, surface); - SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); + SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect); SurfaceControl.setDisplayLayerStack(display, layerStack); } finally { SurfaceControl.closeTransaction(); From a9b8b9b987082f6daa127722a4013e6fb2d7a719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Fri, 6 Jun 2025 00:25:24 +0200 Subject: [PATCH 6/8] Skip DisplayManager when cropping --- .../scrcpy/video/ScreenCapture.java | 61 +++++-------------- 1 file changed, 16 insertions(+), 45 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index ae4b061748..c463c97bd5 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -43,9 +43,7 @@ public class ScreenCapture extends SurfaceCapture { private IBinder display; private VirtualDisplay virtualDisplay; - private AffineMatrix transform; // OpenGL transform (orientation/angle) - private AffineMatrix eventTransform; // full transform used for input events - private Rect orientedCrop; + private AffineMatrix transform; private OpenGLRunner glRunner; public ScreenCapture(VirtualDisplayListener vdListener, Options options) { @@ -87,32 +85,19 @@ public void prepare() throws ConfigurationException { captureOrientation = Orientation.fromRotation(displayInfo.getRotation()); } - boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; + VideoFilter filter = new VideoFilter(displaySize); - // Compute the oriented crop to apply on the display - orientedCrop = null; if (crop != null) { boolean transposed = (displayInfo.getRotation() % 2) != 0; - orientedCrop = transposed ? new Rect(crop.top, crop.left, crop.bottom, crop.right) - : new Rect(crop); - } - - // Compute the transform for input events (includes crop) - VideoFilter eventFilter = new VideoFilter(displaySize); - if (orientedCrop != null) { - eventFilter.addCrop(orientedCrop, false); + filter.addCrop(crop, transposed); } - eventFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - eventFilter.addAngle(angle); - eventTransform = eventFilter.getInverseTransform(); - videoSize = eventFilter.getOutputSize().limit(maxSize).round8(); + boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; + filter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + filter.addAngle(angle); - // Compute the transform for the OpenGL pipeline (no crop) - VideoFilter glFilter = new VideoFilter(displaySize); - glFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - glFilter.addAngle(angle); - transform = glFilter.getInverseTransform(); + transform = filter.getInverseTransform(); + videoSize = filter.getOutputSize().limit(maxSize).round8(); } @Override @@ -128,25 +113,18 @@ public void start(Surface surface) throws IOException { Size inputSize; if (transform != null) { - // If there is a filter, it must receive the content in the display orientation - if (orientedCrop != null) { - inputSize = new Size(orientedCrop.width(), orientedCrop.height()); - } else { - inputSize = displayInfo.getSize(); - } + // If there is a filter, it must receive the full display content + inputSize = displayInfo.getSize(); assert glRunner == null; OpenGLFilter glFilter = new AffineOpenGLFilter(transform); glRunner = new OpenGLRunner(glFilter); surface = glRunner.start(inputSize, videoSize, surface); } else { - if (orientedCrop != null) { - inputSize = new Size(orientedCrop.width(), orientedCrop.height()); - } else { - inputSize = videoSize; - } + // If there is no filter, the display must be rendered at target video size directly + inputSize = videoSize; } - boolean useDisplayManager = orientedCrop == null; + boolean useDisplayManager = crop == null; Exception displayManagerException = null; if (useDisplayManager) { try { @@ -164,8 +142,7 @@ public void start(Surface surface) throws IOException { Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - Rect layerRect = orientedCrop != null ? orientedCrop : deviceSize.toRect(); - setDisplaySurface(display, surface, layerRect, inputSize.toRect(), layerStack); + setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); Ln.d("Display: using SurfaceControl API"); } catch (Exception surfaceControlException) { Ln.e("Could not create display using DisplayManager", displayManagerException); @@ -180,11 +157,11 @@ public void start(Surface surface) throws IOException { if (virtualDisplay == null || displayId == 0) { // Surface control or main display: send all events to the original display, relative to the device size Size deviceSize = displayInfo.getSize(); - positionMapper = PositionMapper.create(videoSize, eventTransform, deviceSize); + positionMapper = PositionMapper.create(videoSize, transform, deviceSize); virtualDisplayId = displayId; } else { // The positions are relative to the virtual display, not the original display (so use inputSize, not deviceSize!) - positionMapper = PositionMapper.create(videoSize, eventTransform, inputSize); + positionMapper = PositionMapper.create(videoSize, transform, inputSize); virtualDisplayId = virtualDisplay.getDisplay().getDisplayId(); } vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper); @@ -232,12 +209,6 @@ private static IBinder createDisplay() throws Exception { return SurfaceControl.createDisplay("scrcpy", secure); } - /** - * Attach a surface to a display and optionally crop it. - * - * @param deviceRect the portion of the device to display (in the current orientation) - * @param displayRect the area of the surface where to render {@code deviceRect} - */ private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack) { SurfaceControl.openTransaction(); try { From 19652c97641476e0e587471d4e2a9db67e0a0f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Fri, 6 Jun 2025 00:25:27 +0200 Subject: [PATCH 7/8] Use rotation for SurfaceControl crop --- .../scrcpy/video/ScreenCapture.java | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index c463c97bd5..1a6d4b7c2e 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -124,25 +124,18 @@ public void start(Surface surface) throws IOException { inputSize = videoSize; } - boolean useDisplayManager = crop == null; - Exception displayManagerException = null; - if (useDisplayManager) { - try { - virtualDisplay = ServiceManager.getDisplayManager() - .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); - Ln.d("Display: using DisplayManager API"); - } catch (Exception e) { - displayManagerException = e; - } - } - - if (virtualDisplay == null) { + try { + virtualDisplay = ServiceManager.getDisplayManager() + .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); + Ln.d("Display: using DisplayManager API"); + } catch (Exception displayManagerException) { try { display = createDisplay(); Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); + int orientation = crop != null ? displayInfo.getRotation() : 0; + setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack, orientation); Ln.d("Display: using SurfaceControl API"); } catch (Exception surfaceControlException) { Ln.e("Could not create display using DisplayManager", displayManagerException); @@ -209,11 +202,11 @@ private static IBinder createDisplay() throws Exception { return SurfaceControl.createDisplay("scrcpy", secure); } - private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack) { + private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack, int orientation) { SurfaceControl.openTransaction(); try { SurfaceControl.setDisplaySurface(display, surface); - SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect); + SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); SurfaceControl.setDisplayLayerStack(display, layerStack); } finally { SurfaceControl.closeTransaction(); From 86a480b8234538f5596029e9c0340c6a74c25464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Norbert=20Musia=C5=82?= <54953461+normusF7@users.noreply.github.com> Date: Fri, 6 Jun 2025 00:33:01 +0200 Subject: [PATCH 8/8] Fix cropping path without altering orientation --- .../scrcpy/video/ScreenCapture.java | 79 ++++++++++++------- 1 file changed, 50 insertions(+), 29 deletions(-) diff --git a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java index 1a6d4b7c2e..cb9ad70e32 100644 --- a/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java +++ b/server/src/main/java/com/genymobile/scrcpy/video/ScreenCapture.java @@ -31,6 +31,7 @@ public class ScreenCapture extends SurfaceCapture { private final int displayId; private int maxSize; private final Rect crop; + private Rect orientedCrop; private Orientation.Lock captureOrientationLock; private Orientation captureOrientation; private final float angle; @@ -43,7 +44,8 @@ public class ScreenCapture extends SurfaceCapture { private IBinder display; private VirtualDisplay virtualDisplay; - private AffineMatrix transform; + private AffineMatrix eventTransform; + private AffineMatrix glTransform; private OpenGLRunner glRunner; public ScreenCapture(VirtualDisplayListener vdListener, Options options) { @@ -85,19 +87,29 @@ public void prepare() throws ConfigurationException { captureOrientation = Orientation.fromRotation(displayInfo.getRotation()); } - VideoFilter filter = new VideoFilter(displaySize); + VideoFilter eventFilter = new VideoFilter(displaySize); if (crop != null) { boolean transposed = (displayInfo.getRotation() % 2) != 0; - filter.addCrop(crop, transposed); + eventFilter.addCrop(crop, transposed); + orientedCrop = transposed ? new Rect(crop.top, crop.left, crop.bottom, crop.right) : new Rect(crop); + } else { + orientedCrop = null; } boolean locked = captureOrientationLock != Orientation.Lock.Unlocked; - filter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); - filter.addAngle(angle); + eventFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + eventFilter.addAngle(angle); + + eventTransform = eventFilter.getInverseTransform(); - transform = filter.getInverseTransform(); - videoSize = filter.getOutputSize().limit(maxSize).round8(); + Size glInputSize = orientedCrop != null ? new Size(orientedCrop.width(), orientedCrop.height()) : displaySize; + VideoFilter glFilter = new VideoFilter(glInputSize); + glFilter.addOrientation(displayInfo.getRotation(), locked, captureOrientation); + glFilter.addAngle(angle); + + glTransform = glFilter.getInverseTransform(); + videoSize = glFilter.getOutputSize().limit(maxSize).round8(); } @Override @@ -112,34 +124,45 @@ public void start(Surface surface) throws IOException { } Size inputSize; - if (transform != null) { - // If there is a filter, it must receive the full display content - inputSize = displayInfo.getSize(); + if (glTransform != null) { + inputSize = orientedCrop != null ? new Size(orientedCrop.width(), orientedCrop.height()) : displayInfo.getSize(); assert glRunner == null; - OpenGLFilter glFilter = new AffineOpenGLFilter(transform); + OpenGLFilter glFilter = new AffineOpenGLFilter(glTransform); glRunner = new OpenGLRunner(glFilter); surface = glRunner.start(inputSize, videoSize, surface); } else { - // If there is no filter, the display must be rendered at target video size directly inputSize = videoSize; } - try { - virtualDisplay = ServiceManager.getDisplayManager() - .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); - Ln.d("Display: using DisplayManager API"); - } catch (Exception displayManagerException) { + if (orientedCrop == null) { + try { + virtualDisplay = ServiceManager.getDisplayManager() + .createVirtualDisplay("scrcpy", inputSize.getWidth(), inputSize.getHeight(), displayId, surface); + Ln.d("Display: using DisplayManager API"); + } catch (Exception displayManagerException) { + try { + display = createDisplay(); + + Size deviceSize = displayInfo.getSize(); + int layerStack = displayInfo.getLayerStack(); + setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack); + Ln.d("Display: using SurfaceControl API"); + } catch (Exception surfaceControlException) { + Ln.e("Could not create display using DisplayManager", displayManagerException); + Ln.e("Could not create display using SurfaceControl", surfaceControlException); + throw new AssertionError("Could not create display"); + } + } + } else { try { display = createDisplay(); - Size deviceSize = displayInfo.getSize(); int layerStack = displayInfo.getLayerStack(); - int orientation = crop != null ? displayInfo.getRotation() : 0; - setDisplaySurface(display, surface, deviceSize.toRect(), inputSize.toRect(), layerStack, orientation); + Rect displayRect = new Rect(0, 0, inputSize.getWidth(), inputSize.getHeight()); + setDisplaySurface(display, surface, orientedCrop, displayRect, layerStack); Ln.d("Display: using SurfaceControl API"); - } catch (Exception surfaceControlException) { - Ln.e("Could not create display using DisplayManager", displayManagerException); - Ln.e("Could not create display using SurfaceControl", surfaceControlException); + } catch (Exception e) { + Ln.e("Could not create display", e); throw new AssertionError("Could not create display"); } } @@ -148,13 +171,11 @@ public void start(Surface surface) throws IOException { int virtualDisplayId; PositionMapper positionMapper; if (virtualDisplay == null || displayId == 0) { - // Surface control or main display: send all events to the original display, relative to the device size Size deviceSize = displayInfo.getSize(); - positionMapper = PositionMapper.create(videoSize, transform, deviceSize); + positionMapper = PositionMapper.create(videoSize, eventTransform, deviceSize); virtualDisplayId = displayId; } else { - // The positions are relative to the virtual display, not the original display (so use inputSize, not deviceSize!) - positionMapper = PositionMapper.create(videoSize, transform, inputSize); + positionMapper = PositionMapper.create(videoSize, eventTransform, inputSize); virtualDisplayId = virtualDisplay.getDisplay().getDisplayId(); } vdListener.onNewVirtualDisplay(virtualDisplayId, positionMapper); @@ -202,11 +223,11 @@ private static IBinder createDisplay() throws Exception { return SurfaceControl.createDisplay("scrcpy", secure); } - private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack, int orientation) { + private static void setDisplaySurface(IBinder display, Surface surface, Rect deviceRect, Rect displayRect, int layerStack) { SurfaceControl.openTransaction(); try { SurfaceControl.setDisplaySurface(display, surface); - SurfaceControl.setDisplayProjection(display, orientation, deviceRect, displayRect); + SurfaceControl.setDisplayProjection(display, 0, deviceRect, displayRect); SurfaceControl.setDisplayLayerStack(display, layerStack); } finally { SurfaceControl.closeTransaction();