Implement MediaCodec decoder pipeline and GPU encoder backend detection

android/app/src/main/java/com/supermarsx/uberdisplay/media/CodecCapabilities.kt

@@ -4,24 +4,26 @@ import android.media.MediaCodecList
 import com.supermarsx.uberdisplay.protocol.CodecConstants
 
 object CodecCapabilities {
+    private val mimeToMask = mapOf(
+        "video/avc" to CodecConstants.CODEC_MASK_H264,
+        "video/hevc" to CodecConstants.CODEC_MASK_H265,
+        "video/av01" to CodecConstants.CODEC_MASK_AV1,
+        "video/x-vnd.on2.vp9" to CodecConstants.CODEC_MASK_VP9,
+        "video/evc" to CodecConstants.CODEC_MASK_EVC,
+        "video/lcevc" to CodecConstants.CODEC_MASK_LCEVC
+    )
+
     fun getCodecMask(): Int {
         return try {
             val list = MediaCodecList(MediaCodecList.ALL_CODECS)
             var mask = 0
             for (info in list.codecInfos) {
                 if (info.isEncoder) continue
                 val types = info.supportedTypes.map { it.lowercase() }
-                if (types.any { it.contains("video/avc") }) {
-                    mask = mask or CodecConstants.CODEC_MASK_H264
-                }
-                if (types.any { it.contains("video/hevc") }) {
-                    mask = mask or CodecConstants.CODEC_MASK_H265
-                }
-                if (types.any { it.contains("video/av01") }) {
-                    mask = mask or CodecConstants.CODEC_MASK_AV1
-                }
-                if (types.any { it.contains("video/x-vnd.on2.vp9") }) {
-                    mask = mask or CodecConstants.CODEC_MASK_VP9
+                for ((mime, codecMask) in mimeToMask) {
+                    if (types.any { it.contains(mime) }) {
+                        mask = mask or codecMask
+                    }
                 }
             }
             if (mask == 0) {

android/app/src/main/java/com/supermarsx/uberdisplay/media/DecoderController.kt

@@ -1,5 +1,7 @@
 package com.supermarsx.uberdisplay.media
 
+import android.media.MediaCodec
+import android.media.MediaFormat
 import android.view.Surface
 import com.supermarsx.uberdisplay.Diagnostics
 import com.supermarsx.uberdisplay.protocol.CodecConstants
@@ -10,7 +12,8 @@ data class DecoderStatus(
     val mimeType: String,
     val width: Int,
     val height: Int,
-    val surfaceBound: Boolean
+    val surfaceBound: Boolean,
+    val decoderStarted: Boolean = false
 )
 
 class DecoderController {
@@ -25,39 +28,145 @@ class DecoderController {
         surfaceBound = false
     )
 
+    private var decoder: MediaCodec? = null
+    private val lock = Any()
+    private var presentationTimeUs = 0L
+
     fun setSurface(surface: Surface?) {
-        this.surface = surface
-        status = status.copy(surfaceBound = surface != null)
+        synchronized(lock) {
+            this.surface = surface
+            status = status.copy(surfaceBound = surface != null)
+            if (surface != null && status.width > 0 && status.height > 0) {
+                initDecoder()
+            } else if (surface == null) {
+                releaseDecoder()
+            }
+        }
     }
 
     fun onConfigure(packet: Packet.Configure) {
-        val codecId = packet.codecId ?: CodecConstants.CODEC_ID_H264
-        val mime = codecIdToMime(codecId)
-        status = status.copy(
-            codecId = codecId,
-            mimeType = mime,
-            width = packet.width,
-            height = packet.height
-        )
-        Diagnostics.logInfo("decoder_config codec=$codecId mime=$mime ${packet.width}x${packet.height}")
+        synchronized(lock) {
+            val codecId = packet.codecId ?: CodecConstants.CODEC_ID_H264
+            val mime = codecIdToMime(codecId)
+            val changed = codecId != status.codecId ||
+                packet.width != status.width ||
+                packet.height != status.height
+            status = status.copy(
+                codecId = codecId,
+                mimeType = mime,
+                width = packet.width,
+                height = packet.height
+            )
+            Diagnostics.logInfo("decoder_config codec=$codecId mime=$mime ${packet.width}x${packet.height}")
+            if (changed && surface != null && packet.width > 0 && packet.height > 0) {
+                initDecoder()
+            }
+        }
     }
 
     fun onFrame(data: ByteArray) {
         if (data.isEmpty()) return
-        // TODO: feed MediaCodec once decoder pipeline is wired.
+        synchronized(lock) {
+            feedDecoder(data)
+        }
+    }
+
+    fun release() {
+        synchronized(lock) {
+            releaseDecoder()
+        }
     }
 
     fun getStatus(): DecoderStatus = status
 
-    private fun codecIdToMime(codecId: Int): String {
-        return when (codecId) {
-            CodecConstants.CODEC_ID_H265 -> "video/hevc"
-            CodecConstants.CODEC_ID_AV1 -> "video/av01"
-            CodecConstants.CODEC_ID_VP9 -> "video/x-vnd.on2.vp9"
-            CodecConstants.CODEC_ID_EVC -> "video/evc"
-            CodecConstants.CODEC_ID_LCEVC -> "video/lcevc"
-            CodecConstants.CODEC_ID_H266 -> "video/avc"
-            else -> "video/avc"
+    private fun initDecoder() {
+        releaseDecoder()
+        val currentSurface = surface ?: return
+        val mime = status.mimeType
+        val format = MediaFormat.createVideoFormat(mime, status.width, status.height)
+        try {
+            val codec = MediaCodec.createDecoderByType(mime)
+            codec.configure(format, currentSurface, null, 0)
+            codec.start()
+            decoder = codec
+            presentationTimeUs = 0L
+            status = status.copy(decoderStarted = true)
+            Diagnostics.logInfo("decoder_init mime=$mime ${status.width}x${status.height}")
+        } catch (e: Exception) {
+            Diagnostics.logError("decoder_init_failed ${e.message}", e)
+            status = status.copy(decoderStarted = false)
+        }
+    }
+
+    private fun feedDecoder(data: ByteArray) {
+        val codec = decoder ?: return
+        try {
+            val inputIndex = codec.dequeueInputBuffer(INPUT_TIMEOUT_US)
+            if (inputIndex >= 0) {
+                val inputBuffer = codec.getInputBuffer(inputIndex) ?: return
+                inputBuffer.clear()
+                val copyLen = data.size.coerceAtMost(inputBuffer.capacity())
+                inputBuffer.put(data, 0, copyLen)
+                codec.queueInputBuffer(inputIndex, 0, copyLen, presentationTimeUs, 0)
+                presentationTimeUs += FRAME_DURATION_US
+            }
+            drainOutput(codec)
+        } catch (e: MediaCodec.CodecException) {
+            Diagnostics.logError("decoder_feed_error ${e.diagnosticInfo}", e)
+            if (!e.isRecoverable) {
+                initDecoder()
+            }
+        } catch (e: IllegalStateException) {
+            Diagnostics.logError("decoder_feed_illegal_state ${e.message}", e)
+        }
+    }
+
+    private fun drainOutput(codec: MediaCodec) {
+        val info = MediaCodec.BufferInfo()
+        while (true) {
+            val outputIndex = codec.dequeueOutputBuffer(info, OUTPUT_TIMEOUT_US)
+            when {
+                outputIndex >= 0 -> {
+                    codec.releaseOutputBuffer(outputIndex, info.size > 0)
+                    if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {
+                        return
+                    }
+                }
+                outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
+                    val newFormat = codec.outputFormat
+                    Diagnostics.logInfo("decoder_format_changed $newFormat")
+                }
+                else -> return
+            }
+        }
+    }
+
+    private fun releaseDecoder() {
+        try {
+            decoder?.stop()
+        } catch (_: Exception) {}
+        try {
+            decoder?.release()
+        } catch (_: Exception) {}
+        decoder = null
+        status = status.copy(decoderStarted = false)
+    }
+
+    companion object {
+        private const val INPUT_TIMEOUT_US = 5_000L
+        private const val OUTPUT_TIMEOUT_US = 0L
+        private const val FRAME_DURATION_US = 16_667L
+
+        fun codecIdToMime(codecId: Int): String {
+            return when (codecId) {
+                CodecConstants.CODEC_ID_H265 -> "video/hevc"
+                CodecConstants.CODEC_ID_AV1 -> "video/av01"
+                CodecConstants.CODEC_ID_VP9 -> "video/x-vnd.on2.vp9"
+                CodecConstants.CODEC_ID_EVC -> "video/evc"
+                CodecConstants.CODEC_ID_LCEVC -> "video/lcevc"
+                CodecConstants.CODEC_ID_H266 -> "video/vvc"
+                else -> "video/avc"
+            }
         }
     }
 }

android/app/src/test/java/com/supermarsx/uberdisplay/DecoderControllerTest.kt

@@ -0,0 +1,66 @@
+package com.supermarsx.uberdisplay
+
+import com.supermarsx.uberdisplay.media.DecoderController
+import com.supermarsx.uberdisplay.protocol.CodecConstants
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+class DecoderControllerTest {
+    @Test
+    fun codecIdToMimeReturnsAvcForH264() {
+        assertEquals("video/avc", DecoderController.codecIdToMime(CodecConstants.CODEC_ID_H264))
+    }
+
+    @Test
+    fun codecIdToMimeReturnsHevcForH265() {
+        assertEquals("video/hevc", DecoderController.codecIdToMime(CodecConstants.CODEC_ID_H265))
+    }
+
+    @Test
+    fun codecIdToMimeReturnsAv01ForAv1() {
+        assertEquals("video/av01", DecoderController.codecIdToMime(CodecConstants.CODEC_ID_AV1))
+    }
+
+    @Test
+    fun codecIdToMimeReturnsVp9ForVp9() {
+        assertEquals("video/x-vnd.on2.vp9", DecoderController.codecIdToMime(CodecConstants.CODEC_ID_VP9))
+    }
+
+    @Test
+    fun codecIdToMimeReturnsEvcForEvc() {
+        assertEquals("video/evc", DecoderController.codecIdToMime(CodecConstants.CODEC_ID_EVC))
+    }
+
+    @Test
+    fun codecIdToMimeReturnsLcevcForLcevc() {
+        assertEquals("video/lcevc", DecoderController.codecIdToMime(CodecConstants.CODEC_ID_LCEVC))
+    }
+
+    @Test
+    fun codecIdToMimeReturnsVvcForH266() {
+        assertEquals("video/vvc", DecoderController.codecIdToMime(CodecConstants.CODEC_ID_H266))
+    }
+
+    @Test
+    fun codecIdToMimeDefaultsToAvc() {
+        assertEquals("video/avc", DecoderController.codecIdToMime(999))
+    }
+
+    @Test
+    fun defaultStatusHasH264Codec() {
+        val controller = DecoderController()
+        val status = controller.getStatus()
+        assertEquals(CodecConstants.CODEC_ID_H264, status.codecId)
+        assertEquals("video/avc", status.mimeType)
+        assertEquals(false, status.surfaceBound)
+        assertEquals(false, status.decoderStarted)
+    }
+
+    @Test
+    fun onFrameIgnoresEmptyData() {
+        val controller = DecoderController()
+        controller.onFrame(byteArrayOf())
+        val status = controller.getStatus()
+        assertEquals(false, status.decoderStarted)
+    }
+}