Add face detection and preview scale

app/build.gradle.kts

@@ -26,6 +26,7 @@ android {
         sourceCompatibility = JavaVersion.VERSION_1_8
         targetCompatibility = JavaVersion.VERSION_1_8
     }
+
     kotlinOptions {
         jvmTarget = "1.8"
         useIR = true
@@ -54,6 +55,10 @@ dependencies {
     implementation("androidx.camera:camera-view:1.0.0-alpha20")
 
 
+    implementation("com.google.mlkit:face-detection:16.0.6")
+    implementation("com.google.android.gms:play-services-mlkit-face-detection:16.1.5")
+
+
     implementation("androidx.core:core-ktx:1.3.2")
     implementation("androidx.appcompat:appcompat:1.2.0")
     implementation("com.google.android.material:material:1.3.0")

app/src/main/java/com/nekdenis/camera/MainActivity.kt

@@ -4,34 +4,46 @@ import android.Manifest
 import android.content.Context
 import android.content.pm.PackageManager
 import android.os.Bundle
+import android.util.Log
 import android.view.ViewGroup
 import android.widget.Toast
+import androidx.compose.runtime.*
 import androidx.activity.ComponentActivity
 import androidx.activity.compose.setContent
 import androidx.camera.core.CameraSelector
+import androidx.camera.core.ImageAnalysis
+import androidx.camera.core.ImageProxy
 import androidx.camera.lifecycle.ProcessCameraProvider
 import androidx.camera.view.PreviewView
-import androidx.compose.foundation.layout.Box
-import androidx.compose.foundation.layout.fillMaxSize
-import androidx.compose.foundation.layout.padding
-import androidx.compose.foundation.layout.wrapContentSize
+import androidx.compose.foundation.Canvas
+import androidx.compose.foundation.layout.*
 import androidx.compose.material.Button
 import androidx.compose.material.Icon
 import androidx.compose.material.MaterialTheme
 import androidx.compose.material.Surface
 import androidx.compose.material.icons.Icons
 import androidx.compose.material.icons.filled.Cameraswitch
-import androidx.compose.runtime.*
 import androidx.compose.ui.Alignment
 import androidx.compose.ui.Modifier
+import androidx.compose.ui.draw.scale
+import androidx.compose.ui.geometry.Offset
+import androidx.compose.ui.geometry.Size
+import androidx.compose.ui.graphics.Color
+import androidx.compose.ui.graphics.drawscope.Stroke
 import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalDensity
 import androidx.compose.ui.platform.LocalLifecycleOwner
+import androidx.compose.ui.unit.Constraints
 import androidx.compose.ui.unit.dp
 import androidx.compose.ui.viewinterop.AndroidView
 import androidx.core.app.ActivityCompat
 import androidx.core.content.ContextCompat
 import androidx.lifecycle.LifecycleOwner
+import com.google.android.gms.tasks.TaskExecutors
 import com.google.common.util.concurrent.ListenableFuture
+import com.google.mlkit.common.MlKitException
+import com.google.mlkit.vision.face.Face
+import com.nekdenis.camera.detection.FaceDetectorProcessor
 import com.nekdenis.camera.ui.theme.CameraComposeWorkshopTheme
 
 class MainActivity : ComponentActivity() {
@@ -77,15 +89,13 @@ class MainActivity : ComponentActivity() {
             CameraComposeWorkshopTheme {
                 // A surface container using the 'background' color from the theme
                 Surface(color = MaterialTheme.colors.background) {
-                    Box(modifier = Modifier.fillMaxSize()) {
-                        var lens by remember { mutableStateOf(CameraSelector.LENS_FACING_FRONT) }
-                        CameraPreview(
-                            cameraLens = lens
-                        )
-                        Controls(
-                            onLensChange = { lens = switchLens(lens) }
-                        )
-                    }
+                    var lens by remember { mutableStateOf(CameraSelector.LENS_FACING_FRONT) }
+                    CameraPreview(
+                        cameraLens = lens
+                    )
+                    Controls(
+                        onLensChange = { lens = switchLens(lens) }
+                    )
                 }
             }
         }
@@ -94,34 +104,87 @@ class MainActivity : ComponentActivity() {
     @Composable
     fun CameraPreview(
         modifier: Modifier = Modifier,
-        cameraLens: Int,
-        scaleType: PreviewView.ScaleType = PreviewView.ScaleType.FILL_CENTER,
+        cameraLens: Int
     ) {
         val lifecycleOwner = LocalLifecycleOwner.current
         val context = LocalContext.current
+        var sourceInfo by remember { mutableStateOf(SourceInfo(10, 10, false)) }
+        var detectedFaces by remember { mutableStateOf<List<Face>>(emptyList()) }
         val previewView = remember { PreviewView(context) }
-        val cameraProvider =
+        val cameraProvider = remember(sourceInfo, cameraLens) {
             ProcessCameraProvider.getInstance(context)
-                .configureCamera(previewView, lifecycleOwner, cameraLens, context)
-
-        AndroidView(
-            modifier = modifier,
-            factory = {
-                previewView.apply {
-                    this.scaleType = scaleType
-                    layoutParams = ViewGroup.LayoutParams(
-                        ViewGroup.LayoutParams.MATCH_PARENT,
-                        ViewGroup.LayoutParams.MATCH_PARENT
-                    )
-                    // Preview is incorrectly scaled in Compose on some devices without this
-                    implementationMode = PreviewView.ImplementationMode.COMPATIBLE
-                }
+                .configureCamera(
+                    previewView, lifecycleOwner, cameraLens, context,
+                    setSourceInfo = { sourceInfo = it },
+                    onFacesDetected = { detectedFaces = it },
+                )
+        }
 
-                previewView
-            })
+        BoxWithConstraints(
+            modifier = Modifier.fillMaxSize(),
+            contentAlignment = Alignment.Center
+        ) {
+            with(LocalDensity.current) {
+                Box(
+                    modifier = Modifier
+                        .size(
+                            height = sourceInfo.height.toDp(),
+                            width = sourceInfo.width.toDp()
+                        )
+                        .scale(
+                            calculateScale(
+                                constraints,
+                                sourceInfo,
+                                PreviewScaleType.CENTER_CROP
+                            )
+                        )
+                )
+                {
+                    CameraPreview(previewView)
+                    DetectedFaces(faces = detectedFaces, sourceInfo = sourceInfo)
+                }
+            }
+        }
     }
 }
 
+@Composable
+private fun CameraPreview(previewView: PreviewView) {
+    AndroidView(
+        modifier = Modifier.fillMaxSize(),
+        factory = {
+            previewView.apply {
+                this.scaleType = PreviewView.ScaleType.FILL_CENTER
+                layoutParams = ViewGroup.LayoutParams(
+                    ViewGroup.LayoutParams.MATCH_PARENT,
+                    ViewGroup.LayoutParams.MATCH_PARENT
+                )
+                // Preview is incorrectly scaled in Compose on some devices without this
+                implementationMode = PreviewView.ImplementationMode.COMPATIBLE
+            }
+
+            previewView
+        })
+}
+
+@Composable
+fun DetectedFaces(
+    faces: List<Face>,
+    sourceInfo: SourceInfo
+) {
+    Canvas(modifier = Modifier.fillMaxSize()) {
+        val needToMirror = sourceInfo.isImageFlipped
+        for (face in faces) {
+            val left =
+                if (needToMirror) size.width - face.boundingBox.right.toFloat() else face.boundingBox.left.toFloat()
+            drawRect(
+                Color.Gray, style = Stroke(2.dp.toPx()),
+                topLeft = Offset(left, face.boundingBox.top.toFloat()),
+                size = Size(face.boundingBox.width().toFloat(), face.boundingBox.height().toFloat())
+            )
+        }
+    }
+}
 
 @Composable
 fun Controls(
@@ -144,23 +207,25 @@ private fun ListenableFuture<ProcessCameraProvider>.configureCamera(
     previewView: PreviewView,
     lifecycleOwner: LifecycleOwner,
     cameraLens: Int,
-    context: Context
+    context: Context,
+    setSourceInfo: (SourceInfo) -> Unit,
+    onFacesDetected: (List<Face>) -> Unit
 ): ListenableFuture<ProcessCameraProvider> {
     addListener({
+        val cameraSelector = CameraSelector.Builder().requireLensFacing(cameraLens).build()
+
         val preview = androidx.camera.core.Preview.Builder()
             .build()
             .apply {
                 setSurfaceProvider(previewView.surfaceProvider)
             }
 
+        val analysis = bindAnalysisUseCase(cameraLens, setSourceInfo, onFacesDetected)
         try {
             get().apply {
                 unbindAll()
-                bindToLifecycle(
-                    lifecycleOwner,
-                    CameraSelector.Builder().requireLensFacing(cameraLens).build(),
-                    preview
-                )
+                bindToLifecycle(lifecycleOwner, cameraSelector, preview)
+                bindToLifecycle(lifecycleOwner, cameraSelector, analysis)
             }
         } catch (exc: Exception) {
             TODO("process errors")
@@ -174,3 +239,79 @@ private fun switchLens(lens: Int) = if (CameraSelector.LENS_FACING_FRONT == lens
 } else {
     CameraSelector.LENS_FACING_FRONT
 }
+
+
+private fun bindAnalysisUseCase(
+    lens: Int,
+    setSourceInfo: (SourceInfo) -> Unit,
+    onFacesDetected: (List<Face>) -> Unit
+): ImageAnalysis? {
+
+    val imageProcessor = try {
+        FaceDetectorProcessor()
+    } catch (e: Exception) {
+        Log.e("CAMERA", "Can not create image processor", e)
+        return null
+    }
+    val builder = ImageAnalysis.Builder()
+    val analysisUseCase = builder.build()
+
+    var sourceInfoUpdated = false
+
+    analysisUseCase.setAnalyzer(
+        TaskExecutors.MAIN_THREAD,
+        { imageProxy: ImageProxy ->
+            if (!sourceInfoUpdated) {
+                setSourceInfo(obtainSourceInfo(lens, imageProxy))
+                sourceInfoUpdated = true
+            }
+            try {
+                imageProcessor.processImageProxy(imageProxy, onFacesDetected)
+            } catch (e: MlKitException) {
+                Log.e(
+                    "CAMERA", "Failed to process image. Error: " + e.localizedMessage
+                )
+            }
+        }
+    )
+    return analysisUseCase
+}
+
+private fun obtainSourceInfo(lens: Int, imageProxy: ImageProxy): SourceInfo {
+    val isImageFlipped = lens == CameraSelector.LENS_FACING_FRONT
+    val rotationDegrees = imageProxy.imageInfo.rotationDegrees
+    return if (rotationDegrees == 0 || rotationDegrees == 180) {
+        SourceInfo(
+            height = imageProxy.height, width = imageProxy.width, isImageFlipped = isImageFlipped
+        )
+    } else {
+        SourceInfo(
+            height = imageProxy.width, width = imageProxy.height, isImageFlipped = isImageFlipped
+        )
+    }
+}
+
+private fun calculateScale(
+    constraints: Constraints,
+    sourceInfo: SourceInfo,
+    scaleType: PreviewScaleType
+): Float {
+    val heightRatio = constraints.maxHeight.toFloat() / sourceInfo.height
+    val widthRatio = constraints.maxWidth.toFloat() / sourceInfo.width
+    return when (scaleType) {
+        PreviewScaleType.FIT_CENTER -> kotlin.math.min(heightRatio, widthRatio)
+        PreviewScaleType.CENTER_CROP -> kotlin.math.max(heightRatio, widthRatio)
+    }
+}
+
+data class SourceInfo(
+    val width: Int,
+    val height: Int,
+    val isImageFlipped: Boolean,
+)
+
+private enum class PreviewScaleType {
+    FIT_CENTER,
+    CENTER_CROP
+}
+

app/src/main/java/com/nekdenis/camera/detection/FaceDetectorProcessor.kt

@@ -0,0 +1,44 @@
+package com.nekdenis.camera.detection
+
+import android.annotation.SuppressLint
+import android.util.Log
+import androidx.camera.core.ImageProxy
+import com.google.android.gms.tasks.TaskExecutors
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.face.Face
+import com.google.mlkit.vision.face.FaceDetection
+import com.google.mlkit.vision.face.FaceDetector
+import com.google.mlkit.vision.face.FaceDetectorOptions
+
+class FaceDetectorProcessor {
+
+    private val detector: FaceDetector
+
+    private val executor = TaskExecutors.MAIN_THREAD
+
+    init {
+        val faceDetectorOptions = FaceDetectorOptions.Builder()
+            .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_NONE)
+            .setContourMode(FaceDetectorOptions.CONTOUR_MODE_NONE)
+            .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_NONE)
+            .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST)
+            .setMinFaceSize(0.4f)
+            .build()
+
+        detector = FaceDetection.getClient(faceDetectorOptions)
+    }
+
+    fun stop() {
+        detector.close()
+    }
+
+    @SuppressLint("UnsafeExperimentalUsageError")
+    fun processImageProxy(image: ImageProxy, onDetectionFinished: (List<Face>) -> Unit) {
+        detector.process(InputImage.fromMediaImage(image.image!!, image.imageInfo.rotationDegrees))
+            .addOnSuccessListener(executor) { results: List<Face> -> onDetectionFinished(results) }
+            .addOnFailureListener(executor) { e: Exception ->
+                Log.e("Camera", "Error detecting face", e)
+            }
+            .addOnCompleteListener { image.close() }
+    }
+}