From 8f321a9bb848d0154aa731f07624eef92b592c6d Mon Sep 17 00:00:00 2001 From: Denis Nek Date: Mon, 5 Apr 2021 23:12:22 -0700 Subject: [PATCH] Add face detection and preview scale --- app/build.gradle.kts | 5 + .../java/com/nekdenis/camera/MainActivity.kt | 219 ++++++++++++++---- .../camera/detection/FaceDetectorProcessor.kt | 44 ++++ 3 files changed, 229 insertions(+), 39 deletions(-) create mode 100644 app/src/main/java/com/nekdenis/camera/detection/FaceDetectorProcessor.kt diff --git a/app/build.gradle.kts b/app/build.gradle.kts index e8957d8..e8e37c3 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -26,6 +26,7 @@ android { sourceCompatibility = JavaVersion.VERSION_1_8 targetCompatibility = JavaVersion.VERSION_1_8 } + kotlinOptions { jvmTarget = "1.8" useIR = true @@ -54,6 +55,10 @@ dependencies { implementation("androidx.camera:camera-view:1.0.0-alpha20") + implementation("com.google.mlkit:face-detection:16.0.6") + implementation("com.google.android.gms:play-services-mlkit-face-detection:16.1.5") + + implementation("androidx.core:core-ktx:1.3.2") implementation("androidx.appcompat:appcompat:1.2.0") implementation("com.google.android.material:material:1.3.0") diff --git a/app/src/main/java/com/nekdenis/camera/MainActivity.kt b/app/src/main/java/com/nekdenis/camera/MainActivity.kt index fd7bdfb..9b67879 100644 --- a/app/src/main/java/com/nekdenis/camera/MainActivity.kt +++ b/app/src/main/java/com/nekdenis/camera/MainActivity.kt @@ -4,34 +4,46 @@ import android.Manifest import android.content.Context import android.content.pm.PackageManager import android.os.Bundle +import android.util.Log import android.view.ViewGroup import android.widget.Toast +import androidx.compose.runtime.* import androidx.activity.ComponentActivity import androidx.activity.compose.setContent import androidx.camera.core.CameraSelector +import androidx.camera.core.ImageAnalysis +import androidx.camera.core.ImageProxy import androidx.camera.lifecycle.ProcessCameraProvider import androidx.camera.view.PreviewView -import androidx.compose.foundation.layout.Box -import androidx.compose.foundation.layout.fillMaxSize -import androidx.compose.foundation.layout.padding -import androidx.compose.foundation.layout.wrapContentSize +import androidx.compose.foundation.Canvas +import androidx.compose.foundation.layout.* import androidx.compose.material.Button import androidx.compose.material.Icon import androidx.compose.material.MaterialTheme import androidx.compose.material.Surface import androidx.compose.material.icons.Icons import androidx.compose.material.icons.filled.Cameraswitch -import androidx.compose.runtime.* import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.scale +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.drawscope.Stroke import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.platform.LocalDensity import androidx.compose.ui.platform.LocalLifecycleOwner +import androidx.compose.ui.unit.Constraints import androidx.compose.ui.unit.dp import androidx.compose.ui.viewinterop.AndroidView import androidx.core.app.ActivityCompat import androidx.core.content.ContextCompat import androidx.lifecycle.LifecycleOwner +import com.google.android.gms.tasks.TaskExecutors import com.google.common.util.concurrent.ListenableFuture +import com.google.mlkit.common.MlKitException +import com.google.mlkit.vision.face.Face +import com.nekdenis.camera.detection.FaceDetectorProcessor import com.nekdenis.camera.ui.theme.CameraComposeWorkshopTheme class MainActivity : ComponentActivity() { @@ -77,15 +89,13 @@ class MainActivity : ComponentActivity() { CameraComposeWorkshopTheme { // A surface container using the 'background' color from the theme Surface(color = MaterialTheme.colors.background) { - Box(modifier = Modifier.fillMaxSize()) { - var lens by remember { mutableStateOf(CameraSelector.LENS_FACING_FRONT) } - CameraPreview( - cameraLens = lens - ) - Controls( - onLensChange = { lens = switchLens(lens) } - ) - } + var lens by remember { mutableStateOf(CameraSelector.LENS_FACING_FRONT) } + CameraPreview( + cameraLens = lens + ) + Controls( + onLensChange = { lens = switchLens(lens) } + ) } } } @@ -94,34 +104,87 @@ class MainActivity : ComponentActivity() { @Composable fun CameraPreview( modifier: Modifier = Modifier, - cameraLens: Int, - scaleType: PreviewView.ScaleType = PreviewView.ScaleType.FILL_CENTER, + cameraLens: Int ) { val lifecycleOwner = LocalLifecycleOwner.current val context = LocalContext.current + var sourceInfo by remember { mutableStateOf(SourceInfo(10, 10, false)) } + var detectedFaces by remember { mutableStateOf>(emptyList()) } val previewView = remember { PreviewView(context) } - val cameraProvider = + val cameraProvider = remember(sourceInfo, cameraLens) { ProcessCameraProvider.getInstance(context) - .configureCamera(previewView, lifecycleOwner, cameraLens, context) - - AndroidView( - modifier = modifier, - factory = { - previewView.apply { - this.scaleType = scaleType - layoutParams = ViewGroup.LayoutParams( - ViewGroup.LayoutParams.MATCH_PARENT, - ViewGroup.LayoutParams.MATCH_PARENT - ) - // Preview is incorrectly scaled in Compose on some devices without this - implementationMode = PreviewView.ImplementationMode.COMPATIBLE - } + .configureCamera( + previewView, lifecycleOwner, cameraLens, context, + setSourceInfo = { sourceInfo = it }, + onFacesDetected = { detectedFaces = it }, + ) + } - previewView - }) + BoxWithConstraints( + modifier = Modifier.fillMaxSize(), + contentAlignment = Alignment.Center + ) { + with(LocalDensity.current) { + Box( + modifier = Modifier + .size( + height = sourceInfo.height.toDp(), + width = sourceInfo.width.toDp() + ) + .scale( + calculateScale( + constraints, + sourceInfo, + PreviewScaleType.CENTER_CROP + ) + ) + ) + { + CameraPreview(previewView) + DetectedFaces(faces = detectedFaces, sourceInfo = sourceInfo) + } + } + } } } +@Composable +private fun CameraPreview(previewView: PreviewView) { + AndroidView( + modifier = Modifier.fillMaxSize(), + factory = { + previewView.apply { + this.scaleType = PreviewView.ScaleType.FILL_CENTER + layoutParams = ViewGroup.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, + ViewGroup.LayoutParams.MATCH_PARENT + ) + // Preview is incorrectly scaled in Compose on some devices without this + implementationMode = PreviewView.ImplementationMode.COMPATIBLE + } + + previewView + }) +} + +@Composable +fun DetectedFaces( + faces: List, + sourceInfo: SourceInfo +) { + Canvas(modifier = Modifier.fillMaxSize()) { + val needToMirror = sourceInfo.isImageFlipped + for (face in faces) { + val left = + if (needToMirror) size.width - face.boundingBox.right.toFloat() else face.boundingBox.left.toFloat() + drawRect( + Color.Gray, style = Stroke(2.dp.toPx()), + topLeft = Offset(left, face.boundingBox.top.toFloat()), + size = Size(face.boundingBox.width().toFloat(), face.boundingBox.height().toFloat()) + ) + } + } +} @Composable fun Controls( @@ -144,23 +207,25 @@ private fun ListenableFuture.configureCamera( previewView: PreviewView, lifecycleOwner: LifecycleOwner, cameraLens: Int, - context: Context + context: Context, + setSourceInfo: (SourceInfo) -> Unit, + onFacesDetected: (List) -> Unit ): ListenableFuture { addListener({ + val cameraSelector = CameraSelector.Builder().requireLensFacing(cameraLens).build() + val preview = androidx.camera.core.Preview.Builder() .build() .apply { setSurfaceProvider(previewView.surfaceProvider) } + val analysis = bindAnalysisUseCase(cameraLens, setSourceInfo, onFacesDetected) try { get().apply { unbindAll() - bindToLifecycle( - lifecycleOwner, - CameraSelector.Builder().requireLensFacing(cameraLens).build(), - preview - ) + bindToLifecycle(lifecycleOwner, cameraSelector, preview) + bindToLifecycle(lifecycleOwner, cameraSelector, analysis) } } catch (exc: Exception) { TODO("process errors") @@ -174,3 +239,79 @@ private fun switchLens(lens: Int) = if (CameraSelector.LENS_FACING_FRONT == lens } else { CameraSelector.LENS_FACING_FRONT } + + +private fun bindAnalysisUseCase( + lens: Int, + setSourceInfo: (SourceInfo) -> Unit, + onFacesDetected: (List) -> Unit +): ImageAnalysis? { + + val imageProcessor = try { + FaceDetectorProcessor() + } catch (e: Exception) { + Log.e("CAMERA", "Can not create image processor", e) + return null + } + val builder = ImageAnalysis.Builder() + val analysisUseCase = builder.build() + + var sourceInfoUpdated = false + + analysisUseCase.setAnalyzer( + TaskExecutors.MAIN_THREAD, + { imageProxy: ImageProxy -> + if (!sourceInfoUpdated) { + setSourceInfo(obtainSourceInfo(lens, imageProxy)) + sourceInfoUpdated = true + } + try { + imageProcessor.processImageProxy(imageProxy, onFacesDetected) + } catch (e: MlKitException) { + Log.e( + "CAMERA", "Failed to process image. Error: " + e.localizedMessage + ) + } + } + ) + return analysisUseCase +} + +private fun obtainSourceInfo(lens: Int, imageProxy: ImageProxy): SourceInfo { + val isImageFlipped = lens == CameraSelector.LENS_FACING_FRONT + val rotationDegrees = imageProxy.imageInfo.rotationDegrees + return if (rotationDegrees == 0 || rotationDegrees == 180) { + SourceInfo( + height = imageProxy.height, width = imageProxy.width, isImageFlipped = isImageFlipped + ) + } else { + SourceInfo( + height = imageProxy.width, width = imageProxy.height, isImageFlipped = isImageFlipped + ) + } +} + +private fun calculateScale( + constraints: Constraints, + sourceInfo: SourceInfo, + scaleType: PreviewScaleType +): Float { + val heightRatio = constraints.maxHeight.toFloat() / sourceInfo.height + val widthRatio = constraints.maxWidth.toFloat() / sourceInfo.width + return when (scaleType) { + PreviewScaleType.FIT_CENTER -> kotlin.math.min(heightRatio, widthRatio) + PreviewScaleType.CENTER_CROP -> kotlin.math.max(heightRatio, widthRatio) + } +} + +data class SourceInfo( + val width: Int, + val height: Int, + val isImageFlipped: Boolean, +) + +private enum class PreviewScaleType { + FIT_CENTER, + CENTER_CROP +} + diff --git a/app/src/main/java/com/nekdenis/camera/detection/FaceDetectorProcessor.kt b/app/src/main/java/com/nekdenis/camera/detection/FaceDetectorProcessor.kt new file mode 100644 index 0000000..75bffd1 --- /dev/null +++ b/app/src/main/java/com/nekdenis/camera/detection/FaceDetectorProcessor.kt @@ -0,0 +1,44 @@ +package com.nekdenis.camera.detection + +import android.annotation.SuppressLint +import android.util.Log +import androidx.camera.core.ImageProxy +import com.google.android.gms.tasks.TaskExecutors +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.face.Face +import com.google.mlkit.vision.face.FaceDetection +import com.google.mlkit.vision.face.FaceDetector +import com.google.mlkit.vision.face.FaceDetectorOptions + +class FaceDetectorProcessor { + + private val detector: FaceDetector + + private val executor = TaskExecutors.MAIN_THREAD + + init { + val faceDetectorOptions = FaceDetectorOptions.Builder() + .setLandmarkMode(FaceDetectorOptions.LANDMARK_MODE_NONE) + .setContourMode(FaceDetectorOptions.CONTOUR_MODE_NONE) + .setClassificationMode(FaceDetectorOptions.CLASSIFICATION_MODE_NONE) + .setPerformanceMode(FaceDetectorOptions.PERFORMANCE_MODE_FAST) + .setMinFaceSize(0.4f) + .build() + + detector = FaceDetection.getClient(faceDetectorOptions) + } + + fun stop() { + detector.close() + } + + @SuppressLint("UnsafeExperimentalUsageError") + fun processImageProxy(image: ImageProxy, onDetectionFinished: (List) -> Unit) { + detector.process(InputImage.fromMediaImage(image.image!!, image.imageInfo.rotationDegrees)) + .addOnSuccessListener(executor) { results: List -> onDetectionFinished(results) } + .addOnFailureListener(executor) { e: Exception -> + Log.e("Camera", "Error detecting face", e) + } + .addOnCompleteListener { image.close() } + } +} \ No newline at end of file