From f359bead5f9271a0fa02d0236e5f9236c6f821aa Mon Sep 17 00:00:00 2001 From: David Govea Date: Mon, 20 Jul 2020 22:17:38 -0700 Subject: [PATCH 1/9] feat(android): upgrade to standalone ML Kit SDKs (#2908) --- android/build.gradle | 5 +- .../barcodedetector/BarcodeFormatUtils.java | 58 ++++----- .../barcodedetector/RNBarcodeDetector.java | 42 +++---- .../tasks/BarcodeDetectorAsyncTask.java | 117 +++++++----------- .../camera/tasks/FaceDetectorAsyncTask.java | 58 +++------ .../camera/tasks/TextRecognizerAsyncTask.java | 89 +++++-------- .../facedetector/FaceDetectorUtils.java | 57 ++++----- .../facedetector/RNFaceDetector.java | 34 ++--- .../tasks/FileFaceDetectionAsyncTask.java | 20 +-- docs/installation.md | 4 +- docs/migrationV2.md | 2 +- examples/mlkit/android/build.gradle | 2 +- 12 files changed, 206 insertions(+), 282 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index 3d217a883..4d97158a3 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -74,6 +74,7 @@ dependencies { implementation "androidx.exifinterface:exifinterface:1.0.0" implementation "androidx.annotation:annotation:1.0.0" implementation "androidx.legacy:legacy-support-v4:1.0.0" - mlkitImplementation "com.google.firebase:firebase-ml-vision:${safeExtGet('firebase-ml-vision', '19.0.3')}" - mlkitImplementation "com.google.firebase:firebase-ml-vision-face-model:${safeExtGet('firebase-ml-vision-face-model', '17.0.2')}" + mlkitImplementation "com.google.android.gms:play-services-mlkit-text-recognition:${safeExtGet('mlkit-text-recognition', '16.0.0')}" + mlkitImplementation "com.google.mlkit:barcode-scanning:${safeExtGet('mlkit-barcode-scanning', '16.0.0')}" + mlkitImplementation "com.google.mlkit:face-detection:${safeExtGet('mlkit-face-detection', '16.0.0')}" } diff --git a/android/src/mlkit/java/org/reactnative/barcodedetector/BarcodeFormatUtils.java b/android/src/mlkit/java/org/reactnative/barcodedetector/BarcodeFormatUtils.java index 84e5ab4b6..93b3ef112 100644 --- a/android/src/mlkit/java/org/reactnative/barcodedetector/BarcodeFormatUtils.java +++ b/android/src/mlkit/java/org/reactnative/barcodedetector/BarcodeFormatUtils.java @@ -1,7 +1,7 @@ package org.reactnative.barcodedetector; import android.util.SparseArray; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; +import com.google.mlkit.vision.barcode.Barcode; import java.util.Collections; import java.util.HashMap; @@ -14,7 +14,7 @@ public class BarcodeFormatUtils { public static final SparseArray TYPES; public static final Map REVERSE_TYPES; - private static final int UNKNOWN_FORMAT_INT = FirebaseVisionBarcode.FORMAT_UNKNOWN; + private static final int UNKNOWN_FORMAT_INT = Barcode.FORMAT_UNKNOWN; private static final String UNKNOWN_TYPE_STRING = "UNKNOWN_TYPE"; private static final String UNKNOWN_FORMAT_STRING = "UNKNOWN_FORMAT"; @@ -22,21 +22,21 @@ public class BarcodeFormatUtils { static { // Initialize integer to string map SparseArray map = new SparseArray<>(); - map.put(FirebaseVisionBarcode.FORMAT_CODE_128, "CODE_128"); - map.put(FirebaseVisionBarcode.FORMAT_CODE_39, "CODE_39"); - map.put(FirebaseVisionBarcode.FORMAT_CODE_93, "CODE_93"); - map.put(FirebaseVisionBarcode.FORMAT_CODABAR, "CODABAR"); - map.put(FirebaseVisionBarcode.FORMAT_DATA_MATRIX, "DATA_MATRIX"); - map.put(FirebaseVisionBarcode.FORMAT_EAN_13, "EAN_13"); - map.put(FirebaseVisionBarcode.FORMAT_EAN_8, "EAN_8"); - map.put(FirebaseVisionBarcode.FORMAT_ITF, "ITF"); - map.put(FirebaseVisionBarcode.FORMAT_QR_CODE, "QR_CODE"); - map.put(FirebaseVisionBarcode.FORMAT_UPC_A, "UPC_A"); - map.put(FirebaseVisionBarcode.FORMAT_UPC_E, "UPC_E"); - map.put(FirebaseVisionBarcode.FORMAT_PDF417, "PDF417"); - map.put(FirebaseVisionBarcode.FORMAT_AZTEC, "AZTEC"); - map.put(FirebaseVisionBarcode.FORMAT_ALL_FORMATS, "ALL"); - map.put(FirebaseVisionBarcode.FORMAT_UPC_A, "UPC_A"); + map.put(Barcode.FORMAT_CODE_128, "CODE_128"); + map.put(Barcode.FORMAT_CODE_39, "CODE_39"); + map.put(Barcode.FORMAT_CODE_93, "CODE_93"); + map.put(Barcode.FORMAT_CODABAR, "CODABAR"); + map.put(Barcode.FORMAT_DATA_MATRIX, "DATA_MATRIX"); + map.put(Barcode.FORMAT_EAN_13, "EAN_13"); + map.put(Barcode.FORMAT_EAN_8, "EAN_8"); + map.put(Barcode.FORMAT_ITF, "ITF"); + map.put(Barcode.FORMAT_QR_CODE, "QR_CODE"); + map.put(Barcode.FORMAT_UPC_A, "UPC_A"); + map.put(Barcode.FORMAT_UPC_E, "UPC_E"); + map.put(Barcode.FORMAT_PDF417, "PDF417"); + map.put(Barcode.FORMAT_AZTEC, "AZTEC"); + map.put(Barcode.FORMAT_ALL_FORMATS, "ALL"); + map.put(Barcode.FORMAT_UPC_A, "UPC_A"); map.put(-1, "None"); FORMATS = map; @@ -53,18 +53,18 @@ public class BarcodeFormatUtils { static { // Initialize integer to string map SparseArray map = new SparseArray<>(); - map.put(FirebaseVisionBarcode.TYPE_CALENDAR_EVENT, "CALENDAR_EVENT"); - map.put(FirebaseVisionBarcode.TYPE_CONTACT_INFO, "CONTACT_INFO"); - map.put(FirebaseVisionBarcode.TYPE_DRIVER_LICENSE, "DRIVER_LICENSE"); - map.put(FirebaseVisionBarcode.TYPE_EMAIL, "EMAIL"); - map.put(FirebaseVisionBarcode.TYPE_GEO, "GEO"); - map.put(FirebaseVisionBarcode.TYPE_ISBN, "ISBN"); - map.put(FirebaseVisionBarcode.TYPE_PHONE, "PHONE"); - map.put(FirebaseVisionBarcode.TYPE_PRODUCT, "PRODUCT"); - map.put(FirebaseVisionBarcode.TYPE_SMS, "SMS"); - map.put(FirebaseVisionBarcode.TYPE_TEXT, "TEXT"); - map.put(FirebaseVisionBarcode.TYPE_URL, "URL"); - map.put(FirebaseVisionBarcode.TYPE_WIFI, "WIFI"); + map.put(Barcode.TYPE_CALENDAR_EVENT, "CALENDAR_EVENT"); + map.put(Barcode.TYPE_CONTACT_INFO, "CONTACT_INFO"); + map.put(Barcode.TYPE_DRIVER_LICENSE, "DRIVER_LICENSE"); + map.put(Barcode.TYPE_EMAIL, "EMAIL"); + map.put(Barcode.TYPE_GEO, "GEO"); + map.put(Barcode.TYPE_ISBN, "ISBN"); + map.put(Barcode.TYPE_PHONE, "PHONE"); + map.put(Barcode.TYPE_PRODUCT, "PRODUCT"); + map.put(Barcode.TYPE_SMS, "SMS"); + map.put(Barcode.TYPE_TEXT, "TEXT"); + map.put(Barcode.TYPE_URL, "URL"); + map.put(Barcode.TYPE_WIFI, "WIFI"); map.put(-1, "None"); TYPES = map; diff --git a/android/src/mlkit/java/org/reactnative/barcodedetector/RNBarcodeDetector.java b/android/src/mlkit/java/org/reactnative/barcodedetector/RNBarcodeDetector.java index 31b676065..137deee51 100644 --- a/android/src/mlkit/java/org/reactnative/barcodedetector/RNBarcodeDetector.java +++ b/android/src/mlkit/java/org/reactnative/barcodedetector/RNBarcodeDetector.java @@ -3,10 +3,10 @@ import android.content.Context; import android.util.Log; -import com.google.firebase.ml.vision.FirebaseVision; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetectorOptions; +import com.google.mlkit.vision.barcode.Barcode; +import com.google.mlkit.vision.barcode.BarcodeScanner; +import com.google.mlkit.vision.barcode.BarcodeScannerOptions; +import com.google.mlkit.vision.barcode.BarcodeScanning; public class RNBarcodeDetector { @@ -14,15 +14,15 @@ public class RNBarcodeDetector { public static int NORMAL_MODE = 0; public static int ALTERNATE_MODE = 1; public static int INVERTED_MODE = 2; - public static int ALL_FORMATS = FirebaseVisionBarcode.FORMAT_ALL_FORMATS; + public static int ALL_FORMATS = Barcode.FORMAT_ALL_FORMATS; - private FirebaseVisionBarcodeDetector mBarcodeDetector = null; - private FirebaseVisionBarcodeDetectorOptions.Builder mBuilder; + private BarcodeScanner mBarcodeScanner = null; + private BarcodeScannerOptions.Builder mBuilder; - private int mBarcodeType = FirebaseVisionBarcode.FORMAT_ALL_FORMATS; + private int mBarcodeType = Barcode.FORMAT_ALL_FORMATS; public RNBarcodeDetector(Context context) { - mBuilder = new FirebaseVisionBarcodeDetectorOptions.Builder().setBarcodeFormats(mBarcodeType); + mBuilder = new BarcodeScannerOptions.Builder().setBarcodeFormats(mBarcodeType); } public boolean isOperational() { @@ -30,12 +30,12 @@ public boolean isOperational() { return true; } - public FirebaseVisionBarcodeDetector getDetector() { + public BarcodeScanner getDetector() { - if (mBarcodeDetector == null) { - createBarcodeDetector(); + if (mBarcodeScanner == null) { + createBarcodeScanner(); } - return mBarcodeDetector; + return mBarcodeScanner; } public void setBarcodeType(int barcodeType) { @@ -48,20 +48,18 @@ public void setBarcodeType(int barcodeType) { public void release() { - if (mBarcodeDetector != null) { + if (mBarcodeScanner != null) { try { - mBarcodeDetector.close(); + mBarcodeScanner.close(); } catch (Exception e) { - Log.e("RNCamera", "Attempt to close BarcodeDetector failed"); + Log.e("RNCamera", "Attempt to close BarcodeScanner failed"); } - mBarcodeDetector = null; + mBarcodeScanner = null; } } - private void createBarcodeDetector() { - FirebaseVisionBarcodeDetectorOptions options = mBuilder.build(); - mBarcodeDetector = FirebaseVision.getInstance() - .getVisionBarcodeDetector(options); - + private void createBarcodeScanner() { + BarcodeScannerOptions options = mBuilder.build(); + mBarcodeScanner = BarcodeScanning.getClient(options); } } diff --git a/android/src/mlkit/java/org/reactnative/camera/tasks/BarcodeDetectorAsyncTask.java b/android/src/mlkit/java/org/reactnative/camera/tasks/BarcodeDetectorAsyncTask.java index bf0f33aaf..714eb6094 100644 --- a/android/src/mlkit/java/org/reactnative/camera/tasks/BarcodeDetectorAsyncTask.java +++ b/android/src/mlkit/java/org/reactnative/camera/tasks/BarcodeDetectorAsyncTask.java @@ -9,10 +9,9 @@ import com.facebook.react.bridge.WritableMap; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcode; -import com.google.firebase.ml.vision.barcode.FirebaseVisionBarcodeDetector; -import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; +import com.google.mlkit.vision.barcode.Barcode; +import com.google.mlkit.vision.barcode.BarcodeScanner; +import com.google.mlkit.vision.common.InputImage; import org.reactnative.barcodedetector.BarcodeFormatUtils; import org.reactnative.barcodedetector.RNBarcodeDetector; @@ -68,19 +67,18 @@ protected Void doInBackground(Void... ignored) { return null; } - final FirebaseVisionImageMetadata metadata = new FirebaseVisionImageMetadata.Builder() - .setWidth(mWidth) - .setHeight(mHeight) - .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_YV12) - .setRotation(getFirebaseRotation()) - .build(); - FirebaseVisionImage image = FirebaseVisionImage.fromByteArray(mImageData, metadata); + InputImage image = InputImage.fromByteArray(mImageData, + mWidth, + mHeight, + mRotation, + InputImage.IMAGE_FORMAT_YV12 + ); - FirebaseVisionBarcodeDetector barcode = mBarcodeDetector.getDetector(); - barcode.detectInImage(image) - .addOnSuccessListener(new OnSuccessListener>() { + BarcodeScanner barcode = mBarcodeDetector.getDetector(); + barcode.process(image) + .addOnSuccessListener(new OnSuccessListener>() { @Override - public void onSuccess(List barcodes) { + public void onSuccess(List barcodes) { WritableArray serializedBarcodes = serializeEventData(barcodes); mDelegate.onBarcodesDetected(serializedBarcodes, mWidth, mHeight, mImageData); mDelegate.onBarcodeDetectingTaskCompleted(); @@ -96,33 +94,10 @@ public void onFailure(Exception e) { return null; } - private int getFirebaseRotation(){ - int result; - switch (mRotation) { - case 0: - result = FirebaseVisionImageMetadata.ROTATION_0; - break; - case 90: - result = FirebaseVisionImageMetadata.ROTATION_90; - break; - case 180: - result = FirebaseVisionImageMetadata.ROTATION_180; - break; - case -90: - result = FirebaseVisionImageMetadata.ROTATION_270; - break; - default: - result = FirebaseVisionImageMetadata.ROTATION_0; - Log.e(TAG, "Bad rotation value: " + mRotation); - } - return result; - } - - - private WritableArray serializeEventData(List barcodes) { + private WritableArray serializeEventData(List barcodes) { WritableArray barcodesList = Arguments.createArray(); - for (FirebaseVisionBarcode barcode: barcodes) { + for (Barcode barcode: barcodes) { // TODO implement position and data from all barcode types Rect bounds = barcode.getBoundingBox(); // Point[] corners = barcode.getCornerPoints(); @@ -135,19 +110,19 @@ private WritableArray serializeEventData(List barcodes) { WritableMap serializedBarcode = Arguments.createMap(); switch (valueType) { - case FirebaseVisionBarcode.TYPE_WIFI: + case Barcode.TYPE_WIFI: String ssid = barcode.getWifi().getSsid(); String password = barcode.getWifi().getPassword(); int type = barcode.getWifi().getEncryptionType(); String typeString = "UNKNOWN"; switch (type) { - case FirebaseVisionBarcode.WiFi.TYPE_OPEN: + case Barcode.WiFi.TYPE_OPEN: typeString = "Open"; break; - case FirebaseVisionBarcode.WiFi.TYPE_WEP: + case Barcode.WiFi.TYPE_WEP: typeString = "WEP"; break; - case FirebaseVisionBarcode.WiFi.TYPE_WPA: + case Barcode.WiFi.TYPE_WPA: typeString = "WPA"; break; } @@ -155,33 +130,33 @@ private WritableArray serializeEventData(List barcodes) { serializedBarcode.putString("password", password); serializedBarcode.putString("ssid", ssid); break; - case FirebaseVisionBarcode.TYPE_URL: + case Barcode.TYPE_URL: String title = barcode.getUrl().getTitle(); String url = barcode.getUrl().getUrl(); serializedBarcode.putString("url", url); serializedBarcode.putString("title", title); break; - case FirebaseVisionBarcode.TYPE_SMS: + case Barcode.TYPE_SMS: String message = barcode.getSms().getMessage(); String phoneNumber = barcode.getSms().getPhoneNumber(); serializedBarcode.putString("message", message); serializedBarcode.putString("title", phoneNumber); break; - case FirebaseVisionBarcode.TYPE_PHONE: + case Barcode.TYPE_PHONE: String number = barcode.getPhone().getNumber(); int typePhone = barcode.getPhone().getType(); serializedBarcode.putString("number", number); String typeStringPhone = getPhoneType(typePhone); serializedBarcode.putString("phoneType", typeStringPhone); break; - case FirebaseVisionBarcode.TYPE_CALENDAR_EVENT: + case Barcode.TYPE_CALENDAR_EVENT: serializedBarcode.putString("description", barcode.getCalendarEvent().getDescription()); serializedBarcode.putString("location", barcode.getCalendarEvent().getLocation()); serializedBarcode.putString("organizer", barcode.getCalendarEvent().getOrganizer()); serializedBarcode.putString("status", barcode.getCalendarEvent().getStatus()); serializedBarcode.putString("summary", barcode.getCalendarEvent().getSummary()); - FirebaseVisionBarcode.CalendarDateTime start = barcode.getCalendarEvent().getStart(); - FirebaseVisionBarcode.CalendarDateTime end = barcode.getCalendarEvent().getEnd(); + Barcode.CalendarDateTime start = barcode.getCalendarEvent().getStart(); + Barcode.CalendarDateTime end = barcode.getCalendarEvent().getEnd(); if (start != null) { serializedBarcode.putString("start", start.getRawValue()); } @@ -189,7 +164,7 @@ private WritableArray serializeEventData(List barcodes) { serializedBarcode.putString("end", start.getRawValue()); } break; - case FirebaseVisionBarcode.TYPE_DRIVER_LICENSE: + case Barcode.TYPE_DRIVER_LICENSE: serializedBarcode.putString("addressCity", barcode.getDriverLicense().getAddressCity()); serializedBarcode.putString("addressState", barcode.getDriverLicense().getAddressState()); serializedBarcode.putString("addressStreet", barcode.getDriverLicense().getAddressStreet()); @@ -205,14 +180,14 @@ private WritableArray serializeEventData(List barcodes) { serializedBarcode.putString("issuingCountry", barcode.getDriverLicense().getIssuingCountry()); serializedBarcode.putString("licenseNumber", barcode.getDriverLicense().getLicenseNumber()); break; - case FirebaseVisionBarcode.TYPE_GEO: + case Barcode.TYPE_GEO: serializedBarcode.putDouble("latitude", barcode.getGeoPoint().getLat()); serializedBarcode.putDouble("longitude", barcode.getGeoPoint().getLng()); break; - case FirebaseVisionBarcode.TYPE_CONTACT_INFO: + case Barcode.TYPE_CONTACT_INFO: serializedBarcode.putString("organization", barcode.getContactInfo().getOrganization()); serializedBarcode.putString("title", barcode.getContactInfo().getTitle()); - FirebaseVisionBarcode.PersonName name = barcode.getContactInfo().getName(); + Barcode.PersonName name = barcode.getContactInfo().getName(); if (name != null) { serializedBarcode.putString("firstName", name.getFirst()); serializedBarcode.putString("lastName", name.getLast()); @@ -222,18 +197,18 @@ private WritableArray serializeEventData(List barcodes) { serializedBarcode.putString("pronunciation", name.getPronunciation()); serializedBarcode.putString("suffix", name.getSuffix()); } - List phones = barcode.getContactInfo().getPhones(); + List phones = barcode.getContactInfo().getPhones(); WritableArray phonesList = Arguments.createArray(); - for (FirebaseVisionBarcode.Phone phone : phones) { + for (Barcode.Phone phone : phones) { WritableMap phoneObject = Arguments.createMap(); phoneObject.putString("number", phone.getNumber()); phoneObject.putString("phoneType", getPhoneType(phone.getType())); phonesList.pushMap(phoneObject); } serializedBarcode.putArray("phones", phonesList); - List addresses = barcode.getContactInfo().getAddresses(); + List addresses = barcode.getContactInfo().getAddresses(); WritableArray addressesList = Arguments.createArray(); - for (FirebaseVisionBarcode.Address address : addresses) { + for (Barcode.Address address : addresses) { WritableMap addressesData = Arguments.createMap(); WritableArray addressesLinesList = Arguments.createArray(); String[] addressesLines = address.getAddressLines(); @@ -245,10 +220,10 @@ private WritableArray serializeEventData(List barcodes) { int addressType = address.getType(); String addressTypeString = "UNKNOWN"; switch(addressType) { - case FirebaseVisionBarcode.Address.TYPE_WORK: + case Barcode.Address.TYPE_WORK: addressTypeString = "Work"; break; - case FirebaseVisionBarcode.Address.TYPE_HOME: + case Barcode.Address.TYPE_HOME: addressTypeString = "Home"; break; } @@ -256,21 +231,21 @@ private WritableArray serializeEventData(List barcodes) { addressesList.pushMap(addressesData); } serializedBarcode.putArray("addresses", addressesList); - List emails = barcode.getContactInfo().getEmails(); + List emails = barcode.getContactInfo().getEmails(); WritableArray emailsList = Arguments.createArray(); - for (FirebaseVisionBarcode.Email email : emails) { + for (Barcode.Email email : emails) { WritableMap emailData = processEmail(email); emailsList.pushMap(emailData); } serializedBarcode.putArray("emails", emailsList); - String[] urls = barcode.getContactInfo().getUrls(); + List urls = barcode.getContactInfo().getUrls(); WritableArray urlsList = Arguments.createArray(); for (String urlContact : urls) { urlsList.pushString(urlContact); } serializedBarcode.putArray("urls", urlsList); break; - case FirebaseVisionBarcode.TYPE_EMAIL: + case Barcode.TYPE_EMAIL: WritableMap emailData = processEmail(barcode.getEmail()); serializedBarcode.putMap("email", emailData); break; @@ -287,7 +262,7 @@ private WritableArray serializeEventData(List barcodes) { return barcodesList; } - private WritableMap processEmail(FirebaseVisionBarcode.Email email) { + private WritableMap processEmail(Barcode.Email email) { WritableMap emailData = Arguments.createMap(); emailData.putString("address", email.getAddress()); emailData.putString("body", email.getBody()); @@ -295,10 +270,10 @@ private WritableMap processEmail(FirebaseVisionBarcode.Email email) { int emailType = email.getType(); String emailTypeString = "UNKNOWN"; switch (emailType) { - case FirebaseVisionBarcode.Email.TYPE_WORK: + case Barcode.Email.TYPE_WORK: emailTypeString = "Work"; break; - case FirebaseVisionBarcode.Email.TYPE_HOME: + case Barcode.Email.TYPE_HOME: emailTypeString = "Home"; break; } @@ -309,16 +284,16 @@ private WritableMap processEmail(FirebaseVisionBarcode.Email email) { private String getPhoneType(int typePhone) { String typeStringPhone = "UNKNOWN"; switch(typePhone) { - case FirebaseVisionBarcode.Phone.TYPE_WORK: + case Barcode.Phone.TYPE_WORK: typeStringPhone = "Work"; break; - case FirebaseVisionBarcode.Phone.TYPE_HOME: + case Barcode.Phone.TYPE_HOME: typeStringPhone = "Home"; break; - case FirebaseVisionBarcode.Phone.TYPE_FAX: + case Barcode.Phone.TYPE_FAX: typeStringPhone = "Fax"; break; - case FirebaseVisionBarcode.Phone.TYPE_MOBILE: + case Barcode.Phone.TYPE_MOBILE: typeStringPhone = "Mobile"; break; } diff --git a/android/src/mlkit/java/org/reactnative/camera/tasks/FaceDetectorAsyncTask.java b/android/src/mlkit/java/org/reactnative/camera/tasks/FaceDetectorAsyncTask.java index a372be384..4c7d57be5 100644 --- a/android/src/mlkit/java/org/reactnative/camera/tasks/FaceDetectorAsyncTask.java +++ b/android/src/mlkit/java/org/reactnative/camera/tasks/FaceDetectorAsyncTask.java @@ -8,10 +8,9 @@ import com.google.android.cameraview.CameraView; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; -import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; -import com.google.firebase.ml.vision.face.FirebaseVisionFace; -import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetector; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.face.Face; +import com.google.mlkit.vision.face.FaceDetector; import org.reactnative.camera.utils.ImageDimensions; import org.reactnative.facedetector.FaceDetectorUtils; @@ -65,20 +64,18 @@ protected Void doInBackground(Void... ignored) { if (isCancelled() || mDelegate == null || mFaceDetector == null) { return null; } - FirebaseVisionImageMetadata metadata = new FirebaseVisionImageMetadata.Builder() - .setWidth(mWidth) - .setHeight(mHeight) - .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_YV12) - .setRotation(getFirebaseRotation()) - .build(); - FirebaseVisionImage image = FirebaseVisionImage.fromByteArray(mImageData, metadata); - - FirebaseVisionFaceDetector detector = mFaceDetector.getDetector(); - detector.detectInImage(image) + InputImage image = InputImage.fromByteArray(mImageData, + mWidth, + mHeight, + mRotation, + InputImage.IMAGE_FORMAT_YV12 + ); + FaceDetector detector = mFaceDetector.getDetector(); + detector.process(image) .addOnSuccessListener( - new OnSuccessListener>() { + new OnSuccessListener>() { @Override - public void onSuccess(List faces) { + public void onSuccess(List faces) { WritableArray facesList = serializeEventData(faces); mDelegate.onFacesDetected(facesList); mDelegate.onFaceDetectingTaskCompleted(); @@ -95,35 +92,10 @@ public void onFailure(Exception e) { return null; } - private int getFirebaseRotation(){ - int result; - switch (mRotation) { - case 0: - result = FirebaseVisionImageMetadata.ROTATION_0; - break; - case 90: - result = FirebaseVisionImageMetadata.ROTATION_90; - break; - case 180: - result = FirebaseVisionImageMetadata.ROTATION_180; - break; - case 270: - result = FirebaseVisionImageMetadata.ROTATION_270; - break; - case -90: - result = FirebaseVisionImageMetadata.ROTATION_270; - break; - default: - result = FirebaseVisionImageMetadata.ROTATION_0; - Log.e(TAG, "Bad rotation value: " + mRotation); - } - return result; - } - - private WritableArray serializeEventData(List faces) { + private WritableArray serializeEventData(List faces) { WritableArray facesList = Arguments.createArray(); - for (FirebaseVisionFace face : faces) { + for (Face face : faces) { WritableMap serializedFace = FaceDetectorUtils.serializeFace(face, mScaleX, mScaleY, mWidth, mHeight, mPaddingLeft, mPaddingTop); if (mImageDimensions.getFacing() == CameraView.FACING_FRONT) { serializedFace = FaceDetectorUtils.rotateFaceX(serializedFace, mImageDimensions.getWidth(), mScaleX); diff --git a/android/src/mlkit/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java b/android/src/mlkit/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java index 9a1e5db86..22f6be55b 100644 --- a/android/src/mlkit/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java +++ b/android/src/mlkit/java/org/reactnative/camera/tasks/TextRecognizerAsyncTask.java @@ -13,11 +13,10 @@ import com.google.android.cameraview.CameraView; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; -import com.google.firebase.ml.vision.FirebaseVision; -import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import com.google.firebase.ml.vision.common.FirebaseVisionImageMetadata; -import com.google.firebase.ml.vision.text.FirebaseVisionText; -import com.google.firebase.ml.vision.text.FirebaseVisionTextRecognizer; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.text.Text; +import com.google.mlkit.vision.text.TextRecognition; +import com.google.mlkit.vision.text.TextRecognizer; import org.reactnative.camera.utils.ImageDimensions; @@ -71,20 +70,19 @@ protected Void doInBackground(Void... ignored) { return null; } - FirebaseVisionImageMetadata metadata = new FirebaseVisionImageMetadata.Builder() - .setWidth(mWidth) - .setHeight(mHeight) - .setFormat(FirebaseVisionImageMetadata.IMAGE_FORMAT_YV12) - .setRotation(getFirebaseRotation()) - .build(); - FirebaseVisionTextRecognizer detector = FirebaseVision.getInstance().getOnDeviceTextRecognizer(); - - FirebaseVisionImage image = FirebaseVisionImage.fromByteArray(mImageData, metadata); - detector.processImage(image) - .addOnSuccessListener(new OnSuccessListener() { + TextRecognizer detector = TextRecognition.getClient(); + + InputImage image = InputImage.fromByteArray(mImageData, + mWidth, + mHeight, + mRotation, + InputImage.IMAGE_FORMAT_YV12 + ); + detector.process(image) + .addOnSuccessListener(new OnSuccessListener() { @Override - public void onSuccess(FirebaseVisionText firebaseVisionText) { - List textBlocks = firebaseVisionText.getTextBlocks(); + public void onSuccess(Text Text) { + List textBlocks = Text.getTextBlocks(); WritableArray serializedData = serializeEventData(textBlocks); mDelegate.onTextRecognized(serializedData); mDelegate.onTextRecognizerTaskCompleted(); @@ -102,31 +100,9 @@ public void onFailure(Exception e) { return null; } - private int getFirebaseRotation(){ - int result; - switch (mRotation) { - case 0: - result = FirebaseVisionImageMetadata.ROTATION_0; - break; - case 90: - result = FirebaseVisionImageMetadata.ROTATION_90; - break; - case 180: - result = FirebaseVisionImageMetadata.ROTATION_180; - break; - case -90: - result = FirebaseVisionImageMetadata.ROTATION_270; - break; - default: - result = FirebaseVisionImageMetadata.ROTATION_0; - Log.e(TAG, "Bad rotation value: " + mRotation); - } - return result; - } - - private WritableArray serializeEventData(List textBlocks) { + private WritableArray serializeEventData(List textBlocks) { WritableArray textBlocksList = Arguments.createArray(); - for (FirebaseVisionText.TextBlock block: textBlocks) { + for (Text.TextBlock block: textBlocks) { WritableMap serializedTextBlock = serializeBloc(block); if (mImageDimensions.getFacing() == CameraView.FACING_FRONT) { serializedTextBlock = rotateTextX(serializedTextBlock); @@ -137,10 +113,10 @@ private WritableArray serializeEventData(List text return textBlocksList; } - private WritableMap serializeBloc(FirebaseVisionText.TextBlock block) { + private WritableMap serializeBloc(Text.TextBlock block) { WritableMap encodedText = Arguments.createMap(); WritableArray lines = Arguments.createArray(); - for (FirebaseVisionText.Line line : block.getLines()) { + for (Text.Line line : block.getLines()) { lines.pushMap(serializeLine(line)); } encodedText.putArray("components", lines); @@ -155,10 +131,10 @@ private WritableMap serializeBloc(FirebaseVisionText.TextBlock block) { return encodedText; } - private WritableMap serializeLine(FirebaseVisionText.Line line) { + private WritableMap serializeLine(Text.Line line) { WritableMap encodedText = Arguments.createMap(); WritableArray lines = Arguments.createArray(); - for (FirebaseVisionText.Element element : line.getElements()) { + for (Text.Element element : line.getElements()) { lines.pushMap(serializeElement(element)); } encodedText.putArray("components", lines); @@ -173,7 +149,7 @@ private WritableMap serializeLine(FirebaseVisionText.Line line) { return encodedText; } - private WritableMap serializeElement(FirebaseVisionText.Element element) { + private WritableMap serializeElement(Text.Element element) { WritableMap encodedText = Arguments.createMap(); encodedText.putString("value", element.getText()); @@ -232,16 +208,17 @@ private WritableMap rotateTextX(WritableMap text) { text.putMap("bounds", newBounds); - ReadableArray oldComponents = text.getArray("components"); - WritableArray newComponents = Arguments.createArray(); - for (int i = 0; i < oldComponents.size(); ++i) { - WritableMap component = Arguments.createMap(); - component.merge(oldComponents.getMap(i)); - rotateTextX(component); - newComponents.pushMap(component); + if (text.hasKey("components")) { + ReadableArray oldComponents = text.getArray("components"); + WritableArray newComponents = Arguments.createArray(); + for (int i = 0; i < oldComponents.size(); ++i) { + WritableMap component = Arguments.createMap(); + component.merge(oldComponents.getMap(i)); + rotateTextX(component); + newComponents.pushMap(component); + } + text.putArray("components", newComponents); } - text.putArray("components", newComponents); - return text; } diff --git a/android/src/mlkit/java/org/reactnative/facedetector/FaceDetectorUtils.java b/android/src/mlkit/java/org/reactnative/facedetector/FaceDetectorUtils.java index 316c0875f..092c63b90 100644 --- a/android/src/mlkit/java/org/reactnative/facedetector/FaceDetectorUtils.java +++ b/android/src/mlkit/java/org/reactnative/facedetector/FaceDetectorUtils.java @@ -1,11 +1,12 @@ package org.reactnative.facedetector; +import android.graphics.PointF; + import com.facebook.react.bridge.Arguments; import com.facebook.react.bridge.ReadableMap; import com.facebook.react.bridge.WritableMap; -import com.google.firebase.ml.vision.common.FirebaseVisionPoint; -import com.google.firebase.ml.vision.face.FirebaseVisionFace; -import com.google.firebase.ml.vision.face.FirebaseVisionFaceLandmark; +import com.google.mlkit.vision.face.Face; +import com.google.mlkit.vision.face.FaceLandmark; public class FaceDetectorUtils { private static final String[] landmarkNames = { @@ -14,16 +15,16 @@ public class FaceDetectorUtils { "rightEarPosition", "rightEyePosition", "rightMouthPosition" }; - public static WritableMap serializeFace(FirebaseVisionFace face) { + public static WritableMap serializeFace(Face face) { return serializeFace(face, 1, 1, 0, 0, 0, 0); } - public static WritableMap serializeFace(FirebaseVisionFace face, double scaleX, double scaleY, int width, int height, int paddingLeft, int paddingTop) { + public static WritableMap serializeFace(Face face, double scaleX, double scaleY, int width, int height, int paddingLeft, int paddingTop) { WritableMap encodedFace = Arguments.createMap(); int id = 0; // If face tracking was enabled: - if (face.getTrackingId() != FirebaseVisionFace.INVALID_ID) { + if (face.getTrackingId() != null) { id = face.getTrackingId(); } @@ -33,31 +34,31 @@ public static WritableMap serializeFace(FirebaseVisionFace face, double scaleX, encodedFace.putDouble("yawAngle", face.getHeadEulerAngleY()); // If classification was enabled: - if (face.getSmilingProbability() != FirebaseVisionFace.UNCOMPUTED_PROBABILITY) { + if (face.getSmilingProbability() != null) { encodedFace.putDouble("smilingProbability", face.getSmilingProbability()); } - if (face.getLeftEyeOpenProbability() != FirebaseVisionFace.UNCOMPUTED_PROBABILITY) { + if (face.getLeftEyeOpenProbability() != null) { encodedFace.putDouble("leftEyeOpenProbability", face.getLeftEyeOpenProbability()); } - if (face.getRightEyeOpenProbability() != FirebaseVisionFace.UNCOMPUTED_PROBABILITY) { + if (face.getRightEyeOpenProbability() != null) { encodedFace.putDouble("rightEyeOpenProbability", face.getRightEyeOpenProbability()); } int[] landmarks = { - FirebaseVisionFaceLandmark.MOUTH_BOTTOM, - FirebaseVisionFaceLandmark.LEFT_CHEEK, - FirebaseVisionFaceLandmark.LEFT_EAR, - FirebaseVisionFaceLandmark.LEFT_EYE, - FirebaseVisionFaceLandmark.MOUTH_LEFT, - FirebaseVisionFaceLandmark.NOSE_BASE, - FirebaseVisionFaceLandmark.RIGHT_CHEEK, - FirebaseVisionFaceLandmark.RIGHT_EAR, - FirebaseVisionFaceLandmark.RIGHT_EYE, - FirebaseVisionFaceLandmark.MOUTH_RIGHT}; + FaceLandmark.MOUTH_BOTTOM, + FaceLandmark.LEFT_CHEEK, + FaceLandmark.LEFT_EAR, + FaceLandmark.LEFT_EYE, + FaceLandmark.MOUTH_LEFT, + FaceLandmark.NOSE_BASE, + FaceLandmark.RIGHT_CHEEK, + FaceLandmark.RIGHT_EAR, + FaceLandmark.RIGHT_EYE, + FaceLandmark.MOUTH_RIGHT}; for (int i = 0; i < landmarks.length; ++i) { - FirebaseVisionFaceLandmark landmark = face.getLandmark(landmarks[i]); + FaceLandmark landmark = face.getLandmark(landmarks[i]); if (landmark != null) { - encodedFace.putMap(landmarkNames[i], mapFromPoint(landmark.getPosition(), scaleX, scaleY, width, height, paddingLeft, paddingTop)); + encodedFace.putMap(landmarkNames[i], mapFromPointF(landmark.getPosition(), scaleX, scaleY, width, height, paddingLeft, paddingTop)); } } @@ -124,19 +125,19 @@ public static WritableMap changeAnglesDirection(WritableMap face) { return face; } - public static WritableMap mapFromPoint(FirebaseVisionPoint point, double scaleX, double scaleY, int width, int height, int paddingLeft, int paddingTop) { + public static WritableMap mapFromPointF(PointF point, double scaleX, double scaleY, int width, int height, int paddingLeft, int paddingTop) { WritableMap map = Arguments.createMap(); - Float x = point.getX(); - Float y = point.getY(); - if (point.getX() < width / 2) { + Float x = point.x; + Float y = point.y; + if (point.x < width / 2) { x = (x + paddingLeft / 2); - } else if (point.getX() > width / 2) { + } else if (point.x > width / 2) { x = (x - paddingLeft / 2); } - if (point.getY() < height / 2) { + if (point.y < height / 2) { y = (y + paddingTop / 2); - } else if (point.getY() > height / 2) { + } else if (point.y > height / 2) { y = (y - paddingTop / 2); } map.putDouble("x", x * scaleX); diff --git a/android/src/mlkit/java/org/reactnative/facedetector/RNFaceDetector.java b/android/src/mlkit/java/org/reactnative/facedetector/RNFaceDetector.java index 9df38ce5b..2e43698f7 100644 --- a/android/src/mlkit/java/org/reactnative/facedetector/RNFaceDetector.java +++ b/android/src/mlkit/java/org/reactnative/facedetector/RNFaceDetector.java @@ -3,24 +3,24 @@ import android.content.Context; import android.util.Log; -import com.google.firebase.ml.vision.FirebaseVision; -import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetector; -import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetectorOptions; +import com.google.mlkit.vision.face.FaceDetection; +import com.google.mlkit.vision.face.FaceDetector; +import com.google.mlkit.vision.face.FaceDetectorOptions; public class RNFaceDetector { - public static int ALL_CLASSIFICATIONS = FirebaseVisionFaceDetectorOptions.ALL_CLASSIFICATIONS; - public static int NO_CLASSIFICATIONS = FirebaseVisionFaceDetectorOptions.NO_CLASSIFICATIONS; - public static int ALL_LANDMARKS = FirebaseVisionFaceDetectorOptions.ALL_LANDMARKS; - public static int NO_LANDMARKS = FirebaseVisionFaceDetectorOptions.NO_LANDMARKS; - public static int ACCURATE_MODE = FirebaseVisionFaceDetectorOptions.ACCURATE; - public static int FAST_MODE = FirebaseVisionFaceDetectorOptions.FAST; + public static int ALL_CLASSIFICATIONS = FaceDetectorOptions.CLASSIFICATION_MODE_ALL; + public static int NO_CLASSIFICATIONS = FaceDetectorOptions.CLASSIFICATION_MODE_NONE; + public static int ALL_LANDMARKS = FaceDetectorOptions.LANDMARK_MODE_ALL; + public static int NO_LANDMARKS = FaceDetectorOptions.LANDMARK_MODE_NONE; + public static int ACCURATE_MODE = FaceDetectorOptions.PERFORMANCE_MODE_ACCURATE; + public static int FAST_MODE = FaceDetectorOptions.PERFORMANCE_MODE_FAST; // TODO contours detection is possible for MLKit-based face detector, implement this feature - public static int ALL_CONTOURS = FirebaseVisionFaceDetectorOptions.ALL_CONTOURS; - public static int NO_CONTOURS = FirebaseVisionFaceDetectorOptions.NO_CONTOURS; + public static int ALL_CONTOURS = FaceDetectorOptions.CONTOUR_MODE_ALL; + public static int NO_CONTOURS = FaceDetectorOptions.CONTOUR_MODE_NONE; - private FirebaseVisionFaceDetector mFaceDetector = null; - private FirebaseVisionFaceDetectorOptions.Builder mBuilder; + private FaceDetector mFaceDetector = null; + private FaceDetectorOptions.Builder mBuilder; private int mClassificationType = NO_CLASSIFICATIONS; private int mLandmarkType = NO_LANDMARKS; @@ -28,7 +28,7 @@ public class RNFaceDetector { private int mMode = FAST_MODE; public RNFaceDetector(Context context) { - mBuilder = new FirebaseVisionFaceDetectorOptions.Builder() + mBuilder = new FaceDetectorOptions.Builder() .setPerformanceMode(mMode) .setLandmarkMode(mLandmarkType) .setClassificationMode(mClassificationType) @@ -40,7 +40,7 @@ public boolean isOperational() { return true; } - public FirebaseVisionFaceDetector getDetector() { + public FaceDetector getDetector() { if (mFaceDetector == null) { createFaceDetector(); @@ -91,7 +91,7 @@ public void release() { } private void createFaceDetector() { - FirebaseVisionFaceDetectorOptions options = mBuilder.build(); - mFaceDetector = FirebaseVision.getInstance().getVisionFaceDetector(options); + FaceDetectorOptions options = mBuilder.build(); + mFaceDetector = FaceDetection.getClient(options); } } diff --git a/android/src/mlkit/java/org/reactnative/facedetector/tasks/FileFaceDetectionAsyncTask.java b/android/src/mlkit/java/org/reactnative/facedetector/tasks/FileFaceDetectionAsyncTask.java index ca5521f7f..073138afd 100644 --- a/android/src/mlkit/java/org/reactnative/facedetector/tasks/FileFaceDetectionAsyncTask.java +++ b/android/src/mlkit/java/org/reactnative/facedetector/tasks/FileFaceDetectionAsyncTask.java @@ -15,9 +15,9 @@ import com.facebook.react.bridge.WritableMap; import com.google.android.gms.tasks.OnFailureListener; import com.google.android.gms.tasks.OnSuccessListener; -import com.google.firebase.ml.vision.common.FirebaseVisionImage; -import com.google.firebase.ml.vision.face.FirebaseVisionFace; -import com.google.firebase.ml.vision.face.FirebaseVisionFaceDetector; +import com.google.mlkit.vision.common.InputImage; +import com.google.mlkit.vision.face.Face; +import com.google.mlkit.vision.face.FaceDetector; import java.io.File; import java.io.IOException; @@ -96,13 +96,13 @@ protected Void doInBackground(Void... voids) { } try { - FirebaseVisionImage image = FirebaseVisionImage.fromFilePath(mContext, Uri.parse(mUri)); - FirebaseVisionFaceDetector detector = mRNFaceDetector.getDetector(); - detector.detectInImage(image) + InputImage image = InputImage.fromFilePath(mContext, Uri.parse(mUri)); + FaceDetector detector = mRNFaceDetector.getDetector(); + detector.process(image) .addOnSuccessListener( - new OnSuccessListener>() { + new OnSuccessListener>() { @Override - public void onSuccess(List faces) { + public void onSuccess(List faces) { serializeEventData(faces); } }) @@ -122,11 +122,11 @@ public void onFailure(Exception e) { return null; } - private void serializeEventData(List faces) { + private void serializeEventData(List faces) { WritableMap result = Arguments.createMap(); WritableArray facesArray = Arguments.createArray(); - for(FirebaseVisionFace face : faces) { + for(Face face : faces) { WritableMap encodedFace = FaceDetectorUtils.serializeFace(face); encodedFace.putDouble("yawAngle", (-encodedFace.getDouble("yawAngle") + 360) % 360); encodedFace.putDouble("rollAngle", (-encodedFace.getDouble("rollAngle") + 360) % 360); diff --git a/docs/installation.md b/docs/installation.md index c1322c49b..c4a6430f8 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -284,8 +284,8 @@ apply plugin: 'com.google.gms.google-services' ... + android:name="com.google.mlkit.vision.DEPENDENCIES" + android:value="ocr" /> ``` diff --git a/docs/migrationV2.md b/docs/migrationV2.md index d72257d2a..7b6f039a2 100644 --- a/docs/migrationV2.md +++ b/docs/migrationV2.md @@ -72,7 +72,7 @@ android { ... diff --git a/examples/mlkit/android/build.gradle b/examples/mlkit/android/build.gradle index b2bee9f60..c4d383e09 100644 --- a/examples/mlkit/android/build.gradle +++ b/examples/mlkit/android/build.gradle @@ -14,7 +14,7 @@ buildscript { } dependencies { classpath 'com.android.tools.build:gradle:3.3.1' - classpath 'com.google.gms:google-services:4.0.1' + classpath 'com.google.gms:google-services:4.3.3' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files From 9d4dd4dcc849b23f4058e1e0cffe3d187dde48d8 Mon Sep 17 00:00:00 2001 From: David Govea Date: Tue, 21 Jul 2020 00:54:47 -0700 Subject: [PATCH 2/9] feat(ios): upgrade to standalone ML Kit SDKs (#2908) --- examples/mlkit/ios/Podfile | 2 +- ios/RN/BarcodeDetectorManagerMlkit.h | 5 +- ios/RN/BarcodeDetectorManagerMlkit.m | 187 +++++++++++++-------------- ios/RN/FaceDetectorManagerMlkit.h | 18 +-- ios/RN/FaceDetectorManagerMlkit.m | 116 ++++++++--------- ios/RN/RNFaceDetectorModuleMLKit.m | 2 +- ios/RN/TextDetectorManager.h | 5 +- ios/RN/TextDetectorManager.m | 19 +-- react-native-camera.podspec | 10 +- 9 files changed, 182 insertions(+), 182 deletions(-) diff --git a/examples/mlkit/ios/Podfile b/examples/mlkit/ios/Podfile index 90eaa2bdf..7d4e96479 100644 --- a/examples/mlkit/ios/Podfile +++ b/examples/mlkit/ios/Podfile @@ -1,5 +1,5 @@ # Uncomment the next line to define a global platform for your project -# platform :ios, '9.0' +platform :ios, '10.0' target 'mlkit' do # Uncomment the next line if you're using Swift or would like to use dynamic frameworks diff --git a/ios/RN/BarcodeDetectorManagerMlkit.h b/ios/RN/BarcodeDetectorManagerMlkit.h index 541218131..4cffd752b 100644 --- a/ios/RN/BarcodeDetectorManagerMlkit.h +++ b/ios/RN/BarcodeDetectorManagerMlkit.h @@ -1,7 +1,8 @@ #import -#if __has_include() - #import +#if __has_include() + #import + #import #endif @interface BarcodeDetectorManagerMlkit : NSObject diff --git a/ios/RN/BarcodeDetectorManagerMlkit.m b/ios/RN/BarcodeDetectorManagerMlkit.m index 3b41d1788..c3432e217 100644 --- a/ios/RN/BarcodeDetectorManagerMlkit.m +++ b/ios/RN/BarcodeDetectorManagerMlkit.m @@ -1,12 +1,11 @@ #import "BarcodeDetectorManagerMlkit.h" #import #import "RNFileSystem.h" -#if __has_include() +#if __has_include() @interface BarcodeDetectorManagerMlkit () -@property(nonatomic, strong) FIRVisionBarcodeDetector *barcodeRecognizer; -@property(nonatomic, strong) FIRVision *vision; -@property(nonatomic, assign) FIRVisionBarcodeFormat setOption; +@property(nonatomic, strong) MLKBarcodeScanner *barcodeRecognizer; +@property(nonatomic, assign) MLKBarcodeFormat setOption; @property(nonatomic, assign) NSInteger detectionMode; @property(nonatomic, assign) float scaleX; @property(nonatomic, assign) float scaleY; @@ -17,8 +16,7 @@ @implementation BarcodeDetectorManagerMlkit - (instancetype)init { if (self = [super init]) { - self.vision = [FIRVision vision]; - self.barcodeRecognizer = [_vision barcodeDetector]; + self.barcodeRecognizer = [MLKBarcodeScanner barcodeScanner]; } return self; } @@ -36,20 +34,20 @@ -(NSInteger)fetchDetectionMode + (NSDictionary *)constants { return @{ - @"CODE_128" : @(FIRVisionBarcodeFormatCode128), - @"CODE_39" : @(FIRVisionBarcodeFormatCode39), - @"CODE_93" : @(FIRVisionBarcodeFormatCode93), - @"CODABAR" : @(FIRVisionBarcodeFormatCodaBar), - @"EAN_13" : @(FIRVisionBarcodeFormatEAN13), - @"EAN_8" : @(FIRVisionBarcodeFormatEAN8), - @"ITF" : @(FIRVisionBarcodeFormatITF), - @"UPC_A" : @(FIRVisionBarcodeFormatUPCA), - @"UPC_E" : @(FIRVisionBarcodeFormatUPCE), - @"QR_CODE" : @(FIRVisionBarcodeFormatQRCode), - @"PDF417" : @(FIRVisionBarcodeFormatPDF417), - @"AZTEC" : @(FIRVisionBarcodeFormatAztec), - @"DATA_MATRIX" : @(FIRVisionBarcodeFormatDataMatrix), - @"ALL" : @(FIRVisionBarcodeFormatAll), + @"CODE_128" : @(MLKBarcodeFormatCode128), + @"CODE_39" : @(MLKBarcodeFormatCode39), + @"CODE_93" : @(MLKBarcodeFormatCode93), + @"CODABAR" : @(MLKBarcodeFormatCodaBar), + @"EAN_13" : @(MLKBarcodeFormatEAN13), + @"EAN_8" : @(MLKBarcodeFormatEAN8), + @"ITF" : @(MLKBarcodeFormatITF), + @"UPC_A" : @(MLKBarcodeFormatUPCA), + @"UPC_E" : @(MLKBarcodeFormatUPCE), + @"QR_CODE" : @(MLKBarcodeFormatQRCode), + @"PDF417" : @(MLKBarcodeFormatPDF417), + @"AZTEC" : @(MLKBarcodeFormatAztec), + @"DATA_MATRIX" : @(MLKBarcodeFormatDataMatrix), + @"ALL" : @(MLKBarcodeFormatAll), }; } @@ -60,11 +58,11 @@ - (void)setType:(id)json queue:(dispatch_queue_t)sessionQueue if (sessionQueue) { dispatch_async(sessionQueue, ^{ self.setOption = requestedValue; - FIRVisionBarcodeDetectorOptions *options = - [[FIRVisionBarcodeDetectorOptions alloc] + MLKBarcodeScannerOptions *options = + [[MLKBarcodeScannerOptions alloc] initWithFormats: requestedValue]; self.barcodeRecognizer = - [self.vision barcodeDetectorWithOptions:options]; + [MLKBarcodeScanner barcodeScannerWithOptions:options]; }); } } @@ -83,10 +81,11 @@ - (void)findBarcodesInFrame:(UIImage *)uiImage { self.scaleX = scaleX; self.scaleY = scaleY; - FIRVisionImage *image = [[FIRVisionImage alloc] initWithImage:uiImage]; + MLKVisionImage *image = [[MLKVisionImage alloc] initWithImage:uiImage]; + image.orientation = uiImage.imageOrientation; NSMutableArray *emptyResult = [[NSMutableArray alloc] init]; - [_barcodeRecognizer detectInImage:image - completion:^(NSArray *barcodes, NSError *error) { + [_barcodeRecognizer processImage:image + completion:^(NSArray *barcodes, NSError *error) { if (error != nil || barcodes == nil) { completed(emptyResult); } else { @@ -98,7 +97,7 @@ - (void)findBarcodesInFrame:(UIImage *)uiImage - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImage *)imageContainingBarcodes { NSMutableArray *result = [[NSMutableArray alloc] init]; - for (FIRVisionBarcode *barcode in barcodes) { + for (MLKBarcode *barcode in barcodes) { NSMutableDictionary *resultDict = [[NSMutableDictionary alloc] initWithCapacity:20]; // Boundaries of a barcode in image @@ -118,24 +117,24 @@ - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImag [UIImageJPEGRepresentation(imageContainingBarcodes, 1.0) writeToFile:path atomically:YES]; [resultDict setObject:path forKey:@"uri"]; - FIRVisionBarcodeValueType valueType = barcode.valueType; + MLKBarcodeValueType valueType = barcode.valueType; [resultDict setObject:[self getType:barcode.valueType] forKey:@"type"]; switch (valueType) { - case FIRVisionBarcodeValueTypeWiFi: + case MLKBarcodeValueTypeWiFi: if(barcode.wifi.ssid) {[resultDict setObject:barcode.wifi.ssid forKey:@"ssid"]; } if(barcode.wifi.password) {[resultDict setObject:barcode.wifi.password forKey:@"password"]; } if(barcode.wifi.type) { NSString *encryptionTypeString = @"UNKNOWN"; int type = barcode.wifi.type; switch (type) { - case FIRVisionBarcodeWiFiEncryptionTypeWEP: + case MLKBarcodeWiFiEncryptionTypeWEP: encryptionTypeString = @"WEP"; break; - case FIRVisionBarcodeWiFiEncryptionTypeWPA: + case MLKBarcodeWiFiEncryptionTypeWPA: encryptionTypeString = @"WPA"; break; - case FIRVisionBarcodeWiFiEncryptionTypeOpen: + case MLKBarcodeWiFiEncryptionTypeOpen: encryptionTypeString = @"Open"; break; default: @@ -144,27 +143,27 @@ - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImag [resultDict setObject:encryptionTypeString forKey:@"encryptionType"]; } break; - case FIRVisionBarcodeValueTypeURL: + case MLKBarcodeValueTypeURL: if(barcode.URL.url) { [resultDict setObject:barcode.URL.url forKey:@"url"]; } if(barcode.URL.title) { [resultDict setObject:barcode.URL.title forKey:@"title"]; } break; - case FIRVisionBarcodeValueTypeContactInfo: + case MLKBarcodeValueTypeContactInfo: if(barcode.contactInfo.addresses) { NSMutableArray *addresses = [[NSMutableArray alloc] init]; - for (FIRVisionBarcodeAddress *address in barcode.contactInfo.addresses) { + for (MLKBarcodeAddress *address in barcode.contactInfo.addresses) { [addresses addObject:[self processAddress:address]]; } [resultDict setObject:addresses forKey:@"addresses"]; } if(barcode.contactInfo.emails) { NSMutableArray *emails = [[NSMutableArray alloc] init]; - for (FIRVisionBarcodeEmail *email in barcode.contactInfo.emails) { + for (MLKBarcodeEmail *email in barcode.contactInfo.emails) { [emails addObject:[self processEmail:email]]; } [resultDict setObject:emails forKey:@"emails"]; } if(barcode.contactInfo.name) { - FIRVisionBarcodePersonName *name = barcode.contactInfo.name; + MLKBarcodePersonName *name = barcode.contactInfo.name; NSObject *nameObject = @{ @"formattedName" : name.formattedName ? name.formattedName : @"", @"firstName" : name.first ? name.first : @"", @@ -178,7 +177,7 @@ - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImag } if(barcode.contactInfo.phones) { NSMutableArray *phones = [[NSMutableArray alloc] init]; - for (FIRVisionBarcodePhone *phone in barcode.contactInfo.phones) { + for (MLKBarcodePhone *phone in barcode.contactInfo.phones) { [phones addObject:[self processPhone:phone]]; } [resultDict setObject:phones forKey:@"phones"]; @@ -186,15 +185,15 @@ - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImag if(barcode.contactInfo.urls) {[resultDict setObject:barcode.contactInfo.urls forKey:@"urls"]; } if(barcode.contactInfo.organization) {[resultDict setObject:barcode.contactInfo.organization forKey:@"organization"]; } break; - case FIRVisionBarcodeValueTypeSMS: + case MLKBarcodeValueTypeSMS: if(barcode.sms.message) {[resultDict setObject:barcode.sms.message forKey:@"message"]; } if(barcode.sms.phoneNumber) {[resultDict setObject:barcode.sms.phoneNumber forKey:@"phoneNumber"]; } break; - case FIRVisionBarcodeValueTypeGeographicCoordinates: + case MLKBarcodeValueTypeGeographicCoordinates: if(barcode.geoPoint.latitude) {[resultDict setObject:@(barcode.geoPoint.latitude) forKey:@"latitude"]; } if(barcode.geoPoint.longitude) {[resultDict setObject:@(barcode.geoPoint.longitude) forKey:@"longitude"]; } break; - case FIRVisionBarcodeValueTypeDriversLicense: + case MLKBarcodeValueTypeDriversLicense: if(barcode.driverLicense.firstName) {[resultDict setObject:barcode.driverLicense.firstName forKey:@"firstName"]; } if(barcode.driverLicense.middleName) {[resultDict setObject:barcode.driverLicense.middleName forKey:@"middleName"]; } if(barcode.driverLicense.lastName) {[resultDict setObject:barcode.driverLicense.lastName forKey:@"lastName"]; } @@ -210,7 +209,7 @@ - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImag if(barcode.driverLicense.issuingDate) {[resultDict setObject:barcode.driverLicense.issuingDate forKey:@"issuingDate"]; } if(barcode.driverLicense.issuingCountry) {[resultDict setObject:barcode.driverLicense.issuingCountry forKey:@"issuingCountry"]; } break; - case FIRVisionBarcodeValueTypeCalendarEvent: + case MLKBarcodeValueTypeCalendarEvent: if(barcode.calendarEvent.eventDescription) {[resultDict setObject:barcode.calendarEvent.eventDescription forKey:@"eventDescription"]; } if(barcode.calendarEvent.location) {[resultDict setObject:barcode.calendarEvent.location forKey:@"location"]; } if(barcode.calendarEvent.organizer) {[resultDict setObject:barcode.calendarEvent.organizer forKey:@"organizer"]; } @@ -223,13 +222,13 @@ - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImag [resultDict setObject:[self processDate:barcode.calendarEvent.end] forKey:@"end"]; } break; - case FIRVisionBarcodeValueTypePhone: + case MLKBarcodeValueTypePhone: if(barcode.phone.number) {[resultDict setObject:barcode.phone.number forKey:@"number"]; } if(barcode.phone.type) { [resultDict setObject:[self getPhoneType:barcode.phone.type] forKey:@"phoneType"]; } break; - case FIRVisionBarcodeValueTypeEmail: + case MLKBarcodeValueTypeEmail: if(barcode.email.address) {[resultDict setObject:barcode.email.address forKey:@"address"]; } if(barcode.email.body) {[resultDict setObject:barcode.email.body forKey:@"body"]; } if(barcode.email.subject) {[resultDict setObject:barcode.email.subject forKey:@"subject"]; } @@ -247,37 +246,37 @@ - (NSString *)getType:(int)type { NSString *barcodeType = @"UNKNOWN"; switch (type) { - case FIRVisionBarcodeValueTypeEmail: + case MLKBarcodeValueTypeEmail: barcodeType = @"EMAIL"; break; - case FIRVisionBarcodeValueTypePhone: + case MLKBarcodeValueTypePhone: barcodeType = @"PHONE"; break; - case FIRVisionBarcodeValueTypeCalendarEvent: + case MLKBarcodeValueTypeCalendarEvent: barcodeType = @"CALENDAR_EVENT"; break; - case FIRVisionBarcodeValueTypeDriversLicense: + case MLKBarcodeValueTypeDriversLicense: barcodeType = @"DRIVER_LICENSE"; break; - case FIRVisionBarcodeValueTypeGeographicCoordinates: + case MLKBarcodeValueTypeGeographicCoordinates: barcodeType = @"GEO"; break; - case FIRVisionBarcodeValueTypeSMS: + case MLKBarcodeValueTypeSMS: barcodeType = @"SMS"; break; - case FIRVisionBarcodeValueTypeContactInfo: + case MLKBarcodeValueTypeContactInfo: barcodeType = @"CONTACT_INFO"; break; - case FIRVisionBarcodeValueTypeWiFi: + case MLKBarcodeValueTypeWiFi: barcodeType = @"WIFI"; break; - case FIRVisionBarcodeValueTypeText: + case MLKBarcodeValueTypeText: barcodeType = @"TEXT"; break; - case FIRVisionBarcodeValueTypeISBN: + case MLKBarcodeValueTypeISBN: barcodeType = @"ISBN"; break; - case FIRVisionBarcodeValueTypeProduct: + case MLKBarcodeValueTypeProduct: barcodeType = @"PRODUCT"; break; default: @@ -290,14 +289,14 @@ - (NSString *)getPhoneType:(int)type { NSString *typeString = @"UNKNOWN"; switch (type) { - case FIRVisionBarcodePhoneTypeFax: + case MLKBarcodePhoneTypeFax: typeString = @"Fax"; break; - case FIRVisionBarcodePhoneTypeHome: + case MLKBarcodePhoneTypeHome: typeString = @"Home"; - case FIRVisionBarcodePhoneTypeWork: + case MLKBarcodePhoneTypeWork: typeString = @"Work"; - case FIRVisionBarcodePhoneTypeMobile: + case MLKBarcodePhoneTypeMobile: typeString = @"Mobile"; default: break; @@ -309,10 +308,10 @@ - (NSString *)getEmailType:(int)type { NSString *typeString = @"UNKNOWN"; switch (type) { - case FIRVisionBarcodeEmailTypeWork: + case MLKBarcodeEmailTypeWork: typeString = @"Work"; break; - case FIRVisionBarcodeEmailTypeHome: + case MLKBarcodeEmailTypeHome: typeString = @"Home"; default: break; @@ -320,7 +319,7 @@ - (NSString *)getEmailType:(int)type return typeString; } -- (NSDictionary *)processPhone:(FIRVisionBarcodePhone *)phone +- (NSDictionary *)processPhone:(MLKBarcodePhone *)phone { NSString *number = @""; NSString *typeString = @"UNKNOWN"; @@ -331,7 +330,7 @@ - (NSDictionary *)processPhone:(FIRVisionBarcodePhone *)phone return @{@"number" : number, @"phoneType" : typeString}; } -- (NSDictionary *)processAddress:(FIRVisionBarcodeAddress *)address +- (NSDictionary *)processAddress:(MLKBarcodeAddress *)address { NSArray *addressLines = [[NSArray alloc] init]; NSString *typeString = @"UNKNOWN"; @@ -339,10 +338,10 @@ - (NSDictionary *)processAddress:(FIRVisionBarcodeAddress *)address int type = address.type; NSString *typeString = @"UNKNOWN"; switch (type) { - case FIRVisionBarcodeAddressTypeWork: + case MLKBarcodeAddressTypeWork: typeString = @"Work"; break; - case FIRVisionBarcodeAddressTypeHome: + case MLKBarcodeAddressTypeHome: typeString = @"Home"; default: break; @@ -352,7 +351,7 @@ - (NSDictionary *)processAddress:(FIRVisionBarcodeAddress *)address return @{@"addressLines" : addressLines, @"addressType" : typeString}; } -- (NSDictionary *)processEmail:(FIRVisionBarcodeEmail *)email +- (NSDictionary *)processEmail:(MLKBarcodeEmail *)email { NSString *subject = @""; NSString *address =@""; @@ -388,10 +387,10 @@ - (NSDictionary *)processBounds:(CGRect)bounds } -- (NSDictionary *)processPoint:(FIRVisionPoint *)point +- (NSDictionary *)processPoint:(MLKVisionPoint *)point { - float originX = [point.x floatValue] * _scaleX; - float originY = [point.y floatValue] * _scaleY; + float originX = point.x * _scaleX; + float originY = point.y * _scaleY; NSDictionary *pointDict = @{ @"x" : @(originX), @"y" : @(originY) @@ -408,46 +407,46 @@ @interface BarcodeDetectorManagerMlkit () @implementation BarcodeDetectorManagerMlkit - (instancetype)init { - self = [super init]; - return self; + self = [super init]; + return self; } - (BOOL)isRealDetector { - return false; + return false; } - (void)findBarcodesInFrame:(UIImage *)image - scaleX:(float)scaleX - scaleY:(float)scaleY - completed:(void (^)(NSArray *result))completed; + scaleX:(float)scaleX + scaleY:(float)scaleY + completed:(void (^)(NSArray *result))completed; { - NSLog(@"BarcodeDetector not installed, stub used!"); - NSArray *barcodes = @[ @"Error, Barcode Detector not installed" ]; - completed(barcodes); + NSLog(@"BarcodeDetector not installed, stub used!"); + NSArray *barcodes = @[ @"Error, Barcode Detector not installed" ]; + completed(barcodes); } + (NSDictionary *)constants { - return @{ - @"CODE_128" : @{}, - @"CODE_39" : @{}, - @"CODE_93" : @{}, - @"CODABAR" : @{}, - @"EAN_13" : @{}, - @"EAN_8" : @{}, - @"ITF" : @{}, - @"UPC_A" : @{}, - @"UPC_E" : @{}, - @"QR_CODE" : @{}, - @"PDF417" : @{}, - @"AZTEC" : @{}, - @"DATA_MATRIX" : @{}, - }; + return @{ + @"CODE_128" : @{}, + @"CODE_39" : @{}, + @"CODE_93" : @{}, + @"CODABAR" : @{}, + @"EAN_13" : @{}, + @"EAN_8" : @{}, + @"ITF" : @{}, + @"UPC_A" : @{}, + @"UPC_E" : @{}, + @"QR_CODE" : @{}, + @"PDF417" : @{}, + @"AZTEC" : @{}, + @"DATA_MATRIX" : @{}, + }; } - (void)setType:(id)json queue:(dispatch_queue_t)sessionQueue { - return; + return; } @end diff --git a/ios/RN/FaceDetectorManagerMlkit.h b/ios/RN/FaceDetectorManagerMlkit.h index 710364d73..317311eb0 100644 --- a/ios/RN/FaceDetectorManagerMlkit.h +++ b/ios/RN/FaceDetectorManagerMlkit.h @@ -1,20 +1,22 @@ #import -#if __has_include() - #import +#if __has_include() + #import + #import + typedef NS_ENUM(NSInteger, RNFaceDetectionMode) { - RNFaceDetectionFastMode = FIRVisionFaceDetectorPerformanceModeFast, - RNFaceDetectionAccurateMode = FIRVisionFaceDetectorPerformanceModeAccurate + RNFaceDetectionFastMode = MLKFaceDetectorPerformanceModeFast, + RNFaceDetectionAccurateMode = MLKFaceDetectorPerformanceModeAccurate }; typedef NS_ENUM(NSInteger, RNFaceDetectionLandmarks) { - RNFaceDetectAllLandmarks = FIRVisionFaceDetectorLandmarkModeAll, - RNFaceDetectNoLandmarks = FIRVisionFaceDetectorLandmarkModeNone + RNFaceDetectAllLandmarks = MLKFaceDetectorLandmarkModeAll, + RNFaceDetectNoLandmarks = MLKFaceDetectorLandmarkModeNone }; typedef NS_ENUM(NSInteger, RNFaceDetectionClassifications) { - RNFaceRunAllClassifications = FIRVisionFaceDetectorClassificationModeAll, - RNFaceRunNoClassifications = FIRVisionFaceDetectorClassificationModeNone + RNFaceRunAllClassifications = MLKFaceDetectorClassificationModeAll, + RNFaceRunNoClassifications = MLKFaceDetectorClassificationModeNone }; #endif diff --git a/ios/RN/FaceDetectorManagerMlkit.m b/ios/RN/FaceDetectorManagerMlkit.m index 0c66763d3..d06491b92 100644 --- a/ios/RN/FaceDetectorManagerMlkit.m +++ b/ios/RN/FaceDetectorManagerMlkit.m @@ -1,11 +1,10 @@ #import "FaceDetectorManagerMlkit.h" #import -#if __has_include() +#if __has_include() @interface FaceDetectorManagerMlkit () -@property(nonatomic, strong) FIRVisionFaceDetector *faceRecognizer; -@property(nonatomic, strong) FIRVision *vision; -@property(nonatomic, strong) FIRVisionFaceDetectorOptions *options; +@property(nonatomic, strong) MLKFaceDetector *faceRecognizer; +@property(nonatomic, strong) MLKFaceDetectorOptions *options; @property(nonatomic, assign) float scaleX; @property(nonatomic, assign) float scaleY; @end @@ -15,13 +14,12 @@ @implementation FaceDetectorManagerMlkit - (instancetype)init { if (self = [super init]) { - self.options = [[FIRVisionFaceDetectorOptions alloc] init]; - self.options.performanceMode = FIRVisionFaceDetectorPerformanceModeFast; - self.options.landmarkMode = FIRVisionFaceDetectorLandmarkModeNone; - self.options.classificationMode = FIRVisionFaceDetectorClassificationModeNone; - - self.vision = [FIRVision vision]; - self.faceRecognizer = [_vision faceDetectorWithOptions:_options]; + self.options = [[MLKFaceDetectorOptions alloc] init]; + self.options.performanceMode = MLKFaceDetectorPerformanceModeFast; + self.options.landmarkMode = MLKFaceDetectorLandmarkModeNone; + self.options.classificationMode = MLKFaceDetectorClassificationModeNone; + + self.faceRecognizer = [MLKFaceDetector faceDetectorWithOptions:_options]; } return self; } @@ -57,7 +55,7 @@ - (void)setTracking:(id)json queue:(dispatch_queue_t)sessionQueue dispatch_async(sessionQueue, ^{ self.options.trackingEnabled = requestedValue; self.faceRecognizer = - [self.vision faceDetectorWithOptions:self.options]; + [MLKFaceDetector faceDetectorWithOptions:self.options]; }); } } @@ -71,7 +69,7 @@ - (void)setLandmarksMode:(id)json queue:(dispatch_queue_t)sessionQueue dispatch_async(sessionQueue, ^{ self.options.landmarkMode = requestedValue; self.faceRecognizer = - [self.vision faceDetectorWithOptions:self.options]; + [MLKFaceDetector faceDetectorWithOptions:self.options]; }); } } @@ -85,7 +83,7 @@ - (void)setPerformanceMode:(id)json queue:(dispatch_queue_t)sessionQueue dispatch_async(sessionQueue, ^{ self.options.performanceMode = requestedValue; self.faceRecognizer = - [self.vision faceDetectorWithOptions:self.options]; + [MLKFaceDetector faceDetectorWithOptions:self.options]; }); } } @@ -99,7 +97,7 @@ - (void)setClassificationMode:(id)json queue:(dispatch_queue_t)sessionQueue dispatch_async(sessionQueue, ^{ self.options.classificationMode = requestedValue; self.faceRecognizer = - [self.vision faceDetectorWithOptions:self.options]; + [MLKFaceDetector faceDetectorWithOptions:self.options]; }); } } @@ -112,11 +110,11 @@ - (void)findFacesInFrame:(UIImage *)uiImage { self.scaleX = scaleX; self.scaleY = scaleY; - FIRVisionImage *image = [[FIRVisionImage alloc] initWithImage:uiImage]; + MLKVisionImage *image = [[MLKVisionImage alloc] initWithImage:uiImage]; NSMutableArray *emptyResult = [[NSMutableArray alloc] init]; [_faceRecognizer processImage:image - completion:^(NSArray *faces, NSError *error) { + completion:^(NSArray *faces, NSError *error) { if (error != nil || faces == nil) { completed(emptyResult); } else { @@ -128,7 +126,7 @@ - (void)findFacesInFrame:(UIImage *)uiImage - (NSArray *)processFaces:(NSArray *)faces { NSMutableArray *result = [[NSMutableArray alloc] init]; - for (FIRVisionFace *face in faces) { + for (MLKFace *face in faces) { NSMutableDictionary *resultDict = [[NSMutableDictionary alloc] initWithCapacity:20]; // Boundaries of face in image @@ -153,71 +151,71 @@ - (NSArray *)processFaces:(NSArray *)faces // If landmark detection was enabled (mouth, ears, eyes, cheeks, and // nose available): /** Midpoint of the left ear tip and left ear lobe. */ - FIRVisionFaceLandmark *leftEar = - [face landmarkOfType:FIRFaceLandmarkTypeLeftEar]; + MLKFaceLandmark *leftEar = + [face landmarkOfType:MLKFaceLandmarkTypeLeftEar]; if (leftEar != nil) { [resultDict setObject:[self processPoint:leftEar.position] forKey:@"leftEarPosition"]; } /** Midpoint of the right ear tip and right ear lobe. */ - FIRVisionFaceLandmark *rightEar = - [face landmarkOfType:FIRFaceLandmarkTypeRightEar]; + MLKFaceLandmark *rightEar = + [face landmarkOfType:MLKFaceLandmarkTypeRightEar]; if (rightEar != nil) { [resultDict setObject:[self processPoint:rightEar.position] forKey:@"rightEarPosition"]; } /** Center of the bottom lip. */ - FIRVisionFaceLandmark *mouthBottom = - [face landmarkOfType:FIRFaceLandmarkTypeMouthBottom]; + MLKFaceLandmark *mouthBottom = + [face landmarkOfType:MLKFaceLandmarkTypeMouthBottom]; if (mouthBottom != nil) { [resultDict setObject:[self processPoint:mouthBottom.position] forKey:@"bottomMouthPosition"]; } /** Right corner of the mouth */ - FIRVisionFaceLandmark *mouthRight = - [face landmarkOfType:FIRFaceLandmarkTypeMouthRight]; + MLKFaceLandmark *mouthRight = + [face landmarkOfType:MLKFaceLandmarkTypeMouthRight]; if (mouthRight != nil) { [resultDict setObject:[self processPoint:mouthRight.position] forKey:@"rightMouthPosition"]; } /** Left corner of the mouth */ - FIRVisionFaceLandmark *mouthLeft = - [face landmarkOfType:FIRFaceLandmarkTypeMouthLeft]; + MLKFaceLandmark *mouthLeft = + [face landmarkOfType:MLKFaceLandmarkTypeMouthLeft]; if (mouthLeft != nil) { [resultDict setObject:[self processPoint:mouthLeft.position] forKey:@"leftMouthPosition"]; } /** Left eye. */ - FIRVisionFaceLandmark *eyeLeft = - [face landmarkOfType:FIRFaceLandmarkTypeLeftEye]; + MLKFaceLandmark *eyeLeft = + [face landmarkOfType:MLKFaceLandmarkTypeLeftEye]; if (eyeLeft != nil) { [resultDict setObject:[self processPoint:eyeLeft.position] forKey:@"leftEyePosition"]; } /** Right eye. */ - FIRVisionFaceLandmark *eyeRight = - [face landmarkOfType:FIRFaceLandmarkTypeRightEye]; + MLKFaceLandmark *eyeRight = + [face landmarkOfType:MLKFaceLandmarkTypeRightEye]; if (eyeRight != nil) { [resultDict setObject:[self processPoint:eyeRight.position] forKey:@"rightEyePosition"]; } /** Left cheek. */ - FIRVisionFaceLandmark *cheekLeft = - [face landmarkOfType:FIRFaceLandmarkTypeLeftCheek]; + MLKFaceLandmark *cheekLeft = + [face landmarkOfType:MLKFaceLandmarkTypeLeftCheek]; if (cheekLeft != nil) { [resultDict setObject:[self processPoint:cheekLeft.position] forKey:@"leftCheekPosition"]; } /** Right cheek. */ - FIRVisionFaceLandmark *cheekRight = - [face landmarkOfType:FIRFaceLandmarkTypeRightCheek]; + MLKFaceLandmark *cheekRight = + [face landmarkOfType:MLKFaceLandmarkTypeRightCheek]; if (cheekRight != nil) { [resultDict setObject:[self processPoint:cheekRight.position] forKey:@"rightCheekPosition"]; } /** Midpoint between the nostrils where the nose meets the face. */ - FIRVisionFaceLandmark *noseBase = - [face landmarkOfType:FIRFaceLandmarkTypeNoseBase]; + MLKFaceLandmark *noseBase = + [face landmarkOfType:MLKFaceLandmarkTypeNoseBase]; if (noseBase != nil) { [resultDict setObject:[self processPoint:noseBase.position] forKey:@"noseBasePosition"]; @@ -256,10 +254,10 @@ - (NSDictionary *)processBounds:(CGRect)bounds return boundsDict; } -- (NSDictionary *)processPoint:(FIRVisionPoint *)point +- (NSDictionary *)processPoint:(MLKVisionPoint *)point { - float originX = [point.x floatValue] * _scaleX; - float originY = [point.y floatValue] * _scaleY; + float originX = point.x * _scaleX; + float originY = point.y * _scaleY; NSDictionary *pointDict = @{ @"x" : @(originX), @@ -277,50 +275,50 @@ @interface FaceDetectorManagerMlkit () @implementation FaceDetectorManagerMlkit - (instancetype)init { - self = [super init]; - return self; + self = [super init]; + return self; } - (BOOL)isRealDetector { - return false; + return false; } - (NSArray *)findFacesInFrame:(UIImage *)image - scaleX:(float)scaleX - scaleY:(float)scaleY - completed:(void (^)(NSArray *result))completed; + scaleX:(float)scaleX + scaleY:(float)scaleY + completed:(void (^)(NSArray *result))completed; { - NSLog(@"FaceDetector not installed, stub used!"); - NSArray *features = @[ @"Error, Face Detector not installed" ]; - return features; + NSLog(@"FaceDetector not installed, stub used!"); + NSArray *features = @[ @"Error, Face Detector not installed" ]; + return features; } - (void)setTracking:(id)json:(dispatch_queue_t)sessionQueue { - return; + return; } - (void)setLandmarksMode:(id)json:(dispatch_queue_t)sessionQueue { - return; + return; } - (void)setPerformanceMode:(id)json:(dispatch_queue_t)sessionQueue { - return; + return; } - (void)setClassificationMode:(id)json:(dispatch_queue_t)sessionQueue { - return; + return; } + (NSDictionary *)constantsToExport { - return @{ - @"Mode" : @{}, - @"Landmarks" : @{}, - @"Classifications" : @{} - }; + return @{ + @"Mode" : @{}, + @"Landmarks" : @{}, + @"Classifications" : @{} + }; } @end diff --git a/ios/RN/RNFaceDetectorModuleMLKit.m b/ios/RN/RNFaceDetectorModuleMLKit.m index d2bacb72f..164f0d23e 100644 --- a/ios/RN/RNFaceDetectorModuleMLKit.m +++ b/ios/RN/RNFaceDetectorModuleMLKit.m @@ -58,7 +58,7 @@ - (NSDictionary *)constantsToExport reject(@"E_FACE_DETECTION_FAILED", [NSString stringWithFormat:@"The file does not exist. Given path: `%@`.", path], nil); return; } - FIRVisionFaceDetectorOptions *newOptions = [[FIRVisionFaceDetectorOptions alloc] init]; + MLKFaceDetectorOptions *newOptions = [[MLKFaceDetectorOptions alloc] init]; if (options[kDetectLandmarksOptionName]) { newOptions.landmarkMode = [options[kDetectLandmarksOptionName] integerValue]; } diff --git a/ios/RN/TextDetectorManager.h b/ios/RN/TextDetectorManager.h index 96c83ed3a..990c73dfb 100644 --- a/ios/RN/TextDetectorManager.h +++ b/ios/RN/TextDetectorManager.h @@ -1,5 +1,6 @@ -#if __has_include() - #import +#if __has_include() + #import + #import #endif @interface TextDetectorManager : NSObject typedef void(^postRecognitionBlock)(NSArray *textBlocks); diff --git a/ios/RN/TextDetectorManager.m b/ios/RN/TextDetectorManager.m index bc8cd67f2..e80f24b7d 100644 --- a/ios/RN/TextDetectorManager.m +++ b/ios/RN/TextDetectorManager.m @@ -1,8 +1,8 @@ #import "TextDetectorManager.h" -#if __has_include() +#if __has_include() @interface TextDetectorManager () -@property(nonatomic, strong) FIRVisionTextRecognizer *textRecognizer; +@property(nonatomic, strong) MLKTextRecognizer *textRecognizer; @property(nonatomic, assign) float scaleX; @property(nonatomic, assign) float scaleY; @end @@ -12,8 +12,8 @@ @implementation TextDetectorManager - (instancetype)init { if (self = [super init]) { - FIRVision *vision = [FIRVision vision]; - self.textRecognizer = [vision onDeviceTextRecognizer]; + self.textRecognizer = [MLKTextRecognizer textRecognizer]; + } return self; } @@ -27,10 +27,11 @@ - (void)findTextBlocksInFrame:(UIImage *)uiImage scaleX:(float)scaleX scaleY:(fl { self.scaleX = scaleX; self.scaleY = scaleY; - FIRVisionImage *image = [[FIRVisionImage alloc] initWithImage:uiImage]; + MLKVisionImage *image = [[MLKVisionImage alloc] initWithImage:uiImage]; + image.orientation = uiImage.imageOrientation; NSMutableArray *textBlocks = [[NSMutableArray alloc] init]; [_textRecognizer processImage:image - completion:^(FIRVisionText *_Nullable result, + completion:^(MLKText *_Nullable result, NSError *_Nullable error) { if (error != nil || result == nil) { completed(textBlocks); @@ -43,7 +44,7 @@ - (void)findTextBlocksInFrame:(UIImage *)uiImage scaleX:(float)scaleX scaleY:(fl - (NSArray *)processBlocks:(NSArray *)features { NSMutableArray *textBlocks = [[NSMutableArray alloc] init]; - for (FIRVisionTextBlock *textBlock in features) { + for (MLKTextBlock *textBlock in features) { NSDictionary *textBlockDict = @{@"type": @"block", @"value" : textBlock.text, @"bounds" : [self processBounds:textBlock.frame], @"components" : [self processLine:textBlock.lines]}; [textBlocks addObject:textBlockDict]; @@ -54,7 +55,7 @@ - (NSArray *)processBlocks:(NSArray *)features -(NSArray *)processLine:(NSArray *)lines { NSMutableArray *lineBlocks = [[NSMutableArray alloc] init]; - for (FIRVisionTextLine *textLine in lines) { + for (MLKTextLine *textLine in lines) { NSDictionary *textLineDict = @{@"type": @"line", @"value" : textLine.text, @"bounds" : [self processBounds:textLine.frame], @"components" : [self processElement:textLine.elements]}; [lineBlocks addObject:textLineDict]; @@ -65,7 +66,7 @@ -(NSArray *)processLine:(NSArray *)lines -(NSArray *)processElement:(NSArray *)elements { NSMutableArray *elementBlocks = [[NSMutableArray alloc] init]; - for (FIRVisionTextElement *textElement in elements) { + for (MLKTextElement *textElement in elements) { NSDictionary *textElementDict = @{@"type": @"element", @"value" : textElement.text, @"bounds" : [self processBounds:textElement.frame]}; [elementBlocks addObject:textElementDict]; diff --git a/react-native-camera.podspec b/react-native-camera.podspec index f8bb4d23e..c9ba1d4b8 100644 --- a/react-native-camera.podspec +++ b/react-native-camera.podspec @@ -26,22 +26,20 @@ Pod::Spec.new do |s| s.subspec "TextDetector" do |ss| ss.dependency 'react-native-camera/RN' ss.dependency 'react-native-camera/RCT' - ss.dependency 'Firebase/MLVision' - ss.dependency 'Firebase/MLVisionTextModel' + ss.dependency 'GoogleMLKit/TextRecognition' + end s.subspec "FaceDetectorMLKit" do |ss| ss.dependency 'react-native-camera/RN' ss.dependency 'react-native-camera/RCT' - ss.dependency 'Firebase/MLVision' - ss.dependency 'Firebase/MLVisionFaceModel' + ss.dependency 'GoogleMLKit/FaceDetection' end s.subspec "BarcodeDetectorMLKit" do |ss| ss.dependency 'react-native-camera/RN' ss.dependency 'react-native-camera/RCT' - ss.dependency 'Firebase/MLVision' - ss.dependency 'Firebase/MLVisionBarcodeModel' + ss.dependency 'GoogleMLKit/BarcodeScanning' end s.default_subspecs = "RN", "RCT" From 84f7442745bebf58bbce8392cbecffabaa97870b Mon Sep 17 00:00:00 2001 From: David Govea Date: Tue, 28 Jul 2020 18:17:38 -0700 Subject: [PATCH 3/9] fix(ios): use correct spelling for "pronunciation" It appears that this was a typo -- Android was already using correct spelling --- ios/RN/BarcodeDetectorManagerMlkit.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/RN/BarcodeDetectorManagerMlkit.m b/ios/RN/BarcodeDetectorManagerMlkit.m index c3432e217..4ade834db 100644 --- a/ios/RN/BarcodeDetectorManagerMlkit.m +++ b/ios/RN/BarcodeDetectorManagerMlkit.m @@ -170,7 +170,7 @@ - (NSArray *)processBarcodes:(NSArray *)barcodes imageContainingBarcodes:(UIImag @"middleName" : name.middle ? name.middle : @"", @"lastName" : name.last ? name.last : @"", @"prefix" : name.prefix ? name.prefix : @"", - @"pronounciation" : name.pronounciation ? name.pronounciation : @"", + @"pronunciation" : name.pronunciation ? name.pronunciation : @"", @"suffix" : name.suffix ? name.suffix : @"", }; [resultDict setObject:nameObject forKey:@"name"]; From 39fcc5116a095bec6203b225d555e348cabc5551 Mon Sep 17 00:00:00 2001 From: David Govea Date: Tue, 28 Jul 2020 19:12:35 -0700 Subject: [PATCH 4/9] chore(example): upgrade RN 0.59.x to fix build in Xcode11 --- examples/mlkit/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/mlkit/package.json b/examples/mlkit/package.json index 4fe57bfb6..820a84d6b 100644 --- a/examples/mlkit/package.json +++ b/examples/mlkit/package.json @@ -8,7 +8,7 @@ }, "dependencies": { "react": "16.8.3", - "react-native": "0.59.1" + "react-native": "~0.59.9" }, "devDependencies": { "@babel/core": "^7.6.2", From 84ef60a4ee00b279c45a694981270c803e80ad17 Mon Sep 17 00:00:00 2001 From: David Govea Date: Tue, 28 Jul 2020 19:49:51 -0700 Subject: [PATCH 5/9] feat(mlkit): add on-device image labeling from ML Kit --- android/build.gradle | 1 + .../reactnative/camera/CameraViewManager.java | 7 ++ .../org/reactnative/camera/RNCameraView.java | 59 +++++++++- .../camera/RNCameraViewHelper.java | 27 +++++ .../events/LabelDetectionErrorEvent.java | 54 +++++++++ .../camera/events/LabelsDetectedEvent.java | 61 ++++++++++ .../tasks/ImageLabelerAsyncTaskDelegate.java | 13 ++ .../camera/tasks/ImageLabelerAsyncTask.java | 111 ++++++++++++++++++ .../imagelabeler/RNImageLabeler.java | 48 ++++++++ ios/RN/LabelDetectorManagerMlkit.h | 16 +++ ios/RN/LabelDetectorManagerMlkit.m | 90 ++++++++++++++ ios/RN/RNCamera.h | 4 + ios/RN/RNCamera.m | 90 +++++++++++++- ios/RN/RNCameraManager.m | 9 +- react-native-camera.podspec | 6 + src/RNCamera.js | 37 +++++- 16 files changed, 619 insertions(+), 14 deletions(-) create mode 100644 android/src/main/java/org/reactnative/camera/events/LabelDetectionErrorEvent.java create mode 100644 android/src/main/java/org/reactnative/camera/events/LabelsDetectedEvent.java create mode 100644 android/src/main/java/org/reactnative/camera/tasks/ImageLabelerAsyncTaskDelegate.java create mode 100644 android/src/mlkit/java/org/reactnative/camera/tasks/ImageLabelerAsyncTask.java create mode 100644 android/src/mlkit/java/org/reactnative/imagelabeler/RNImageLabeler.java create mode 100644 ios/RN/LabelDetectorManagerMlkit.h create mode 100644 ios/RN/LabelDetectorManagerMlkit.m diff --git a/android/build.gradle b/android/build.gradle index 4d97158a3..3c7929505 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -77,4 +77,5 @@ dependencies { mlkitImplementation "com.google.android.gms:play-services-mlkit-text-recognition:${safeExtGet('mlkit-text-recognition', '16.0.0')}" mlkitImplementation "com.google.mlkit:barcode-scanning:${safeExtGet('mlkit-barcode-scanning', '16.0.0')}" mlkitImplementation "com.google.mlkit:face-detection:${safeExtGet('mlkit-face-detection', '16.0.0')}" + mlkitImplementation "com.google.mlkit:image-labeling:${safeExtGet('mlkit-image-labeling', '16.0.0')}" } diff --git a/android/src/main/java/org/reactnative/camera/CameraViewManager.java b/android/src/main/java/org/reactnative/camera/CameraViewManager.java index 83517d87a..71c0fafb2 100644 --- a/android/src/main/java/org/reactnative/camera/CameraViewManager.java +++ b/android/src/main/java/org/reactnative/camera/CameraViewManager.java @@ -21,8 +21,10 @@ public enum Events { EVENT_ON_BAR_CODE_READ("onBarCodeRead"), EVENT_ON_FACES_DETECTED("onFacesDetected"), EVENT_ON_BARCODES_DETECTED("onGoogleVisionBarcodesDetected"), + EVENT_ON_LABELS_DETECTED("onLabelsDetected"), EVENT_ON_FACE_DETECTION_ERROR("onFaceDetectionError"), EVENT_ON_BARCODE_DETECTION_ERROR("onGoogleVisionBarcodeDetectionError"), + EVENT_ON_LABEL_DETECTION_ERROR("onLabelDetectionError"), EVENT_ON_TEXT_RECOGNIZED("onTextRecognized"), EVENT_ON_PICTURE_TAKEN("onPictureTaken"), EVENT_ON_PICTURE_SAVED("onPictureSaved"), @@ -217,6 +219,11 @@ public void setTextRecognizing(RNCameraView view, boolean textRecognizerEnabled) view.setShouldRecognizeText(textRecognizerEnabled); } + @ReactProp(name = "labelDetectorEnabled") + public void setLabelDetecting(RNCameraView view, boolean labelDetectorEnabled) { + view.setShouldDetectLabels(labelDetectorEnabled); + } + /**---limit scan area addition---**/ @ReactProp(name = "rectOfInterest") public void setRectOfInterest(RNCameraView view, ReadableMap coordinates) { diff --git a/android/src/main/java/org/reactnative/camera/RNCameraView.java b/android/src/main/java/org/reactnative/camera/RNCameraView.java index 99e00d541..678a06815 100644 --- a/android/src/main/java/org/reactnative/camera/RNCameraView.java +++ b/android/src/main/java/org/reactnative/camera/RNCameraView.java @@ -30,6 +30,7 @@ import org.reactnative.camera.tasks.*; import org.reactnative.camera.utils.RNFileUtils; import org.reactnative.facedetector.RNFaceDetector; +import org.reactnative.imagelabeler.RNImageLabeler; import java.io.ByteArrayOutputStream; import java.io.File; @@ -39,7 +40,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; public class RNCameraView extends CameraView implements LifecycleEventListener, BarCodeScannerAsyncTaskDelegate, FaceDetectorAsyncTaskDelegate, - BarcodeDetectorAsyncTaskDelegate, TextRecognizerAsyncTaskDelegate, PictureSavedDelegate { + BarcodeDetectorAsyncTaskDelegate, TextRecognizerAsyncTaskDelegate, ImageLabelerAsyncTaskDelegate, PictureSavedDelegate { private ThemedReactContext mThemedReactContext; private Queue mPictureTakenPromises = new ConcurrentLinkedQueue<>(); private Map mPictureTakenOptions = new ConcurrentHashMap<>(); @@ -64,16 +65,19 @@ public class RNCameraView extends CameraView implements LifecycleEventListener, public volatile boolean faceDetectorTaskLock = false; public volatile boolean googleBarcodeDetectorTaskLock = false; public volatile boolean textRecognizerTaskLock = false; + public volatile boolean imageLabelerTaskLock = false; // Scanning-related properties private MultiFormatReader mMultiFormatReader; private RNFaceDetector mFaceDetector; private RNBarcodeDetector mGoogleBarcodeDetector; + private RNImageLabeler mImageLabeler; private boolean mShouldDetectFaces = false; private boolean mShouldGoogleDetectBarcodes = false; private boolean mShouldScanBarCodes = false; private boolean mShouldRecognizeText = false; private boolean mShouldDetectTouches = false; + private boolean mShouldDetectLabels = false; private int mFaceDetectorMode = RNFaceDetector.FAST_MODE; private int mFaceDetectionLandmarks = RNFaceDetector.NO_LANDMARKS; private int mFaceDetectionClassifications = RNFaceDetector.NO_CLASSIFICATIONS; @@ -166,7 +170,9 @@ public void onFramePreview(CameraView cameraView, byte[] data, int width, int he boolean willCallFaceTask = mShouldDetectFaces && !faceDetectorTaskLock && cameraView instanceof FaceDetectorAsyncTaskDelegate; boolean willCallGoogleBarcodeTask = mShouldGoogleDetectBarcodes && !googleBarcodeDetectorTaskLock && cameraView instanceof BarcodeDetectorAsyncTaskDelegate; boolean willCallTextTask = mShouldRecognizeText && !textRecognizerTaskLock && cameraView instanceof TextRecognizerAsyncTaskDelegate; - if (!willCallBarCodeTask && !willCallFaceTask && !willCallGoogleBarcodeTask && !willCallTextTask) { + boolean willCallLabelTask = mShouldDetectLabels && !imageLabelerTaskLock && cameraView instanceof ImageLabelerAsyncTaskDelegate; + + if (!willCallBarCodeTask && !willCallFaceTask && !willCallGoogleBarcodeTask && !willCallTextTask && !willCallLabelTask) { return; } @@ -211,6 +217,12 @@ correctRotation, getResources().getDisplayMetrics().density, getFacing(), TextRecognizerAsyncTaskDelegate delegate = (TextRecognizerAsyncTaskDelegate) cameraView; new TextRecognizerAsyncTask(delegate, mThemedReactContext, data, width, height, correctRotation, getResources().getDisplayMetrics().density, getFacing(), getWidth(), getHeight(), mPaddingX, mPaddingY).execute(); } + + if (willCallLabelTask) { + imageLabelerTaskLock = true; + ImageLabelerAsyncTaskDelegate delegate = (ImageLabelerAsyncTaskDelegate) cameraView; + new ImageLabelerAsyncTask(delegate, mImageLabeler, data, width, height, correctRotation, getResources().getDisplayMetrics().density, getFacing(), getWidth(), getHeight(), mPaddingX, mPaddingY).execute(); + } } }); } @@ -362,7 +374,7 @@ public void setShouldScanBarCodes(boolean shouldScanBarCodes) { initBarcodeReader(); } this.mShouldScanBarCodes = shouldScanBarCodes; - setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText); + setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels); } public void onBarCodeRead(Result barCode, int width, int height, byte[] imageData) { @@ -483,7 +495,7 @@ public void setShouldDetectFaces(boolean shouldDetectFaces) { setupFaceDetector(); } this.mShouldDetectFaces = shouldDetectFaces; - setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText); + setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels); } public void onFacesDetected(WritableArray data) { @@ -520,7 +532,7 @@ public void setShouldGoogleDetectBarcodes(boolean shouldDetectBarcodes) { setupBarcodeDetector(); } this.mShouldGoogleDetectBarcodes = shouldDetectBarcodes; - setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText); + setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels); } public void setGoogleVisionBarcodeType(int barcodeType) { @@ -571,6 +583,41 @@ public void onBarcodeDetectingTaskCompleted() { googleBarcodeDetectorTaskLock = false; } + /** + * Initial setup of the image labeler + */ + private void setupImageLabeler() { + mImageLabeler = new RNImageLabeler(mThemedReactContext); + } + + public void setShouldDetectLabels(boolean shouldDetectLabels) { + if (shouldDetectLabels && mImageLabeler == null) { + setupImageLabeler(); + } + this.mShouldDetectLabels = shouldDetectLabels; + setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels); + } + + public void onLabelsDetected(WritableArray labelsDetected) { + if (!mShouldDetectLabels) { + return; + } + RNCameraViewHelper.emitLabelsDetectedEvent(this, labelsDetected); + } + + public void onImageLabelingError(RNImageLabeler imageLabeler) { + if (!mShouldDetectLabels) { + return; + } + + RNCameraViewHelper.emitImageLabelingErrorEvent(this, imageLabeler); + } + + @Override + public void onImageLabelingTaskCompleted() { + imageLabelerTaskLock = false; + } + /** * * Text recognition @@ -578,7 +625,7 @@ public void onBarcodeDetectingTaskCompleted() { public void setShouldRecognizeText(boolean shouldRecognizeText) { this.mShouldRecognizeText = shouldRecognizeText; - setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText); + setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels); } public void onTextRecognized(WritableArray serializedData) { diff --git a/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java b/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java index b5916e736..446f51c0f 100644 --- a/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java +++ b/android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java @@ -19,6 +19,7 @@ import org.reactnative.camera.events.*; import org.reactnative.barcodedetector.RNBarcodeDetector; import org.reactnative.facedetector.RNFaceDetector; +import org.reactnative.imagelabeler.RNImageLabeler; import java.text.SimpleDateFormat; import java.util.Calendar; @@ -334,6 +335,32 @@ public void run() { }); } + // Image labeling events + + public static void emitLabelsDetectedEvent(final ViewGroup view, final WritableArray data) { + + final ReactContext reactContext = (ReactContext) view.getContext(); + reactContext.runOnNativeModulesQueueThread(new Runnable() { + @Override + public void run() { + LabelsDetectedEvent event = LabelsDetectedEvent.obtain(view.getId(), data); + reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event); + } + }); + } + + public static void emitImageLabelingErrorEvent(final ViewGroup view, final RNImageLabeler imageLabeler) { + + final ReactContext reactContext = (ReactContext) view.getContext(); + reactContext.runOnNativeModulesQueueThread(new Runnable() { + @Override + public void run() { + LabelDetectionErrorEvent event = LabelDetectionErrorEvent.obtain(view.getId(), imageLabeler); + reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event); + } + }); + } + // Utilities public static int getCorrectCameraRotation(int rotation, int facing, int cameraOrientation) { diff --git a/android/src/main/java/org/reactnative/camera/events/LabelDetectionErrorEvent.java b/android/src/main/java/org/reactnative/camera/events/LabelDetectionErrorEvent.java new file mode 100644 index 000000000..162fb19fd --- /dev/null +++ b/android/src/main/java/org/reactnative/camera/events/LabelDetectionErrorEvent.java @@ -0,0 +1,54 @@ +package org.reactnative.camera.events; + +import androidx.core.util.Pools; + +import com.facebook.react.bridge.Arguments; +import com.facebook.react.bridge.WritableMap; +import com.facebook.react.uimanager.events.Event; +import com.facebook.react.uimanager.events.RCTEventEmitter; + +import org.reactnative.camera.CameraViewManager; +import org.reactnative.imagelabeler.RNImageLabeler; + +public class LabelDetectionErrorEvent extends Event { + private static final Pools.SynchronizedPool EVENTS_POOL = new Pools.SynchronizedPool<>(3); + private RNImageLabeler mImageLabeler; + + private LabelDetectionErrorEvent() { + } + + public static LabelDetectionErrorEvent obtain(int viewTag, RNImageLabeler imageLabeler) { + LabelDetectionErrorEvent event = EVENTS_POOL.acquire(); + if (event == null) { + event = new LabelDetectionErrorEvent(); + } + event.init(viewTag, imageLabeler); + return event; + } + + private void init(int viewTag, RNImageLabeler imageLabeler) { + super.init(viewTag); + mImageLabeler = imageLabeler; + } + + @Override + public short getCoalescingKey() { + return 0; + } + + @Override + public String getEventName() { + return CameraViewManager.Events.EVENT_ON_LABEL_DETECTION_ERROR.toString(); + } + + @Override + public void dispatch(RCTEventEmitter rctEventEmitter) { + rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData()); + } + + private WritableMap serializeEventData() { + WritableMap map = Arguments.createMap(); + map.putBoolean("isOperational", mImageLabeler != null && mImageLabeler.isOperational()); + return map; + } +} diff --git a/android/src/main/java/org/reactnative/camera/events/LabelsDetectedEvent.java b/android/src/main/java/org/reactnative/camera/events/LabelsDetectedEvent.java new file mode 100644 index 000000000..5512ae2b1 --- /dev/null +++ b/android/src/main/java/org/reactnative/camera/events/LabelsDetectedEvent.java @@ -0,0 +1,61 @@ +package org.reactnative.camera.events; + +import androidx.core.util.Pools; + +import com.facebook.react.bridge.Arguments; +import com.facebook.react.bridge.WritableArray; +import com.facebook.react.bridge.WritableMap; +import com.facebook.react.uimanager.events.Event; +import com.facebook.react.uimanager.events.RCTEventEmitter; + +import org.reactnative.camera.CameraViewManager; + +public class LabelsDetectedEvent extends Event { + private static final Pools.SynchronizedPool EVENTS_POOL = + new Pools.SynchronizedPool<>(3); + + private WritableArray mData; + + private LabelsDetectedEvent() {} + + public static LabelsDetectedEvent obtain(int viewTag, WritableArray data) { + LabelsDetectedEvent event = EVENTS_POOL.acquire(); + if (event == null) { + event = new LabelsDetectedEvent(); + } + event.init(viewTag, data); + return event; + } + + private void init(int viewTag, WritableArray data) { + super.init(viewTag); + mData = data; + } + + @Override + public short getCoalescingKey() { + if (mData.size() > Short.MAX_VALUE) { + return Short.MAX_VALUE; + } + + return (short) mData.size(); + } + + @Override + public String getEventName() { + return CameraViewManager.Events.EVENT_ON_LABELS_DETECTED.toString(); + } + + @Override + public void dispatch(RCTEventEmitter rctEventEmitter) { + rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData()); + } + + private WritableMap serializeEventData() { + WritableMap event = Arguments.createMap(); + event.putString("type", "label"); + event.putArray("labels", mData); + event.putInt("target", getViewTag()); + return event; + } +} diff --git a/android/src/main/java/org/reactnative/camera/tasks/ImageLabelerAsyncTaskDelegate.java b/android/src/main/java/org/reactnative/camera/tasks/ImageLabelerAsyncTaskDelegate.java new file mode 100644 index 000000000..f7a59a967 --- /dev/null +++ b/android/src/main/java/org/reactnative/camera/tasks/ImageLabelerAsyncTaskDelegate.java @@ -0,0 +1,13 @@ +package org.reactnative.camera.tasks; + +import com.facebook.react.bridge.WritableArray; +import org.reactnative.imagelabeler.RNImageLabeler; + +public interface ImageLabelerAsyncTaskDelegate { + + void onLabelsDetected(WritableArray labels); + + void onImageLabelingError(RNImageLabeler imageLabeler); + + void onImageLabelingTaskCompleted(); +} diff --git a/android/src/mlkit/java/org/reactnative/camera/tasks/ImageLabelerAsyncTask.java b/android/src/mlkit/java/org/reactnative/camera/tasks/ImageLabelerAsyncTask.java new file mode 100644 index 000000000..96c0d394d --- /dev/null +++ b/android/src/mlkit/java/org/reactnative/camera/tasks/ImageLabelerAsyncTask.java @@ -0,0 +1,111 @@ +package org.reactnative.camera.tasks; + +//import android.graphics.Point; +import android.graphics.Rect; +import android.util.Log; + +import com.facebook.react.bridge.Arguments; +import com.facebook.react.bridge.WritableArray; +import com.facebook.react.bridge.WritableMap; +import com.google.android.gms.tasks.OnFailureListener; +import com.google.android.gms.tasks.OnSuccessListener; +import com.google.mlkit.vision.label.ImageLabel; +import com.google.mlkit.vision.label.ImageLabeler; +import com.google.mlkit.vision.common.InputImage; + +import org.reactnative.imagelabeler.RNImageLabeler; +import org.reactnative.camera.utils.ImageDimensions; + +import java.util.List; + +public class ImageLabelerAsyncTask extends android.os.AsyncTask { + + private byte[] mImageData; + private int mWidth; + private int mHeight; + private int mRotation; + private RNImageLabeler mImageLabeler; + private ImageLabelerAsyncTaskDelegate mDelegate; + private double mScaleX; + private double mScaleY; + private ImageDimensions mImageDimensions; + private int mPaddingLeft; + private int mPaddingTop; + private String TAG = "RNCamera"; + + public ImageLabelerAsyncTask( + ImageLabelerAsyncTaskDelegate delegate, + RNImageLabeler imageLabeler, + byte[] imageData, + int width, + int height, + int rotation, + float density, + int facing, + int viewWidth, + int viewHeight, + int viewPaddingLeft, + int viewPaddingTop + ) { + mImageData = imageData; + mWidth = width; + mHeight = height; + mRotation = rotation; + mDelegate = delegate; + mImageLabeler = imageLabeler; + mImageDimensions = new ImageDimensions(width, height, rotation, facing); + mScaleX = (double) (viewWidth) / (mImageDimensions.getWidth() * density); + mScaleY = 1 / density; + mPaddingLeft = viewPaddingLeft; + mPaddingTop = viewPaddingTop; + } + + @Override + protected Void doInBackground(Void... ignored) { + if (isCancelled() || mDelegate == null || mImageLabeler == null) { + return null; + } + + InputImage image = InputImage.fromByteArray(mImageData, + mWidth, + mHeight, + mRotation, + InputImage.IMAGE_FORMAT_YV12 + ); + + ImageLabeler labeler = mImageLabeler.getDetector(); + labeler.process(image) + .addOnSuccessListener(new OnSuccessListener>() { + @Override + public void onSuccess(List labels) { + WritableArray serializedLabels = serializeEventData(labels); + mDelegate.onLabelsDetected(serializedLabels); + mDelegate.onImageLabelingTaskCompleted(); + } + }) + .addOnFailureListener(new OnFailureListener() { + @Override + public void onFailure(Exception e) { + Log.e(TAG, "Text recognition task failed" + e); + mDelegate.onImageLabelingTaskCompleted(); + } + }); + return null; + } + + private WritableArray serializeEventData(List labels) { + WritableArray labelsList = Arguments.createArray(); + + for (ImageLabel label: labels) { + String text = label.getText(); + double confidence = (double) label.getConfidence(); + + WritableMap serializedImageLabel = Arguments.createMap(); + serializedImageLabel.putString("text", text); + serializedImageLabel.putDouble("confidence", confidence); + labelsList.pushMap(serializedImageLabel); + } + + return labelsList; + } +} diff --git a/android/src/mlkit/java/org/reactnative/imagelabeler/RNImageLabeler.java b/android/src/mlkit/java/org/reactnative/imagelabeler/RNImageLabeler.java new file mode 100644 index 000000000..b6c329071 --- /dev/null +++ b/android/src/mlkit/java/org/reactnative/imagelabeler/RNImageLabeler.java @@ -0,0 +1,48 @@ +package org.reactnative.imagelabeler; + +import android.content.Context; +import android.util.Log; + +import com.google.mlkit.vision.label.ImageLabeler; +import com.google.mlkit.vision.label.ImageLabeling; +import com.google.mlkit.vision.label.defaults.ImageLabelerOptions; + + +public class RNImageLabeler { + + private ImageLabeler mImageLabeler = null; + private ImageLabelerOptions.Builder mBuilder; + + public RNImageLabeler(Context context) { + mBuilder = new ImageLabelerOptions.Builder(); + } + + public boolean isOperational() { + // Legacy api from GMV + return true; + } + + public ImageLabeler getDetector() { + + if (mImageLabeler == null) { + createImageLabeler(); + } + return mImageLabeler; + } + + public void release() { + if (mImageLabeler != null) { + try { + mImageLabeler.close(); + } catch (Exception e) { + Log.e("RNCamera", "Attempt to close ImageLabeler failed"); + } + mImageLabeler = null; + } + } + + private void createImageLabeler() { + ImageLabelerOptions options = mBuilder.build(); + mImageLabeler = ImageLabeling.getClient(options); + } +} diff --git a/ios/RN/LabelDetectorManagerMlkit.h b/ios/RN/LabelDetectorManagerMlkit.h new file mode 100644 index 000000000..c2d7030de --- /dev/null +++ b/ios/RN/LabelDetectorManagerMlkit.h @@ -0,0 +1,16 @@ + +#import +#if __has_include() + #import + #import +#endif + +@interface LabelDetectorManagerMlkit : NSObject +typedef void(^postRecognitionBlock)(NSArray *labels); + +- (instancetype)init; + +-(BOOL)isRealDetector; +-(void)findLabelsInFrame:(UIImage *)image scaleX:(float)scaleX scaleY:(float)scaleY completed:(postRecognitionBlock)completed; + +@end diff --git a/ios/RN/LabelDetectorManagerMlkit.m b/ios/RN/LabelDetectorManagerMlkit.m new file mode 100644 index 000000000..46557f8bc --- /dev/null +++ b/ios/RN/LabelDetectorManagerMlkit.m @@ -0,0 +1,90 @@ +#import "LabelDetectorManagerMlkit.h" +#import +#if __has_include() + +@interface LabelDetectorManagerMlkit () +@property(nonatomic, strong) MLKImageLabeler *imageLabeler; +@property(nonatomic, strong) MLKImageLabelerOptions *options; +@property(nonatomic, assign) float scaleX; +@property(nonatomic, assign) float scaleY; +@end + +@implementation LabelDetectorManagerMlkit + +- (instancetype)init +{ + if (self = [super init]) { + self.options = [[MLKImageLabelerOptions alloc] init]; + + self.imageLabeler = [MLKImageLabeler imageLabelerWithOptions:_options]; + } + return self; +} + +- (BOOL)isRealDetector +{ + return true; +} + +- (void)findLabelsInFrame:(UIImage *)uiImage + scaleX:(float)scaleX + scaleY:(float)scaleY + completed:(void (^)(NSArray *result))completed +{ + self.scaleX = scaleX; + self.scaleY = scaleY; + MLKVisionImage *image = [[MLKVisionImage alloc] initWithImage:uiImage]; + NSMutableArray *emptyResult = [[NSMutableArray alloc] init]; + [_imageLabeler + processImage:image + completion:^(NSArray *labels, NSError *error) { + if (error != nil || labels == nil) { + completed(emptyResult); + } else { + completed([self processLabels:labels]); + } + }]; +} + +- (NSArray *)processLabels:(NSArray *)labels +{ + NSMutableArray *result = [[NSMutableArray alloc] init]; + for (MLKImageLabel *label in labels) { + NSMutableDictionary *resultDict = + [[NSMutableDictionary alloc] initWithCapacity:2]; + [resultDict setObject:label.text forKey:@"text"]; + [resultDict setObject:@(label.confidence) forKey:@"confidence"]; + [result addObject:resultDict]; + } + return result; +} + +@end +#else + +@interface LabelDetectorManagerMlkit () +@end + +@implementation LabelDetectorManagerMlkit + +- (instancetype)init { + self = [super init]; + return self; +} + +- (BOOL)isRealDetector { + return false; +} + +- (NSArray *)findLabelsInFrame:(UIImage *)image + scaleX:(float)scaleX + scaleY:(float)scaleY + completed:(void (^)(NSArray *result))completed; +{ + NSLog(@"LabelDetector not installed, stub used!"); + NSArray *features = @[ @"Error, Label Detector not installed" ]; + return features; +} + +@end +#endif diff --git a/ios/RN/RNCamera.h b/ios/RN/RNCamera.h index 9a4034871..987f6dcc7 100644 --- a/ios/RN/RNCamera.h +++ b/ios/RN/RNCamera.h @@ -5,6 +5,7 @@ #import "FaceDetectorManagerMlkit.h" #import "BarcodeDetectorManagerMlkit.h" +#import "LabelDetectorManagerMlkit.h" #import "TextDetectorManager.h" @class RNCamera; @@ -48,6 +49,7 @@ @property(nonatomic, assign) BOOL canReadText; @property(nonatomic, assign) BOOL canDetectFaces; @property(nonatomic, assign) BOOL canDetectBarcodes; +@property(nonatomic, assign) BOOL canDetectLabels; @property(nonatomic, assign) BOOL captureAudio; @property(nonatomic, assign) BOOL keepAudioSession; @property(nonatomic, assign) BOOL useNativeZoom; @@ -101,10 +103,12 @@ - (void)setupOrDisableTextDetector; - (void)setupOrDisableFaceDetector; - (void)setupOrDisableBarcodeDetector; +- (void)setupOrDisableLabelDetector; - (void)onReady:(NSDictionary *)event; - (void)onMountingError:(NSDictionary *)event; - (void)onCodeRead:(NSDictionary *)event; - (void)onFacesDetected:(NSDictionary *)event; +- (void)onLabelsDetected:(NSDictionary *)event; - (void)onPictureTaken:(NSDictionary *)event; - (void)onPictureSaved:(NSDictionary *)event; - (void)onRecordingStart:(NSDictionary *)event; diff --git a/ios/RN/RNCamera.m b/ios/RN/RNCamera.m index a450a1e0d..5a8163f77 100644 --- a/ios/RN/RNCamera.m +++ b/ios/RN/RNCamera.m @@ -20,6 +20,7 @@ @interface RNCamera () @property (nonatomic, strong) RCTPromiseRejectBlock videoRecordedReject; @property (nonatomic, strong) id textDetector; @property (nonatomic, strong) id faceDetector; +@property (nonatomic, strong) id labelDetector; @property (nonatomic, strong) id barcodeDetector; @property (nonatomic, copy) RCTDirectEventBlock onCameraReady; @@ -30,6 +31,7 @@ @interface RNCamera () @property (nonatomic, copy) RCTDirectEventBlock onTouch; @property (nonatomic, copy) RCTDirectEventBlock onTextRecognized; @property (nonatomic, copy) RCTDirectEventBlock onFacesDetected; +@property (nonatomic, copy) RCTDirectEventBlock onLabelsDetected; @property (nonatomic, copy) RCTDirectEventBlock onGoogleVisionBarcodesDetected; @property (nonatomic, copy) RCTDirectEventBlock onPictureTaken; @property (nonatomic, copy) RCTDirectEventBlock onPictureSaved; @@ -37,8 +39,10 @@ @interface RNCamera () @property (nonatomic, copy) RCTDirectEventBlock onRecordingEnd; @property (nonatomic, assign) BOOL finishedReadingText; @property (nonatomic, assign) BOOL finishedDetectingFace; +@property (nonatomic, assign) BOOL finishedDetectingLabel; @property (nonatomic, copy) NSDate *startText; @property (nonatomic, copy) NSDate *startFace; +@property (nonatomic, copy) NSDate *startLabel; @property (nonatomic, copy) RCTDirectEventBlock onSubjectAreaChanged; @property (nonatomic, assign) BOOL isFocusedOnPoint; @@ -65,10 +69,13 @@ - (id)initWithBridge:(RCTBridge *)bridge self.textDetector = [self createTextDetector]; self.faceDetector = [self createFaceDetectorMlKit]; self.barcodeDetector = [self createBarcodeDetectorMlKit]; + self.labelDetector = [self createLabelDetectorMlKit]; self.finishedReadingText = true; self.finishedDetectingFace = true; + self.finishedDetectingLabel = true; self.startText = [NSDate date]; self.startFace = [NSDate date]; + self.startLabel = [NSDate date]; #if !(TARGET_IPHONE_SIMULATOR) self.previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:self.session]; @@ -1016,6 +1023,9 @@ - (void)record:(NSDictionary *)options resolve:(RCTPromiseResolveBlock)resolve r if ([self.barcodeDetector isRealDetector]) { [self stopBarcodeDetection]; } + if ([self.labelDetector isRealDetector]) { + [self stopLabelDetection]; + } [self setupMovieFileCapture]; } @@ -1289,7 +1299,7 @@ - (void)startSession // (see comment in -record), we go ahead and add the AVCaptureMovieFileOutput // to avoid an exposure rack on some devices that can cause the first few // frames of the recorded output to be underexposed. - if (![self.faceDetector isRealDetector] && ![self.textDetector isRealDetector] && ![self.barcodeDetector isRealDetector]) { + if (![self.faceDetector isRealDetector] && ![self.textDetector isRealDetector] && ![self.barcodeDetector isRealDetector] && ![self.labelDetector isRealDetector]) { [self setupMovieFileCapture]; } [self setupOrDisableBarcodeScanner]; @@ -1315,6 +1325,9 @@ - (void)stopSession if ([self.barcodeDetector isRealDetector]) { [self stopBarcodeDetection]; } + if ([self.labelDetector isRealDetector]) { + [self stopLabelDetection]; + } [self.previewLayer removeFromSuperlayer]; [self.session commitConfiguration]; [self.session stopRunning]; @@ -1906,7 +1919,7 @@ - (void)cleanupCamera { self.orientation = nil; self.isRecordingInterrupted = NO; - if ([self.textDetector isRealDetector] || [self.faceDetector isRealDetector]) { + if ([self.textDetector isRealDetector] || [self.faceDetector isRealDetector] || [self.labelDetector isRealDetector]) { [self cleanupMovieFileCapture]; } @@ -1922,6 +1935,10 @@ - (void)cleanupCamera { [self setupOrDisableBarcodeDetector]; } + if ([self.labelDetector isRealDetector]) { + [self setupOrDisableLabelDetector]; + } + // reset preset to current default AVCaptureSessionPreset preset = [self getDefaultPreset]; if (self.session.sessionPreset != preset) { @@ -2152,6 +2169,61 @@ - (void)stopTextRecognition self.videoDataOutput = nil; } +# pragma mark - LabelDetectorMlkit + +-(id)createLabelDetectorMlKit +{ + Class labelDetectorManagerClassMlkit = NSClassFromString(@"LabelDetectorManagerMlkit"); + return [[labelDetectorManagerClassMlkit alloc] init]; +} + +- (void)setupOrDisableLabelDetector +{ + if (self.canDetectLabels && [self.labelDetector isRealDetector]){ + AVCaptureSessionPreset preset = [self getDefaultPresetVideo]; + + self.session.sessionPreset = preset; + if (!self.videoDataOutput) { + self.videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; + if (![self.session canAddOutput:_videoDataOutput]) { + NSLog(@"Failed to setup video data output"); + [self stopLabelDetection]; + return; + } + + NSDictionary *rgbOutputSettings = [NSDictionary + dictionaryWithObject:[NSNumber numberWithInt:kCMPixelFormat_32BGRA] + forKey:(id)kCVPixelBufferPixelFormatTypeKey]; + [self.videoDataOutput setVideoSettings:rgbOutputSettings]; + [self.videoDataOutput setAlwaysDiscardsLateVideoFrames:YES]; + [self.videoDataOutput setSampleBufferDelegate:self queue:self.sessionQueue]; + [self.session addOutput:_videoDataOutput]; + } + } else { + [self stopLabelDetection]; + } +} + +- (void)stopLabelDetection +{ + if (self.videoDataOutput && !self.canReadText) { + [self.session removeOutput:self.videoDataOutput]; + } + self.videoDataOutput = nil; + AVCaptureSessionPreset preset = [self getDefaultPreset]; + if (self.session.sessionPreset != preset) { + [self updateSessionPreset: preset]; + } +} + +- (void)onLabelsDetected:(NSDictionary *)event +{ + if (_onLabelsDetected && _session) { + _onLabelsDetected(event); + } +} + + # pragma mark - mlkit - (void)captureOutput:(AVCaptureOutput *)captureOutput @@ -2170,10 +2242,12 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput NSDate *methodFinish = [NSDate date]; NSTimeInterval timePassedSinceSubmittingForText = [methodFinish timeIntervalSinceDate:self.startText]; NSTimeInterval timePassedSinceSubmittingForFace = [methodFinish timeIntervalSinceDate:self.startFace]; + NSTimeInterval timePassedSinceSubmittingForLabel = [methodFinish timeIntervalSinceDate:self.startLabel]; BOOL canSubmitForTextDetection = timePassedSinceSubmittingForText > 0.5 && _finishedReadingText && self.canReadText && [self.textDetector isRealDetector]; BOOL canSubmitForFaceDetection = timePassedSinceSubmittingForFace > 0.5 && _finishedDetectingFace && self.canDetectFaces && [self.faceDetector isRealDetector]; BOOL canSubmitForBarcodeDetection = self.canDetectBarcodes && [self.barcodeDetector isRealDetector]; - if (canSubmitForFaceDetection || canSubmitForTextDetection || canSubmitForBarcodeDetection) { + BOOL canSubmitForLabelDetection = timePassedSinceSubmittingForLabel > 0.5 && _finishedDetectingLabel && self.canDetectLabels && [self.labelDetector isRealDetector]; + if (canSubmitForFaceDetection || canSubmitForTextDetection || canSubmitForBarcodeDetection || canSubmitForLabelDetection) { CGSize previewSize = CGSizeMake(_previewLayer.frame.size.width, _previewLayer.frame.size.height); NSInteger position = self.videoCaptureDeviceInput.device.position; UIImage *image = [RNCameraUtils convertBufferToUIImage:sampleBuffer previewSize:previewSize position:position]; @@ -2228,6 +2302,16 @@ - (void)captureOutput:(AVCaptureOutput *)captureOutput [self onBarcodesDetected:eventBarcode]; }]; } + // find labels + if (canSubmitForLabelDetection) { + _finishedDetectingLabel = false; + self.startLabel = [NSDate date]; + [self.labelDetector findLabelsInFrame:image scaleX:scaleX scaleY:scaleY completed:^(NSArray * labels) { + NSDictionary *eventLabel = @{@"type" : @"label", @"labels" : labels}; + [self onLabelsDetected:eventLabel]; + self.finishedDetectingLabel = true; + }]; + } } } diff --git a/ios/RN/RNCameraManager.m b/ios/RN/RNCameraManager.m index 6a1f990b8..abf250f51 100644 --- a/ios/RN/RNCameraManager.m +++ b/ios/RN/RNCameraManager.m @@ -19,6 +19,7 @@ @implementation RNCameraManager RCT_EXPORT_VIEW_PROPERTY(onMountError, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onBarCodeRead, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onFacesDetected, RCTDirectEventBlock); +RCT_EXPORT_VIEW_PROPERTY(onLabelsDetected, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onGoogleVisionBarcodesDetected, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onPictureTaken, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onPictureSaved, RCTDirectEventBlock); @@ -93,7 +94,7 @@ - (NSDictionary *)constantsToExport - (NSArray *)supportedEvents { - return @[@"onCameraReady", @"onAudioInterrupted", @"onAudioConnected", @"onMountError", @"onBarCodeRead", @"onFacesDetected", @"onPictureTaken", @"onPictureSaved", @"onRecordingStart", @"onRecordingEnd", @"onTextRecognized", @"onGoogleVisionBarcodesDetected", @"onSubjectAreaChanged",@"onTouch"]; + return @[@"onCameraReady", @"onAudioInterrupted", @"onAudioConnected", @"onMountError", @"onBarCodeRead", @"onFacesDetected", @"onLabelsDetected", @"onPictureTaken", @"onPictureSaved", @"onRecordingStart", @"onRecordingEnd", @"onTextRecognized", @"onGoogleVisionBarcodesDetected", @"onSubjectAreaChanged",@"onTouch"]; } + (NSDictionary *)validCodecTypes @@ -333,6 +334,12 @@ + (NSDictionary *)barcodeDetectorConstants [view setupOrDisableTextDetector]; } +RCT_CUSTOM_VIEW_PROPERTY(labelDetectorEnabled, BOOL, RNCamera) +{ + view.canDetectLabels = [RCTConvert BOOL:json]; + [view setupOrDisableLabelDetector]; +} + RCT_CUSTOM_VIEW_PROPERTY(captureAudio, BOOL, RNCamera) { [view setCaptureAudio:[RCTConvert BOOL:json]]; diff --git a/react-native-camera.podspec b/react-native-camera.podspec index c9ba1d4b8..b46432ece 100644 --- a/react-native-camera.podspec +++ b/react-native-camera.podspec @@ -42,6 +42,12 @@ Pod::Spec.new do |s| ss.dependency 'GoogleMLKit/BarcodeScanning' end + s.subspec "LabelDetectorMLKit" do |ss| + ss.dependency 'react-native-camera/RN' + ss.dependency 'react-native-camera/RCT' + ss.dependency 'GoogleMLKit/ImageLabeling' + end + s.default_subspecs = "RN", "RCT" s.preserve_paths = 'LICENSE', 'README.md', 'package.json', 'index.js' diff --git a/src/RNCamera.js b/src/RNCamera.js index 0d9b51642..74c89d0f1 100644 --- a/src/RNCamera.js +++ b/src/RNCamera.js @@ -222,6 +222,11 @@ type Phone = { phoneType?: 'UNKNOWN' | 'Work' | 'Home' | 'Fax' | 'Mobile', }; +type ImageLabel = { + text: string, + confidence: number, +}; + type RecordingOptions = { maxDuration?: number, maxFileSize?: number, @@ -272,13 +277,23 @@ type PropsType = typeof View.props & { barCodeTypes?: Array, googleVisionBarcodeType?: number, googleVisionBarcodeMode?: number, - whiteBalance?: number | string | {temperature: number, tint: number, redGainOffset?: number, greenGainOffset?: number, blueGainOffset?: number }, + whiteBalance?: + | number + | string + | { + temperature: number, + tint: number, + redGainOffset?: number, + greenGainOffset?: number, + blueGainOffset?: number, + }, faceDetectionLandmarks?: number, autoFocus?: string | boolean | number, autoFocusPointOfInterest?: { x: number, y: number }, faceDetectionClassifications?: number, onFacesDetected?: ({ faces: Array }) => void, onTextRecognized?: ({ textBlocks: Array }) => void, + onLabelsDetected?: ({ labels: Array }) => void, captureAudio?: boolean, keepAudioSession?: boolean, useCamera2Api?: boolean, @@ -413,6 +428,7 @@ export default class Camera extends React.Component { onGoogleVisionBarcodesDetected: PropTypes.func, onFacesDetected: PropTypes.func, onTextRecognized: PropTypes.func, + onLabelsDetected: PropTypes.func, onSubjectAreaChanged: PropTypes.func, trackingEnabled: PropTypes.bool, faceDetectionMode: PropTypes.number, @@ -425,11 +441,17 @@ export default class Camera extends React.Component { cameraId: PropTypes.string, flashMode: PropTypes.oneOfType([PropTypes.string, PropTypes.number]), exposure: PropTypes.number, - whiteBalance: PropTypes.oneOfType([PropTypes.string, PropTypes.number, - PropTypes.shape({ temperature: PropTypes.number, tint: PropTypes.number, + whiteBalance: PropTypes.oneOfType([ + PropTypes.string, + PropTypes.number, + PropTypes.shape({ + temperature: PropTypes.number, + tint: PropTypes.number, redGainOffset: PropTypes.number, greenGainOffset: PropTypes.number, - blueGainOffset: PropTypes.number })]), + blueGainOffset: PropTypes.number, + }), + ]), autoFocus: PropTypes.oneOfType([PropTypes.string, PropTypes.number, PropTypes.bool]), autoFocusPointOfInterest: PropTypes.shape({ x: PropTypes.number, y: PropTypes.number }), permissionDialogTitle: PropTypes.string, @@ -845,6 +867,7 @@ export default class Camera extends React.Component { onBarCodeRead={this._onObjectDetected(this.props.onBarCodeRead)} onTouch={this._onTouch} onFacesDetected={this._onObjectDetected(this.props.onFacesDetected)} + onLabelsDetected={this._onObjectDetected(this.props.onLabelsDetected)} onTextRecognized={this._onObjectDetected(this.props.onTextRecognized)} onPictureSaved={this._onPictureSaved} onSubjectAreaChanged={this._onSubjectAreaChanged} @@ -874,6 +897,10 @@ export default class Camera extends React.Component { newProps.faceDetectorEnabled = true; } + if (props.onLabelsDetected) { + newProps.labelDetectorEnabled = true; + } + if (props.onTap || props.onDoubleTap) { newProps.touchDetectorEnabled = true; } @@ -910,6 +937,7 @@ const RNCamera = requireNativeComponent('RNCamera', Camera, { googleVisionBarcodeDetectorEnabled: true, faceDetectorEnabled: true, textRecognizerEnabled: true, + labelDetectorEnabled: true, importantForAccessibility: true, onBarCodeRead: true, onGoogleVisionBarcodesDetected: true, @@ -918,6 +946,7 @@ const RNCamera = requireNativeComponent('RNCamera', Camera, { onAudioConnected: true, onPictureSaved: true, onFaceDetected: true, + onLabelsDetected: true, onTouch: true, onLayout: true, onMountError: true, From 8c5e427949ffd2c4cdcfb441e131aa71d5dcabf3 Mon Sep 17 00:00:00 2001 From: David Govea Date: Tue, 28 Jul 2020 19:46:29 -0700 Subject: [PATCH 6/9] chore(example): add label detection to mlkit example app --- examples/mlkit/App.js | 52 +++++++++++++++++++++++-- examples/mlkit/README.md | 2 + examples/mlkit/android/app/build.gradle | 1 + examples/mlkit/ios/Podfile | 3 +- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/examples/mlkit/App.js b/examples/mlkit/App.js index 7ea2e374d..070b78816 100644 --- a/examples/mlkit/App.js +++ b/examples/mlkit/App.js @@ -40,9 +40,11 @@ export default class CameraScreen extends React.Component { canDetectFaces: false, canDetectText: false, canDetectBarcode: false, + canDetectLabels: false, faces: [], textBlocks: [], barcodes: [], + labels: [], }; toggleFacing() { @@ -234,8 +236,24 @@ export default class CameraScreen extends React.Component { ); + labelsDetected = ({ labels = [] }) => this.setState({ labels }); + + renderLabels = () => ( + + {this.state.labels.sort((a, b) => b.confidence - a.confidence).map(this.renderLabel)} + + ); + + renderLabel = ({ text, confidence }) => ( + + + {`${text} ${confidence.toFixed(2)}`} + + + ); + renderCamera() { - const { canDetectFaces, canDetectText, canDetectBarcode } = this.state; + const { canDetectFaces, canDetectText, canDetectBarcode, canDetectLabels } = this.state; return ( { @@ -271,6 +289,7 @@ export default class CameraScreen extends React.Component { onFacesDetected={canDetectFaces ? this.facesDetected : null} onTextRecognized={canDetectText ? this.textRecognized : null} onGoogleVisionBarcodesDetected={canDetectBarcode ? this.barcodeRecognized : null} + onLabelsDetected={canDetectLabels ? this.labelsDetected : null} googleVisionBarcodeType={RNCamera.Constants.GoogleVisionBarcodeDetection.BarcodeType.ALL} googleVisionBarcodeMode={ RNCamera.Constants.GoogleVisionBarcodeDetection.BarcodeMode.ALTERNATE @@ -306,20 +325,25 @@ export default class CameraScreen extends React.Component { }} > - + {!canDetectFaces ? 'Detect Faces' : 'Detecting Faces'} - + {!canDetectText ? 'Detect Text' : 'Detecting Text'} - + {!canDetectBarcode ? 'Detect Barcode' : 'Detecting Barcode'} + + + {!canDetectLabels ? 'Detect Labels' : 'Detecting Labels'} + + ); } @@ -435,6 +460,16 @@ const styles = StyleSheet.create({ color: 'white', fontSize: 15, }, + detectText: { + color: 'white', + fontSize: 12, + textAlign: 'center', + }, + labelText: { + color: 'white', + fontWeight: 'bold', + fontSize: 15, + }, zoomText: { position: 'absolute', bottom: 70, @@ -451,6 +486,15 @@ const styles = StyleSheet.create({ left: 0, top: 0, }, + labelsContainer: { + position: 'absolute', + bottom: 0, + right: 0, + left: 0, + top: 0, + paddingTop: 150, + alignItems: 'center', + }, face: { padding: 10, borderWidth: 2, diff --git a/examples/mlkit/README.md b/examples/mlkit/README.md index 47f417507..2768c88d4 100644 --- a/examples/mlkit/README.md +++ b/examples/mlkit/README.md @@ -10,6 +10,8 @@ Face Recognition: draws polygons around faces and red circles on top of face lan Text Recognition: draws polygons around text blocks and recognized within them. +Label Detection: lists detected labels with confidence levels. + ### Setup 1. Run `yarn install`. diff --git a/examples/mlkit/android/app/build.gradle b/examples/mlkit/android/app/build.gradle index 234bd0d4c..dca61e2b8 100644 --- a/examples/mlkit/android/app/build.gradle +++ b/examples/mlkit/android/app/build.gradle @@ -102,6 +102,7 @@ android { minSdkVersion rootProject.ext.minSdkVersion targetSdkVersion rootProject.ext.targetSdkVersion missingDimensionStrategy 'react-native-camera', 'mlkit' + multiDexEnabled true versionCode 1 versionName "1.0" } diff --git a/examples/mlkit/ios/Podfile b/examples/mlkit/ios/Podfile index 7d4e96479..6f38e849b 100644 --- a/examples/mlkit/ios/Podfile +++ b/examples/mlkit/ios/Podfile @@ -27,7 +27,8 @@ target 'mlkit' do pod 'react-native-camera', path: '../../../', subspecs: [ 'TextDetector', 'FaceDetectorMLKit', - 'BarcodeDetectorMLKit' + 'BarcodeDetectorMLKit', + 'LabelDetectorMLKit', ] pod 'Firebase/Core' From 24abc82f033a1a98ebfaea104fdf2e949160b767 Mon Sep 17 00:00:00 2001 From: David Govea Date: Tue, 21 Jul 2020 01:32:59 -0700 Subject: [PATCH 7/9] chore(example): remove firebase from example mlkit apps --- examples/mlkit/android/app/build.gradle | 2 +- examples/mlkit/ios/Podfile | 2 -- examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj | 2 -- examples/mlkit/ios/mlkit/AppDelegate.m | 3 --- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/examples/mlkit/android/app/build.gradle b/examples/mlkit/android/app/build.gradle index dca61e2b8..b9f70f0e4 100644 --- a/examples/mlkit/android/app/build.gradle +++ b/examples/mlkit/android/app/build.gradle @@ -166,4 +166,4 @@ task downloadDependencies() { } } -apply plugin: 'com.google.gms.google-services' +// apply plugin: 'com.google.gms.google-services' diff --git a/examples/mlkit/ios/Podfile b/examples/mlkit/ios/Podfile index 6f38e849b..d94018d26 100644 --- a/examples/mlkit/ios/Podfile +++ b/examples/mlkit/ios/Podfile @@ -31,6 +31,4 @@ target 'mlkit' do 'LabelDetectorMLKit', ] - pod 'Firebase/Core' - end diff --git a/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj b/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj index 4814bda10..db9a9d672 100644 --- a/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj +++ b/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj @@ -23,7 +23,6 @@ 13B07FC11A68108700A75B9A /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; }; 140ED2AC1D01E1AD002B40FF /* libReact.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 146834041AC3E56700842450 /* libReact.a */; }; 146834051AC3E58100842450 /* libReact.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 146834041AC3E56700842450 /* libReact.a */; }; - 2622130A222217F8009EBD58 /* GoogleService-Info.plist in Resources */ = {isa = PBXBuildFile; fileRef = 262212E4222217F8009EBD58 /* GoogleService-Info.plist */; }; 2D02E4BC1E0B4A80006451C7 /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB01A68108700A75B9A /* AppDelegate.m */; }; 2D02E4BD1E0B4A84006451C7 /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; }; 2D02E4BF1E0B4AB3006451C7 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; }; @@ -1078,7 +1077,6 @@ files = ( 13B07FBF1A68108700A75B9A /* Images.xcassets in Resources */, 13B07FBD1A68108700A75B9A /* LaunchScreen.xib in Resources */, - 2622130A222217F8009EBD58 /* GoogleService-Info.plist in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/examples/mlkit/ios/mlkit/AppDelegate.m b/examples/mlkit/ios/mlkit/AppDelegate.m index 443e2c256..9fbe295ba 100644 --- a/examples/mlkit/ios/mlkit/AppDelegate.m +++ b/examples/mlkit/ios/mlkit/AppDelegate.m @@ -7,7 +7,6 @@ #import "AppDelegate.h" -#import #import #import @@ -17,8 +16,6 @@ - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:( { NSURL *jsCodeLocation; - [FIRApp configure]; - jsCodeLocation = [[RCTBundleURLProvider sharedSettings] jsBundleURLForBundleRoot:@"index" fallbackResource:nil]; RCTRootView *rootView = [[RCTRootView alloc] initWithBundleURL:jsCodeLocation From 313828fbc1416c29dfbe3266d653683062fbefbd Mon Sep 17 00:00:00 2001 From: David Govea Date: Tue, 28 Jul 2020 17:35:29 -0700 Subject: [PATCH 8/9] chore(example): update yarnlock & pods in mlkit example --- examples/mlkit/ios/Podfile.lock | 315 +++++++++--------- .../mlkit/ios/mlkit.xcodeproj/project.pbxproj | 12 +- examples/mlkit/yarn.lock | 8 +- 3 files changed, 161 insertions(+), 174 deletions(-) diff --git a/examples/mlkit/ios/Podfile.lock b/examples/mlkit/ios/Podfile.lock index f9a129d60..cc81fb3b2 100644 --- a/examples/mlkit/ios/Podfile.lock +++ b/examples/mlkit/ios/Podfile.lock @@ -1,189 +1,170 @@ PODS: - boost-for-react-native (1.63.0) - DoubleConversion (1.1.6) - - Firebase/Core (5.16.0): - - Firebase/CoreOnly - - FirebaseAnalytics (= 5.5.0) - - Firebase/CoreOnly (5.16.0): - - FirebaseCore (= 5.2.0) - - Firebase/MLVision (5.16.0): - - Firebase/CoreOnly - - FirebaseMLVision (= 0.14.0) - - Firebase/MLVisionBarcodeModel (5.16.0): - - Firebase/CoreOnly - - FirebaseMLVisionBarcodeModel (= 0.14.0) - - Firebase/MLVisionFaceModel (5.16.0): - - Firebase/CoreOnly - - FirebaseMLVisionFaceModel (= 0.14.0) - - Firebase/MLVisionTextModel (5.16.0): - - Firebase/CoreOnly - - FirebaseMLVisionTextModel (= 0.14.0) - - FirebaseAnalytics (5.5.0): - - FirebaseCore (~> 5.2) - - FirebaseInstanceID (~> 3.4) - - GoogleAppMeasurement (= 5.5.0) - - GoogleUtilities/AppDelegateSwizzler (~> 5.2) - - GoogleUtilities/MethodSwizzler (~> 5.2) - - GoogleUtilities/Network (~> 5.2) - - "GoogleUtilities/NSData+zlib (~> 5.2)" - - nanopb (~> 0.3) - - FirebaseCore (5.2.0): - - GoogleUtilities/Logger (~> 5.2) - - FirebaseInstanceID (3.4.0): - - FirebaseCore (~> 5.2) - - GoogleUtilities/Environment (~> 5.3) - - GoogleUtilities/UserDefaults (~> 5.3) - - FirebaseMLCommon (0.14.0): - - FirebaseCore (~> 5.2) - - FirebaseInstanceID (~> 3.4) - - GoogleUtilities/UserDefaults (~> 5.3) - - GTMSessionFetcher/Core (~> 1.1) - - FirebaseMLVision (0.14.0): - - FirebaseCore (~> 5.2) - - FirebaseMLCommon (~> 0.14) - - GoogleAPIClientForREST/Core (~> 1.3) - - GoogleAPIClientForREST/Vision (~> 1.3) - - GoogleMobileVision/Detector (~> 1.4) - - FirebaseMLVisionBarcodeModel (0.14.0): - - GoogleMobileVision/BarcodeDetector (~> 1.4) - - FirebaseMLVisionFaceModel (0.14.0): - - GoogleMobileVision/FaceDetector (~> 1.4) - - FirebaseMLVisionTextModel (0.14.0): - - GoogleMobileVision/TextDetector (~> 1.4) - Folly (2018.10.22.00): - boost-for-react-native - DoubleConversion - glog - glog (0.3.5) - - GoogleAPIClientForREST/Core (1.3.8): - - GTMSessionFetcher (>= 1.1.7) - - GoogleAPIClientForREST/Vision (1.3.8): - - GoogleAPIClientForREST/Core - - GTMSessionFetcher (>= 1.1.7) - - GoogleAppMeasurement (5.5.0): - - GoogleUtilities/AppDelegateSwizzler (~> 5.2) - - GoogleUtilities/MethodSwizzler (~> 5.2) - - GoogleUtilities/Network (~> 5.2) - - "GoogleUtilities/NSData+zlib (~> 5.2)" - - nanopb (~> 0.3) - - GoogleMobileVision/BarcodeDetector (1.5.0): - - GoogleMobileVision/Detector (~> 1.5) - - GoogleMobileVision/Detector (1.5.0): - - GoogleToolboxForMac/Logger (~> 2.1) - - "GoogleToolboxForMac/NSData+zlib (~> 2.1)" - - GTMSessionFetcher/Core (~> 1.1) - - Protobuf (~> 3.1) - - GoogleMobileVision/FaceDetector (1.5.0): - - GoogleMobileVision/Detector (~> 1.5) - - GoogleMobileVision/TextDetector (1.5.0): - - GoogleMobileVision/Detector (~> 1.5) - - GoogleToolboxForMac/Defines (2.2.0) - - GoogleToolboxForMac/Logger (2.2.0): - - GoogleToolboxForMac/Defines (= 2.2.0) - - "GoogleToolboxForMac/NSData+zlib (2.2.0)": - - GoogleToolboxForMac/Defines (= 2.2.0) - - GoogleUtilities/AppDelegateSwizzler (5.3.7): + - GoogleDataTransport (7.1.0): + - nanopb (~> 1.30905.0) + - GoogleMLKit/BarcodeScanning (0.62.0): + - GoogleMLKit/MLKitCore + - MLKitBarcodeScanning (~> 0.62.0) + - GoogleMLKit/FaceDetection (0.62.0): + - GoogleMLKit/MLKitCore + - MLKitFaceDetection (~> 0.62.0) + - GoogleMLKit/ImageLabeling (0.62.0): + - GoogleMLKit/MLKitCore + - MLKitImageLabeling (~> 0.62.0) + - GoogleMLKit/MLKitCore (0.62.0): + - MLKitCommon (~> 0.62.0) + - GoogleMLKit/TextRecognition (0.62.0): + - GoogleMLKit/MLKitCore + - MLKitTextRecognition (~> 0.62.0) + - GoogleToolboxForMac/DebugUtils (2.2.2): + - GoogleToolboxForMac/Defines (= 2.2.2) + - GoogleToolboxForMac/Defines (2.2.2) + - GoogleToolboxForMac/Logger (2.2.2): + - GoogleToolboxForMac/Defines (= 2.2.2) + - "GoogleToolboxForMac/NSData+zlib (2.2.2)": + - GoogleToolboxForMac/Defines (= 2.2.2) + - "GoogleToolboxForMac/NSDictionary+URLArguments (2.2.2)": + - GoogleToolboxForMac/DebugUtils (= 2.2.2) + - GoogleToolboxForMac/Defines (= 2.2.2) + - "GoogleToolboxForMac/NSString+URLArguments (= 2.2.2)" + - "GoogleToolboxForMac/NSString+URLArguments (2.2.2)" + - GoogleUtilities/Environment (6.7.1): + - PromisesObjC (~> 1.2) + - GoogleUtilities/Logger (6.7.1): - GoogleUtilities/Environment + - GoogleUtilities/UserDefaults (6.7.1): - GoogleUtilities/Logger - - GoogleUtilities/Network - - GoogleUtilities/Environment (5.3.7) - - GoogleUtilities/Logger (5.3.7): - - GoogleUtilities/Environment - - GoogleUtilities/MethodSwizzler (5.3.7): + - GoogleUtilitiesComponents (1.0.0): - GoogleUtilities/Logger - - GoogleUtilities/Network (5.3.7): - - GoogleUtilities/Logger - - "GoogleUtilities/NSData+zlib" - - GoogleUtilities/Reachability - - "GoogleUtilities/NSData+zlib (5.3.7)" - - GoogleUtilities/Reachability (5.3.7): - - GoogleUtilities/Logger - - GoogleUtilities/UserDefaults (5.3.7): - - GoogleUtilities/Logger - - GTMSessionFetcher (1.2.1): - - GTMSessionFetcher/Full (= 1.2.1) - - GTMSessionFetcher/Core (1.2.1) - - GTMSessionFetcher/Full (1.2.1): - - GTMSessionFetcher/Core (= 1.2.1) - - nanopb (0.3.901): - - nanopb/decode (= 0.3.901) - - nanopb/encode (= 0.3.901) - - nanopb/decode (0.3.901) - - nanopb/encode (0.3.901) - - Protobuf (3.7.0) - - React (0.59.1): - - React/Core (= 0.59.1) - - react-native-camera/BarcodeDetectorMLKit (2.2.2): - - Firebase/MLVision - - Firebase/MLVisionBarcodeModel + - GTMSessionFetcher/Core (1.4.0) + - MLKitBarcodeScanning (0.62.0): + - MLKitCommon (~> 0.62) + - MLKitVision (~> 0.62) + - MLKitCommon (0.62.0): + - GoogleDataTransport (~> 7.0) + - GoogleToolboxForMac/Logger (~> 2.1) + - "GoogleToolboxForMac/NSData+zlib (~> 2.1)" + - "GoogleToolboxForMac/NSDictionary+URLArguments (~> 2.1)" + - GoogleUtilities/UserDefaults (~> 6.0) + - GoogleUtilitiesComponents (~> 1.0) + - GTMSessionFetcher/Core (~> 1.1) + - Protobuf (~> 3.12) + - MLKitFaceDetection (0.62.0): + - MLKitCommon (~> 0.62) + - MLKitVision (~> 0.62) + - MLKitImageLabeling (0.62.0): + - MLKitCommon (~> 0.62) + - MLKitImageLabelingCommon (~> 0.62) + - MLKitVision (~> 0.62) + - MLKitVisionKit (~> 0.62) + - MLKitImageLabelingCommon (0.62.0): + - MLKitCommon (~> 0.62) + - MLKitVision (~> 0.62) + - MLKitObjectDetectionCommon (0.62.0): + - MLKitCommon (~> 0.62) + - MLKitVision (~> 0.62) + - MLKitTextRecognition (0.62.0): + - MLKitCommon (~> 0.62) + - MLKitVision (~> 0.62) + - MLKitVision (0.62.0): + - GoogleToolboxForMac/Logger (~> 2.1) + - "GoogleToolboxForMac/NSData+zlib (~> 2.1)" + - GTMSessionFetcher/Core (~> 1.1) + - MLKitCommon (~> 0.62) + - Protobuf (~> 3.12) + - MLKitVisionKit (0.62.0): + - MLKitCommon (~> 0.62) + - MLKitImageLabelingCommon (~> 0.62) + - MLKitObjectDetectionCommon (~> 0.62) + - MLKitVision (~> 0.62) + - nanopb (1.30905.0): + - nanopb/decode (= 1.30905.0) + - nanopb/encode (= 1.30905.0) + - nanopb/decode (1.30905.0) + - nanopb/encode (1.30905.0) + - PromisesObjC (1.2.9) + - Protobuf (3.12.0) + - React (0.59.10): + - React/Core (= 0.59.10) + - react-native-camera/BarcodeDetectorMLKit (3.35.0): + - GoogleMLKit/BarcodeScanning + - React + - react-native-camera/RCT + - react-native-camera/RN + - react-native-camera/FaceDetectorMLKit (3.35.0): + - GoogleMLKit/FaceDetection - React - react-native-camera/RCT - react-native-camera/RN - - react-native-camera/FaceDetectorMLKit (2.2.2): - - Firebase/MLVision - - Firebase/MLVisionFaceModel + - react-native-camera/LabelDetectorMLKit (3.35.0): + - GoogleMLKit/ImageLabeling - React - react-native-camera/RCT - react-native-camera/RN - - react-native-camera/RCT (2.2.2): + - react-native-camera/RCT (3.35.0): - React - - react-native-camera/RN (2.2.2): + - react-native-camera/RN (3.35.0): - React - - react-native-camera/TextDetector (2.2.2): - - Firebase/MLVision - - Firebase/MLVisionTextModel + - react-native-camera/TextDetector (3.35.0): + - GoogleMLKit/TextRecognition - React - react-native-camera/RCT - react-native-camera/RN - - React/Core (0.59.1): - - yoga (= 0.59.1.React) - - React/CxxBridge (0.59.1): + - React/Core (0.59.10): + - yoga (= 0.59.10.React) + - React/CxxBridge (0.59.10): - Folly (= 2018.10.22.00) - React/Core - React/cxxreact - React/jsiexecutor - - React/cxxreact (0.59.1): + - React/cxxreact (0.59.10): - boost-for-react-native (= 1.63.0) - DoubleConversion - Folly (= 2018.10.22.00) - glog - React/jsinspector - - React/DevSupport (0.59.1): + - React/DevSupport (0.59.10): - React/Core - React/RCTWebSocket - - React/fishhook (0.59.1) - - React/jsi (0.59.1): + - React/fishhook (0.59.10) + - React/jsi (0.59.10): - DoubleConversion - Folly (= 2018.10.22.00) - glog - - React/jsiexecutor (0.59.1): + - React/jsiexecutor (0.59.10): - DoubleConversion - Folly (= 2018.10.22.00) - glog - React/cxxreact - React/jsi - - React/jsinspector (0.59.1) - - React/RCTAnimation (0.59.1): + - React/jsinspector (0.59.10) + - React/RCTAnimation (0.59.10): - React/Core - - React/RCTBlob (0.59.1): + - React/RCTBlob (0.59.10): - React/Core - - React/RCTNetwork (0.59.1): + - React/RCTNetwork (0.59.10): - React/Core - - React/RCTText (0.59.1): + - React/RCTText (0.59.10): - React/Core - - React/RCTWebSocket (0.59.1): + - React/RCTWebSocket (0.59.10): - React/Core - React/fishhook - React/RCTBlob - - yoga (0.59.1.React) + - yoga (0.59.10.React) DEPENDENCIES: - DoubleConversion (from `../node_modules/react-native/third-party-podspecs/DoubleConversion.podspec`) - - Firebase/Core - Folly (from `../node_modules/react-native/third-party-podspecs/Folly.podspec`) - glog (from `../node_modules/react-native/third-party-podspecs/glog.podspec`) - react-native-camera/BarcodeDetectorMLKit (from `../../../`) - react-native-camera/FaceDetectorMLKit (from `../../../`) + - react-native-camera/LabelDetectorMLKit (from `../../../`) - react-native-camera/TextDetector (from `../../../`) - React/Core (from `../node_modules/react-native`) - React/CxxBridge (from `../node_modules/react-native`) @@ -195,24 +176,25 @@ DEPENDENCIES: - yoga (from `../node_modules/react-native/ReactCommon/yoga`) SPEC REPOS: - https://github.com/cocoapods/specs.git: + trunk: - boost-for-react-native - - Firebase - - FirebaseAnalytics - - FirebaseCore - - FirebaseInstanceID - - FirebaseMLCommon - - FirebaseMLVision - - FirebaseMLVisionBarcodeModel - - FirebaseMLVisionFaceModel - - FirebaseMLVisionTextModel - - GoogleAPIClientForREST - - GoogleAppMeasurement - - GoogleMobileVision + - GoogleDataTransport + - GoogleMLKit - GoogleToolboxForMac - GoogleUtilities + - GoogleUtilitiesComponents - GTMSessionFetcher + - MLKitBarcodeScanning + - MLKitCommon + - MLKitFaceDetection + - MLKitImageLabeling + - MLKitImageLabelingCommon + - MLKitObjectDetectionCommon + - MLKitTextRecognition + - MLKitVision + - MLKitVisionKit - nanopb + - PromisesObjC - Protobuf EXTERNAL SOURCES: @@ -232,29 +214,30 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: boost-for-react-native: 39c7adb57c4e60d6c5479dd8623128eb5b3f0f2c DoubleConversion: bb338842f62ab1d708ceb63ec3d999f0f3d98ecd - Firebase: 749a8ff4962f9d8c79dda1966de20f6f77583d67 - FirebaseAnalytics: d35d47c03c50c73c14a7fd31463c5775843e78a9 - FirebaseCore: ea2d1816723ef21492b8e9113303e1350db5e08c - FirebaseInstanceID: 97ea7a5dca9afd72c79bfcdddb7a44aa1cbb42a1 - FirebaseMLCommon: d8a789e36a7faa175b1a5d1139e7fc7323c8db7b - FirebaseMLVision: 07c0da3ceaa5ecde621528a985748d6098a84388 - FirebaseMLVisionBarcodeModel: 7ae6e777c4c268505f08761afdfb64613c41204d - FirebaseMLVisionFaceModel: 8c80355e22cfcf100ad2ac9c618536d19daf266b - FirebaseMLVisionTextModel: c6b3bf6129cb97cba51f8f90e80b40a66228dfa1 Folly: de497beb10f102453a1afa9edbf8cf8a251890de glog: aefd1eb5dda2ab95ba0938556f34b98e2da3a60d - GoogleAPIClientForREST: 5447a194eae517986cafe6421a5330b80b820591 - GoogleAppMeasurement: 621f3bc6211d5ba548debe01fafad30cf5ab6859 - GoogleMobileVision: a1f93108b3527d67339e2de80e1db76645f9e8b9 - GoogleToolboxForMac: ff31605b7d66400dcec09bed5861689aebadda4d - GoogleUtilities: 111a012f4c3a29c9e7c954c082fafd6ee3c999c0 - GTMSessionFetcher: 32aeca0aa144acea523e1c8e053089dec2cb98ca - nanopb: 2901f78ea1b7b4015c860c2fdd1ea2fee1a18d48 - Protobuf: 7a877b7f3e5964e3fce995e2eb323dbc6831bb5a - React: 1d605e098d69bdf08960787f3446f0a9dc2e2ccf - react-native-camera: 9c50d7def800895e7991ccda6203929553ceec9c - yoga: 128daf064cacaede0c3bb27424b6b4c71052e6cd + GoogleDataTransport: af0c79193dc59acd37630b4833d0dc6912ae6bd5 + GoogleMLKit: b753199bd6be50b49d55ffe0c60ad19a8aec56f9 + GoogleToolboxForMac: 800648f8b3127618c1b59c7f97684427630c5ea3 + GoogleUtilities: e121a3867449ce16b0e35ddf1797ea7a389ffdf2 + GoogleUtilitiesComponents: a69c0b3b369ba443e988141e75ef49d9010b1c80 + GTMSessionFetcher: 6f5c8abbab8a9bce4bb3f057e317728ec6182b10 + MLKitBarcodeScanning: 111a08148e542f6deb5a49294c4d8b2436a0daf9 + MLKitCommon: 9890ef9e9c374f53a48e435a4c6ce208c8b16a1e + MLKitFaceDetection: 9e89118a072bee6c7c3e4319c7c95ed393ffd16d + MLKitImageLabeling: 6db831e7820daa5b756fb5bfbcb52ef814269d35 + MLKitImageLabelingCommon: 63910a359a2aaf1db0ac690cc809e9b5894d502a + MLKitObjectDetectionCommon: 7211f3ed1ffdaf31aee377979ed9bbf0f57d89e4 + MLKitTextRecognition: 8dd0aa91588d3b15bfa4d2546a077c1bd2e8544a + MLKitVision: 6f0f4e5168becf2bb2e053ddfe26da2a27cc4906 + MLKitVisionKit: 04ea6daf52feeef6778059b24ed7e95298277ee8 + nanopb: c43f40fadfe79e8b8db116583945847910cbabc9 + PromisesObjC: b48e0338dbbac2207e611750777895f7a5811b75 + Protobuf: 2793fcd0622a00b546c60e7cbbcc493e043e9bb9 + React: 36d0768f9e93be2473b37e7fa64f92c1d5341eef + react-native-camera: 90ac3ddebb968711a4a0ad4aaf5c219c0a834cbe + yoga: 684513b14b03201579ba3cee20218c9d1298b0cc -PODFILE CHECKSUM: ea5c4e2c8f2c607fe7c1cd69dc26067aef6c2ce1 +PODFILE CHECKSUM: 1104028fea44e9e9f51f181120a7cac032c04156 -COCOAPODS: 1.5.3 +COCOAPODS: 1.9.3 diff --git a/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj b/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj index db9a9d672..83bddb8db 100644 --- a/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj +++ b/examples/mlkit/ios/mlkit.xcodeproj/project.pbxproj @@ -1150,18 +1150,22 @@ files = ( ); inputPaths = ( - "${SRCROOT}/Pods/Target Support Files/Pods-mlkit/Pods-mlkit-resources.sh", - "${PODS_CONFIGURATION_BUILD_DIR}/GoogleMobileVision/GoogleMVFaceDetectorResources.bundle", - "${PODS_CONFIGURATION_BUILD_DIR}/GoogleMobileVision/GoogleMVTextDetectorResources.bundle", + "${PODS_ROOT}/Target Support Files/Pods-mlkit/Pods-mlkit-resources.sh", + "${PODS_CONFIGURATION_BUILD_DIR}/MLKitFaceDetection/GoogleMVFaceDetectorResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/MLKitImageLabeling/MLKitImageLabelingResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/MLKitObjectDetectionCommon/MLKitObjectDetectionCommonResources.bundle", + "${PODS_CONFIGURATION_BUILD_DIR}/MLKitTextRecognition/GoogleMVTextDetectorResources.bundle", ); name = "[CP] Copy Pods Resources"; outputPaths = ( "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVFaceDetectorResources.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/MLKitImageLabelingResources.bundle", + "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/MLKitObjectDetectionCommonResources.bundle", "${TARGET_BUILD_DIR}/${UNLOCALIZED_RESOURCES_FOLDER_PATH}/GoogleMVTextDetectorResources.bundle", ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "\"${SRCROOT}/Pods/Target Support Files/Pods-mlkit/Pods-mlkit-resources.sh\"\n"; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-mlkit/Pods-mlkit-resources.sh\"\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ diff --git a/examples/mlkit/yarn.lock b/examples/mlkit/yarn.lock index c75046a0e..e39aeb62a 100644 --- a/examples/mlkit/yarn.lock +++ b/examples/mlkit/yarn.lock @@ -4853,10 +4853,10 @@ react-is@^16.8.1, react-is@^16.8.3, react-is@^16.8.4: resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.12.0.tgz#2cc0fe0fba742d97fd527c42a13bec4eeb06241c" integrity sha512-rPCkf/mWBtKc97aLL9/txD8DZdemK0vkA3JMLShjlJB3Pj3s+lpf1KaBzMfQrAmhMQB0n1cU/SUGgKKBCe837Q== -react-native@0.59.1: - version "0.59.1" - resolved "https://registry.yarnpkg.com/react-native/-/react-native-0.59.1.tgz#3274dcc0f9558799bd41c8092acac2a79bd627ce" - integrity sha512-KA/MyoCQLGavPBc/Q2QRfdbxHuKHTcYTks9xwwRPzgDqTfmTuDcyoC6jUBTfhZ0LEes5GMoKVM4eVrYUbprOmw== +react-native@~0.59.9: + version "0.59.10" + resolved "https://registry.yarnpkg.com/react-native/-/react-native-0.59.10.tgz#352f381e382f93a0403be499c9e384bf51c2591c" + integrity sha512-guB9YW+pBqS1dnfZ4ntzjINopiCUAbdmshU2wMWD1W32fRczLAopi/7Q2iHKP8LTCdxuYZV3fa9Mew5PSuANAw== dependencies: "@babel/runtime" "^7.0.0" "@react-native-community/cli" "^1.2.1" From 751a4026880556648fe4aa4dd6f7ddaf82b73532 Mon Sep 17 00:00:00 2001 From: David Govea Date: Sun, 27 Dec 2020 18:01:37 -0800 Subject: [PATCH 9/9] fix(ios,mlkit): Update & correct conditional imports in iOS --- ios/RN/LabelDetectorManagerMlkit.h | 1 + ios/RN/RNCameraManager.m | 4 ++-- ios/RN/RNFaceDetectorModuleMLKit.m | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ios/RN/LabelDetectorManagerMlkit.h b/ios/RN/LabelDetectorManagerMlkit.h index c2d7030de..be2cdb82b 100644 --- a/ios/RN/LabelDetectorManagerMlkit.h +++ b/ios/RN/LabelDetectorManagerMlkit.h @@ -3,6 +3,7 @@ #if __has_include() #import #import + #import #endif @interface LabelDetectorManagerMlkit : NSObject diff --git a/ios/RN/RNCameraManager.m b/ios/RN/RNCameraManager.m index abf250f51..f6f9d99f2 100644 --- a/ios/RN/RNCameraManager.m +++ b/ios/RN/RNCameraManager.m @@ -162,7 +162,7 @@ + (NSDictionary *)pictureSizes + (NSDictionary *)faceDetectorConstants { -#if __has_include() +#if __has_include() return [FaceDetectorManagerMlkit constants]; #else return [NSDictionary new]; @@ -171,7 +171,7 @@ + (NSDictionary *)faceDetectorConstants + (NSDictionary *)barcodeDetectorConstants { -#if __has_include() +#if __has_include() return [BarcodeDetectorManagerMlkit constants]; #else return [NSDictionary new]; diff --git a/ios/RN/RNFaceDetectorModuleMLKit.m b/ios/RN/RNFaceDetectorModuleMLKit.m index 164f0d23e..c8a1c2935 100644 --- a/ios/RN/RNFaceDetectorModuleMLKit.m +++ b/ios/RN/RNFaceDetectorModuleMLKit.m @@ -1,5 +1,5 @@ #import "RNFaceDetectorModuleMLKit.h" -#if __has_include() +#if __has_include() #import "RNFileSystem.h" #import "RNImageUtils.h"