WIP: Standalone mlkit

android/build.gradle

@@ -74,6 +74,8 @@ dependencies {
   implementation "androidx.exifinterface:exifinterface:1.0.0"
   implementation "androidx.annotation:annotation:1.0.0"
   implementation "androidx.legacy:legacy-support-v4:1.0.0"
-  mlkitImplementation "com.google.firebase:firebase-ml-vision:${safeExtGet('firebase-ml-vision', '19.0.3')}"
-  mlkitImplementation "com.google.firebase:firebase-ml-vision-face-model:${safeExtGet('firebase-ml-vision-face-model', '17.0.2')}"
+  mlkitImplementation "com.google.android.gms:play-services-mlkit-text-recognition:${safeExtGet('mlkit-text-recognition', '16.0.0')}"
+  mlkitImplementation "com.google.mlkit:barcode-scanning:${safeExtGet('mlkit-barcode-scanning', '16.0.0')}"
+  mlkitImplementation "com.google.mlkit:face-detection:${safeExtGet('mlkit-face-detection', '16.0.0')}"
+  mlkitImplementation "com.google.mlkit:image-labeling:${safeExtGet('mlkit-image-labeling', '16.0.0')}"
 }

android/src/main/java/org/reactnative/camera/CameraViewManager.java

@@ -21,8 +21,10 @@ public enum Events {
     EVENT_ON_BAR_CODE_READ("onBarCodeRead"),
     EVENT_ON_FACES_DETECTED("onFacesDetected"),
     EVENT_ON_BARCODES_DETECTED("onGoogleVisionBarcodesDetected"),
+    EVENT_ON_LABELS_DETECTED("onLabelsDetected"),
     EVENT_ON_FACE_DETECTION_ERROR("onFaceDetectionError"),
     EVENT_ON_BARCODE_DETECTION_ERROR("onGoogleVisionBarcodeDetectionError"),
+    EVENT_ON_LABEL_DETECTION_ERROR("onLabelDetectionError"),
     EVENT_ON_TEXT_RECOGNIZED("onTextRecognized"),
     EVENT_ON_PICTURE_TAKEN("onPictureTaken"),
     EVENT_ON_PICTURE_SAVED("onPictureSaved"),
@@ -217,6 +219,11 @@ public void setTextRecognizing(RNCameraView view, boolean textRecognizerEnabled)
     view.setShouldRecognizeText(textRecognizerEnabled);
   }
 
+  @ReactProp(name = "labelDetectorEnabled")
+  public void setLabelDetecting(RNCameraView view, boolean labelDetectorEnabled) {
+    view.setShouldDetectLabels(labelDetectorEnabled);
+  }
+
   /**---limit scan area addition---**/
   @ReactProp(name = "rectOfInterest")
   public void setRectOfInterest(RNCameraView view, ReadableMap coordinates) {

android/src/main/java/org/reactnative/camera/RNCameraView.java

@@ -30,6 +30,7 @@
 import org.reactnative.camera.tasks.*;
 import org.reactnative.camera.utils.RNFileUtils;
 import org.reactnative.facedetector.RNFaceDetector;
+import org.reactnative.imagelabeler.RNImageLabeler;
 
 import java.io.ByteArrayOutputStream;
 import java.io.File;
@@ -39,7 +40,7 @@
 import java.util.concurrent.ConcurrentLinkedQueue;
 
 public class RNCameraView extends CameraView implements LifecycleEventListener, BarCodeScannerAsyncTaskDelegate, FaceDetectorAsyncTaskDelegate,
-    BarcodeDetectorAsyncTaskDelegate, TextRecognizerAsyncTaskDelegate, PictureSavedDelegate {
+    BarcodeDetectorAsyncTaskDelegate, TextRecognizerAsyncTaskDelegate, ImageLabelerAsyncTaskDelegate, PictureSavedDelegate {
   private ThemedReactContext mThemedReactContext;
   private Queue<Promise> mPictureTakenPromises = new ConcurrentLinkedQueue<>();
   private Map<Promise, ReadableMap> mPictureTakenOptions = new ConcurrentHashMap<>();
@@ -64,16 +65,19 @@ public class RNCameraView extends CameraView implements LifecycleEventListener,
   public volatile boolean faceDetectorTaskLock = false;
   public volatile boolean googleBarcodeDetectorTaskLock = false;
   public volatile boolean textRecognizerTaskLock = false;
+  public volatile boolean imageLabelerTaskLock = false;
 
   // Scanning-related properties
   private MultiFormatReader mMultiFormatReader;
   private RNFaceDetector mFaceDetector;
   private RNBarcodeDetector mGoogleBarcodeDetector;
+  private RNImageLabeler mImageLabeler;
   private boolean mShouldDetectFaces = false;
   private boolean mShouldGoogleDetectBarcodes = false;
   private boolean mShouldScanBarCodes = false;
   private boolean mShouldRecognizeText = false;
   private boolean mShouldDetectTouches = false;
+  private boolean mShouldDetectLabels = false;
   private int mFaceDetectorMode = RNFaceDetector.FAST_MODE;
   private int mFaceDetectionLandmarks = RNFaceDetector.NO_LANDMARKS;
   private int mFaceDetectionClassifications = RNFaceDetector.NO_CLASSIFICATIONS;
@@ -166,7 +170,9 @@ public void onFramePreview(CameraView cameraView, byte[] data, int width, int he
         boolean willCallFaceTask = mShouldDetectFaces && !faceDetectorTaskLock && cameraView instanceof FaceDetectorAsyncTaskDelegate;
         boolean willCallGoogleBarcodeTask = mShouldGoogleDetectBarcodes && !googleBarcodeDetectorTaskLock && cameraView instanceof BarcodeDetectorAsyncTaskDelegate;
         boolean willCallTextTask = mShouldRecognizeText && !textRecognizerTaskLock && cameraView instanceof TextRecognizerAsyncTaskDelegate;
-        if (!willCallBarCodeTask && !willCallFaceTask && !willCallGoogleBarcodeTask && !willCallTextTask) {
+        boolean willCallLabelTask = mShouldDetectLabels && !imageLabelerTaskLock && cameraView instanceof ImageLabelerAsyncTaskDelegate;
+
+        if (!willCallBarCodeTask && !willCallFaceTask && !willCallGoogleBarcodeTask && !willCallTextTask && !willCallLabelTask) {
           return;
         }
 
@@ -211,6 +217,12 @@ correctRotation, getResources().getDisplayMetrics().density, getFacing(),
           TextRecognizerAsyncTaskDelegate delegate = (TextRecognizerAsyncTaskDelegate) cameraView;
           new TextRecognizerAsyncTask(delegate, mThemedReactContext, data, width, height, correctRotation, getResources().getDisplayMetrics().density, getFacing(), getWidth(), getHeight(), mPaddingX, mPaddingY).execute();
         }
+
+        if (willCallLabelTask) {
+          imageLabelerTaskLock = true;
+          ImageLabelerAsyncTaskDelegate delegate = (ImageLabelerAsyncTaskDelegate) cameraView;
+          new ImageLabelerAsyncTask(delegate, mImageLabeler, data, width, height, correctRotation, getResources().getDisplayMetrics().density, getFacing(), getWidth(), getHeight(), mPaddingX, mPaddingY).execute();
+        }
       }
     });
   }
@@ -362,7 +374,7 @@ public void setShouldScanBarCodes(boolean shouldScanBarCodes) {
       initBarcodeReader();
     }
     this.mShouldScanBarCodes = shouldScanBarCodes;
-    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
+    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
   }
 
   public void onBarCodeRead(Result barCode, int width, int height, byte[] imageData) {
@@ -483,7 +495,7 @@ public void setShouldDetectFaces(boolean shouldDetectFaces) {
       setupFaceDetector();
     }
     this.mShouldDetectFaces = shouldDetectFaces;
-    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
+    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
   }
 
   public void onFacesDetected(WritableArray data) {
@@ -520,7 +532,7 @@ public void setShouldGoogleDetectBarcodes(boolean shouldDetectBarcodes) {
       setupBarcodeDetector();
     }
     this.mShouldGoogleDetectBarcodes = shouldDetectBarcodes;
-    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
+    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
   }
 
   public void setGoogleVisionBarcodeType(int barcodeType) {
@@ -571,14 +583,49 @@ public void onBarcodeDetectingTaskCompleted() {
     googleBarcodeDetectorTaskLock = false;
   }
 
+    /**
+   * Initial setup of the image labeler
+   */
+  private void setupImageLabeler() {
+    mImageLabeler = new RNImageLabeler(mThemedReactContext);
+  }
+
+  public void setShouldDetectLabels(boolean shouldDetectLabels) {
+    if (shouldDetectLabels && mImageLabeler == null) {
+      setupImageLabeler();
+    }
+    this.mShouldDetectLabels = shouldDetectLabels;
+    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
+  }
+
+  public void onLabelsDetected(WritableArray labelsDetected) {
+    if (!mShouldDetectLabels) {
+      return;
+    }
+    RNCameraViewHelper.emitLabelsDetectedEvent(this, labelsDetected);
+  }
+
+  public void onImageLabelingError(RNImageLabeler imageLabeler) {
+    if (!mShouldDetectLabels) {
+      return;
+    }
+
+    RNCameraViewHelper.emitImageLabelingErrorEvent(this, imageLabeler);
+  }
+
+  @Override
+  public void onImageLabelingTaskCompleted() {
+    imageLabelerTaskLock = false;
+  }
+
   /**
    *
    * Text recognition
    */
 
   public void setShouldRecognizeText(boolean shouldRecognizeText) {
     this.mShouldRecognizeText = shouldRecognizeText;
-    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText);
+    setScanning(mShouldDetectFaces || mShouldGoogleDetectBarcodes || mShouldScanBarCodes || mShouldRecognizeText || mShouldDetectLabels);
   }
 
   public void onTextRecognized(WritableArray serializedData) {

android/src/main/java/org/reactnative/camera/RNCameraViewHelper.java

@@ -19,6 +19,7 @@
 import org.reactnative.camera.events.*;
 import org.reactnative.barcodedetector.RNBarcodeDetector;
 import org.reactnative.facedetector.RNFaceDetector;
+import org.reactnative.imagelabeler.RNImageLabeler;
 
 import java.text.SimpleDateFormat;
 import java.util.Calendar;
@@ -334,6 +335,32 @@ public void run() {
     });
   }
 
+  // Image labeling events
+
+  public static void emitLabelsDetectedEvent(final ViewGroup view, final WritableArray data) {
+
+    final ReactContext reactContext = (ReactContext) view.getContext();
+    reactContext.runOnNativeModulesQueueThread(new Runnable() {
+      @Override
+      public void run() {
+        LabelsDetectedEvent event = LabelsDetectedEvent.obtain(view.getId(), data);
+        reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event);
+      }
+    });
+  }
+
+  public static void emitImageLabelingErrorEvent(final ViewGroup view, final RNImageLabeler imageLabeler) {
+
+    final ReactContext reactContext = (ReactContext) view.getContext();
+    reactContext.runOnNativeModulesQueueThread(new Runnable() {
+      @Override
+      public void run() {
+        LabelDetectionErrorEvent event = LabelDetectionErrorEvent.obtain(view.getId(), imageLabeler);
+        reactContext.getNativeModule(UIManagerModule.class).getEventDispatcher().dispatchEvent(event);
+      }
+    });
+  }
+
   // Utilities
 
   public static int getCorrectCameraRotation(int rotation, int facing, int cameraOrientation) {

android/src/main/java/org/reactnative/camera/events/LabelDetectionErrorEvent.java

@@ -0,0 +1,54 @@
+package org.reactnative.camera.events;
+
+import androidx.core.util.Pools;
+
+import com.facebook.react.bridge.Arguments;
+import com.facebook.react.bridge.WritableMap;
+import com.facebook.react.uimanager.events.Event;
+import com.facebook.react.uimanager.events.RCTEventEmitter;
+
+import org.reactnative.camera.CameraViewManager;
+import org.reactnative.imagelabeler.RNImageLabeler;
+
+public class LabelDetectionErrorEvent extends Event<LabelDetectionErrorEvent> {
+  private static final Pools.SynchronizedPool<LabelDetectionErrorEvent> EVENTS_POOL = new Pools.SynchronizedPool<>(3);
+  private RNImageLabeler mImageLabeler;
+
+  private LabelDetectionErrorEvent() {
+  }
+
+  public static LabelDetectionErrorEvent obtain(int viewTag, RNImageLabeler imageLabeler) {
+    LabelDetectionErrorEvent event = EVENTS_POOL.acquire();
+    if (event == null) {
+      event = new LabelDetectionErrorEvent();
+    }
+    event.init(viewTag, imageLabeler);
+    return event;
+  }
+
+  private void init(int viewTag, RNImageLabeler imageLabeler) {
+    super.init(viewTag);
+    mImageLabeler = imageLabeler;
+  }
+
+  @Override
+  public short getCoalescingKey() {
+    return 0;
+  }
+
+  @Override
+  public String getEventName() {
+    return CameraViewManager.Events.EVENT_ON_LABEL_DETECTION_ERROR.toString();
+  }
+
+  @Override
+  public void dispatch(RCTEventEmitter rctEventEmitter) {
+    rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData());
+  }
+
+  private WritableMap serializeEventData() {
+    WritableMap map = Arguments.createMap();
+    map.putBoolean("isOperational", mImageLabeler != null && mImageLabeler.isOperational());
+    return map;
+  }
+}

android/src/main/java/org/reactnative/camera/events/LabelsDetectedEvent.java

@@ -0,0 +1,61 @@
+package org.reactnative.camera.events;
+
+import androidx.core.util.Pools;
+
+import com.facebook.react.bridge.Arguments;
+import com.facebook.react.bridge.WritableArray;
+import com.facebook.react.bridge.WritableMap;
+import com.facebook.react.uimanager.events.Event;
+import com.facebook.react.uimanager.events.RCTEventEmitter;
+
+import org.reactnative.camera.CameraViewManager;
+
+public class LabelsDetectedEvent extends Event<LabelsDetectedEvent> {
+  private static final Pools.SynchronizedPool<LabelsDetectedEvent> EVENTS_POOL =
+      new Pools.SynchronizedPool<>(3);
+
+  private WritableArray mData;
+
+  private LabelsDetectedEvent() {}
+
+  public static LabelsDetectedEvent obtain(int viewTag, WritableArray data) {
+    LabelsDetectedEvent event = EVENTS_POOL.acquire();
+    if (event == null) {
+      event = new LabelsDetectedEvent();
+    }
+    event.init(viewTag, data);
+    return event;
+  }
+
+  private void init(int viewTag, WritableArray data) {
+    super.init(viewTag);
+    mData = data;
+  }
+
+  @Override
+  public short getCoalescingKey() {
+    if (mData.size() > Short.MAX_VALUE) {
+      return Short.MAX_VALUE;
+    }
+
+    return (short) mData.size();
+  }
+
+  @Override
+  public String getEventName() {
+    return CameraViewManager.Events.EVENT_ON_LABELS_DETECTED.toString();
+  }
+
+  @Override
+  public void dispatch(RCTEventEmitter rctEventEmitter) {
+    rctEventEmitter.receiveEvent(getViewTag(), getEventName(), serializeEventData());
+  }
+
+  private WritableMap serializeEventData() {
+    WritableMap event = Arguments.createMap();
+    event.putString("type", "label");
+    event.putArray("labels", mData);
+    event.putInt("target", getViewTag());
+    return event;
+  }
+}

android/src/main/java/org/reactnative/camera/tasks/ImageLabelerAsyncTaskDelegate.java

@@ -0,0 +1,13 @@
+package org.reactnative.camera.tasks;
+
+import com.facebook.react.bridge.WritableArray;
+import org.reactnative.imagelabeler.RNImageLabeler;
+
+public interface ImageLabelerAsyncTaskDelegate {
+
+    void onLabelsDetected(WritableArray labels);
+
+    void onImageLabelingError(RNImageLabeler imageLabeler);
+
+    void onImageLabelingTaskCompleted();
+}