INMO Air3 native client + ArUco fiducial alignment (phone validated, glasses WIP)

android/.gitignore

@@ -7,3 +7,7 @@
 /captures
 .externalNativeBuild
 .cxx
+
+# Local AAR drop-ins (e.g. INMO air3_core) — keep the README, ignore the binaries
+/app/libs/*.aar
+!/app/libs/README.md

android/app/build.gradle.kts

@@ -13,6 +13,10 @@ android {
         targetSdk = 34
         versionCode = 1
         versionName = "0.1.0"
+
+        // OpenCV native is large; restrict to arm64 (modern phones, incl. the Pixel 9a).
+        // Add "armeabi-v7a" here if you need to run on an older 32-bit device.
+        ndk { abiFilters += "arm64-v8a" }
     }
 
     buildTypes {
@@ -40,6 +44,12 @@ android {
 }
 
 dependencies {
+    // INMO Air3 native fusion (decision 011). Local .aar dropped into app/libs/.
+    // Absent by default — the app builds and falls back to Game Rotation Vector until
+    // the .aar is present. See app/libs/README.md. fileTree is used so this bypasses
+    // the FAIL_ON_PROJECT_REPOS restriction in settings.gradle.
+    implementation(fileTree(mapOf("dir" to "libs", "include" to listOf("*.aar"))))
+
     // Compose
     val composeBom = platform("androidx.compose:compose-bom:2024.01.00")
     implementation(composeBom)
@@ -60,6 +70,9 @@ dependencies {
     // ARCore
     implementation("com.google.ar:core:1.41.0")
 
+    // OpenCV (objdetect/ArucoDetector) for fiducial alignment to the hub frame.
+    implementation("org.opencv:opencv:4.11.0")
+
     // Core
     implementation("androidx.core:core-ktx:1.12.0")
     implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.7.0")

android/app/libs/README.md

@@ -0,0 +1,39 @@
+# Local AAR drop-in — INMO Air3 native fusion
+
+This directory holds local `.aar` dependencies that are **not** committed to git
+(see `.gitignore`). The build picks up any `*.aar` here automatically via the
+`fileTree(... "libs" ...)` dependency in `app/build.gradle.kts`.
+
+## What to drop here
+
+`air3_core-debug.aar` (or `air3_core-release.aar`) from the INMO Air3 SDK. This is
+the library that provides `com.inmo.air3_core.atw.GyroRotation`, whose `vQuat` field
+is the pre-calibrated head-orientation quaternion we consume in
+[`InmoFusionTracker.kt`](../src/main/java/com/pri4l/hub/InmoFusionTracker.kt).
+
+### How to obtain it
+
+1. INMO Air3 Unity SDK: <https://github.com/INMOXR/air3-unity-sdk>
+   The `air3_core` AAR ships inside the Unity package
+   (`Assets/Plugins/Android/` or the `.unitypackage`). Extract it.
+2. Alternatively pull it off-device from an installed INMO app:
+   `adb shell pm path <inmo.package>` → `adb pull` the APK → unzip and look for the
+   bundled native libs / classes. (The AAR repackages `libinmoair3.so` + the
+   `com.inmo.air3_core.*` classes.)
+3. Community wiki for pointers: <https://github.com/sam1am/inmo_air_3_wiki>
+
+## After dropping it in
+
+- Rebuild. `InmoFusionTracker.isAvailable()` flips to `true` and
+  `HeadTrackerFactory` selects the INMO path automatically.
+- Launch `GlassesTestActivity` (standalone cubes) and check logcat for:
+  `head tracker source: INMO GyroRotation.vQuat (native fusion)`.
+- Verify on-device and adjust the assumptions flagged in `InmoFusionTracker.kt`:
+  - vQuat component order ([x,y,z,w] vs [w,x,y,z]) — `copyQuat()`
+  - field type (float[] vs Quaternion object) — `readQuat()`
+  - lifecycle (getInstance / constructor / start signature) — `tryGetInstance` / `tryInvokeStart`
+  - eye-axis handedness — `axisFix`
+
+If the glasses still spin wrong after the quaternion reads correctly, fix `axisFix`
+(a constant eye-frame rotation) rather than re-introducing `remapCoordinateSystem`
+on the raw sensor path — that path cannot represent the mounting offset (decision 011).

android/app/src/main/AndroidManifest.xml

@@ -1,5 +1,14 @@
 <?xml version="1.0" encoding="utf-8"?>
-<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools">
+
+    <!--
+      air3_core.aar declares minSdkVersion 34, but its classes only ever load on the
+      INMO glasses (Android 14), gated by InmoFusionTracker.isAvailable() reflection.
+      Override the merge so the app stays installable on API 26 phones (the hub); the
+      INMO path simply never activates there. See decision 011.
+    -->
+    <uses-sdk tools:overrideLibrary="com.inmo.air3_core" />
 
     <uses-permission android:name="android.permission.INTERNET" />
     <uses-permission android:name="android.permission.CAMERA" />

android/app/src/main/java/com/pri4l/hub/FiducialAligner.kt

@@ -0,0 +1,173 @@
+package com.pri4l.hub
+
+import android.media.Image
+import android.opengl.Matrix
+import android.util.Log
+import com.google.ar.core.Frame
+import com.google.ar.core.TrackingState
+import com.google.ar.core.exceptions.NotYetAvailableException
+import org.opencv.calib3d.Calib3d
+import org.opencv.core.CvType
+import org.opencv.core.Mat
+import org.opencv.core.MatOfDouble
+import org.opencv.core.MatOfPoint2f
+import org.opencv.core.MatOfPoint3f
+import org.opencv.core.Point
+import org.opencv.core.Point3
+import org.opencv.objdetect.ArucoDetector
+import org.opencv.objdetect.Objdetect
+
+/**
+ * Detects an ArUco marker (DICT_4X4_50, id [TARGET_ID]) in the ARCore camera image and returns
+ * the marker's pose in the ARCore world frame. Used to align ARCore to the hub map frame: the
+ * tag is placed at the hub origin, so feeding the tag's AR-world pose to
+ * [FrameAlignment.align] (with hub pose = origin) links the two frames automatically — no need
+ * to physically stand the phone at the D435 (decision 009's manual flow).
+ *
+ * Coordinate handling: solvePnP yields the tag pose in the OpenCV camera frame (x-right, y-down,
+ * z-forward). ARCore's camera frame is x-right, y-up, z-backward, so we flip Y and Z
+ * (C = diag(1,-1,-1)) before composing with the ARCore camera pose.
+ */
+class FiducialAligner {
+
+    private val detector = ArucoDetector(Objdetect.getPredefinedDictionary(Objdetect.DICT_4X4_50))
+
+    private val camMatrix = Mat(3, 3, CvType.CV_64F)
+    private val distCoeffs = MatOfDouble(0.0, 0.0, 0.0, 0.0, 0.0)
+    private val objPoints = MatOfPoint3f()
+    private val rvec = Mat()
+    private val tvec = Mat()
+    private val rMat = Mat()
+
+    private val tArcamTag = FloatArray(16)
+    private val camPose = FloatArray(16)
+    private val tagWorld = FloatArray(16)
+
+    @Volatile var lastRangeM = 0f
+        private set
+
+    /**
+     * @return the tag pose in ARCore world as ([x,y,z], [qx,qy,qz,qw]), or null if no marker /
+     *         not tracking / image unavailable.
+     */
+    fun detectTagInArWorld(frame: Frame): Pair<FloatArray, FloatArray>? {
+        val cam = frame.camera
+        if (cam.trackingState != TrackingState.TRACKING) return null
+
+        val image: Image = try {
+            frame.acquireCameraImage()
+        } catch (e: NotYetAvailableException) {
+            return null
+        }
+        try {
+            val gray = grayFromY(image) ?: return null
+
+            val corners = ArrayList<Mat>()
+            val ids = Mat()
+            try {
+                detector.detectMarkers(gray, corners, ids)
+                if (ids.empty()) return null
+
+                var idx = -1
+                for (i in 0 until ids.rows()) {
+                    if (ids.get(i, 0)[0].toInt() == TARGET_ID) { idx = i; break }
+                }
+                if (idx < 0) return null
+
+                // Intrinsics from ARCore, matching the sensor-native acquired image.
+                val intr = cam.imageIntrinsics
+                val fl = intr.focalLength
+                val pp = intr.principalPoint
+                camMatrix.put(0, 0, fl[0].toDouble(), 0.0, pp[0].toDouble(),
+                                    0.0, fl[1].toDouble(), pp[1].toDouble(),
+                                    0.0, 0.0, 1.0)
+
+                val L = MARKER_SIZE_M.toDouble()
+                objPoints.fromArray(
+                    Point3(-L / 2, L / 2, 0.0), Point3(L / 2, L / 2, 0.0),
+                    Point3(L / 2, -L / 2, 0.0), Point3(-L / 2, -L / 2, 0.0)
+                )
+                val img = MatOfPoint2f()
+                val pts = ArrayList<Point>(4)
+                val c = corners[idx]
+                for (k in 0 until 4) { val v = c.get(0, k); pts.add(Point(v[0], v[1])) }
+                img.fromList(pts)
+
+                val ok = Calib3d.solvePnP(objPoints, img, camMatrix, distCoeffs, rvec, tvec)
+                img.release()
+                if (!ok) return null
+
+                Calib3d.Rodrigues(rvec, rMat)
+                val r = DoubleArray(9); rMat.get(0, 0, r)   // row-major tag->cv-cam
+                val t = DoubleArray(3); tvec.get(0, 0, t)
+                lastRangeM = Math.sqrt(t[0] * t[0] + t[1] * t[1] + t[2] * t[2]).toFloat()
+
+                // C = diag(1,-1,-1): OpenCV cam -> ARCore cam (negate rows 1,2 of R and t1,t2).
+                // T_arcam_tag column-major.
+                tArcamTag[0] = r[0].toFloat(); tArcamTag[1] = (-r[3]).toFloat(); tArcamTag[2] = (-r[6]).toFloat(); tArcamTag[3] = 0f
+                tArcamTag[4] = r[1].toFloat(); tArcamTag[5] = (-r[4]).toFloat(); tArcamTag[6] = (-r[7]).toFloat(); tArcamTag[7] = 0f
+                tArcamTag[8] = r[2].toFloat(); tArcamTag[9] = (-r[5]).toFloat(); tArcamTag[10] = (-r[8]).toFloat(); tArcamTag[11] = 0f
+                tArcamTag[12] = t[0].toFloat(); tArcamTag[13] = (-t[1]).toFloat(); tArcamTag[14] = (-t[2]).toFloat(); tArcamTag[15] = 1f
+
+                // tagWorld = cameraPose(world<-arcam) * T_arcam_tag
+                cam.pose.toMatrix(camPose, 0)
+                Matrix.multiplyMM(tagWorld, 0, camPose, 0, tArcamTag, 0)
+
+                val pos = floatArrayOf(tagWorld[12], tagWorld[13], tagWorld[14])
+                val rot = matrixToQuaternion(tagWorld)
+                Log.w("Pri4L", "FIDUCIAL: range=%.2fm tagWorld=[%.2f, %.2f, %.2f]"
+                    .format(lastRangeM, pos[0], pos[1], pos[2]))
+                return Pair(pos, rot)
+            } finally {
+                gray.release(); ids.release(); corners.forEach { it.release() }
+            }
+        } catch (e: Throwable) {
+            Log.e("Pri4L", "fiducial detect failed", e)
+            return null
+        } finally {
+            image.close()
+        }
+    }
+
+    private fun grayFromY(image: Image): Mat? {
+        if (image.planes.isEmpty()) return null
+        val y = image.planes[0]
+        val buf = y.buffer
+        val w = image.width
+        val h = image.height
+        val gray = Mat(h, w, CvType.CV_8UC1)
+        val stride = y.rowStride
+        if (stride == w) {
+            val data = ByteArray(w * h); buf.get(data); gray.put(0, 0, data)
+        } else {
+            val row = ByteArray(w)
+            for (r in 0 until h) { buf.position(r * stride); buf.get(row, 0, w); gray.put(r, 0, row) }
+        }
+        return gray
+    }
+
+    private fun matrixToQuaternion(m: FloatArray): FloatArray {
+        val trace = m[0] + m[5] + m[10]
+        val q = FloatArray(4)
+        if (trace > 0) {
+            val s = 0.5f / Math.sqrt((trace + 1.0).toDouble()).toFloat()
+            q[3] = 0.25f / s; q[0] = (m[6] - m[9]) * s; q[1] = (m[8] - m[2]) * s; q[2] = (m[1] - m[4]) * s
+        } else if (m[0] > m[5] && m[0] > m[10]) {
+            val s = 2f * Math.sqrt((1.0 + m[0] - m[5] - m[10]).toDouble()).toFloat()
+            q[3] = (m[6] - m[9]) / s; q[0] = 0.25f * s; q[1] = (m[4] + m[1]) / s; q[2] = (m[8] + m[2]) / s
+        } else if (m[5] > m[10]) {
+            val s = 2f * Math.sqrt((1.0 + m[5] - m[0] - m[10]).toDouble()).toFloat()
+            q[3] = (m[8] - m[2]) / s; q[0] = (m[4] + m[1]) / s; q[1] = 0.25f * s; q[2] = (m[9] + m[6]) / s
+        } else {
+            val s = 2f * Math.sqrt((1.0 + m[10] - m[0] - m[5]).toDouble()).toFloat()
+            q[3] = (m[1] - m[4]) / s; q[0] = (m[8] + m[2]) / s; q[1] = (m[9] + m[6]) / s; q[2] = 0.25f * s
+        }
+        return q
+    }
+
+    companion object {
+        private const val TARGET_ID = 0
+        /** Printed black-square width in metres. MUST match the physical print. */
+        const val MARKER_SIZE_M = 0.15f
+    }
+}

android/app/src/main/java/com/pri4l/hub/GlassesRenderer.kt

@@ -12,7 +12,7 @@ import javax.microedition.khronos.opengles.GL10
  * No camera background — renders on transparent/black background.
  */
 class GlassesRenderer(
-    private val tracker: GlassesTracker,
+    private val tracker: HeadTracker,
     private val getHubAnchors: () -> List<FloatArray>,
     private val getPhoneAnchors: () -> List<FloatArray>,
     private val getAlignmentMatrix: () -> FloatArray?

android/app/src/main/java/com/pri4l/hub/GlassesTestActivity.kt

@@ -5,6 +5,7 @@ import android.opengl.GLES20
 import android.opengl.GLSurfaceView
 import android.opengl.Matrix
 import android.os.Bundle
+import android.view.KeyEvent
 import android.view.WindowManager
 import javax.microedition.khronos.egl.EGLConfig
 import javax.microedition.khronos.opengles.GL10
@@ -16,7 +17,7 @@ import javax.microedition.khronos.opengles.GL10
 class GlassesTestActivity : Activity() {
 
     private lateinit var glView: GLSurfaceView
-    private lateinit var tracker: GlassesTracker
+    private lateinit var tracker: HeadTracker
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
@@ -25,8 +26,10 @@ class GlassesTestActivity : Activity() {
         window.addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN)
         window.addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)
 
-        tracker = GlassesTracker(this)
+        // Picks INMO native fusion if the air3_core AAR is present, else Game Rotation Vector.
+        tracker = HeadTrackerFactory.create(this)
         tracker.start()
+        android.util.Log.w("Pri4L", "GlassesTest head tracker: ${tracker.sourceName}")
 
         glView = GLSurfaceView(this).apply {
             setEGLContextClientVersion(2)
@@ -48,9 +51,18 @@ class GlassesTestActivity : Activity() {
         glView.onPause()
         tracker.stop()
     }
+
+    // Look straight ahead, then tap the temple touchpad (any key) to redefine "forward".
+    // The auto-recenter on first frame can capture a stale/pitched reference, leaving
+    // content offset (e.g. cubes sitting below the gaze); this lets the wearer fix it.
+    override fun onKeyDown(keyCode: Int, event: KeyEvent?): Boolean {
+        tracker.recenter()
+        android.util.Log.w("Pri4L", "recenter requested (keyCode=$keyCode)")
+        return true
+    }
 }
 
-class HeadTrackedCubeRenderer(private val tracker: GlassesTracker) : GLSurfaceView.Renderer {
+class HeadTrackedCubeRenderer(private val tracker: HeadTracker) : GLSurfaceView.Renderer {
 
     private var cubeRenderer = CubeRenderer()
     private val mvpMatrix = FloatArray(16)

android/app/src/main/java/com/pri4l/hub/GlassesTracker.kt

@@ -12,7 +12,9 @@ import android.opengl.Matrix
  * Provides a view matrix suitable for rendering anchors relative to head orientation.
  * No positional tracking — anchors will rotate correctly but no parallax.
  */
-class GlassesTracker(context: Context) : SensorEventListener {
+class GlassesTracker(context: Context) : SensorEventListener, HeadTracker {
+
+    override val sourceName: String = "Android Game Rotation Vector (WIP/broken axes)"
 
     private val sensorManager = context.getSystemService(Context.SENSOR_SERVICE) as SensorManager
     private val rotationSensor = sensorManager.getDefaultSensor(Sensor.TYPE_GAME_ROTATION_VECTOR)
@@ -21,34 +23,37 @@ class GlassesTracker(context: Context) : SensorEventListener {
     private val viewMatrix = FloatArray(16)
 
     @Volatile
-    var isTracking = false
+    override var isTracking = false
         private set
 
     init {
         Matrix.setIdentityM(rotationMatrix, 0)
         Matrix.setIdentityM(viewMatrix, 0)
     }
 
-    fun start() {
+    override fun start() {
         if (rotationSensor != null) {
             sensorManager.registerListener(this, rotationSensor, SensorManager.SENSOR_DELAY_GAME)
             isTracking = true
         }
     }
 
-    fun stop() {
+    override fun stop() {
         sensorManager.unregisterListener(this)
         isTracking = false
     }
 
     private val remappedMatrix = FloatArray(16)
 
-    fun getViewMatrix(dest: FloatArray) {
+    override fun getViewMatrix(dest: FloatArray) {
         synchronized(rotationMatrix) {
-            // Glasses are landscape with screen facing outward.
-            // Use AXIS_Y, AXIS_MINUS_X: equivalent to 90° CW rotation of device,
-            // which maps landscape-native sensor data to GL correctly.
-            // head yaw → GL yaw, head pitch → GL pitch, head roll → GL roll
+            // WIP / KNOWN-BROKEN: head yaw currently shows up as roll. See decision 011.
+            // remapCoordinateSystem can only express axis-aligned 90° permutations and
+            // cannot represent the real (non-axis-aligned) IMU-to-eye mounting offset of
+            // the INMO chassis, so NO axis pair fixes all three of yaw/pitch/roll.
+            // Do not keep permuting these axes — replace this path with INMO's calibrated
+            // fusion quaternion (GyroRotation.vQuat) or an empirically-solved correction
+            // quaternion C applied as view = C · Rᵀ. This call is a placeholder.
             SensorManager.remapCoordinateSystem(
                 rotationMatrix,
                 SensorManager.AXIS_Y,