feat: add LipSyncAvatoon component and Android reference

android_reference/IMPLEMENTATION_GUIDE.md

@@ -0,0 +1,111 @@
+# Android 3D Avatar Lip Sync Implementation Guide
+
+This guide explains how to implement the 3D Avatar with "Lip Sync Only" mode (random mouth movement, no audio) and locked camera panning in a native Android application using Kotlin and [Sceneview](https://github.com/Sceneview/sceneview).
+
+## 1. Project Setup
+
+### Dependencies (build.gradle.kts)
+Ensure you have the Sceneview dependency in your module-level `build.gradle.kts`:
+
+```kotlin
+dependencies {
+    implementation("io.github.sceneview.android:sceneview:2.0.3") // Check for latest version
+}
+```
+
+## 2. Helper Class
+We have created a helper class `LipSyncManager` that handles the morph target manipulation. 
+Copy the file `LipSyncManager.kt` into your source set (e.g., `app/src/main/java/com/example/yourapp/LipSyncManager.kt`).
+
+**Key Features of LipSyncManager:**
+- **Randomized Visemes**: Picks random mouth shapes to simulate talking.
+- **Smooth Interpolation**: Uses `lerp` to smoothly transition between mouth shapes so it looks natural, not robotic.
+- **Toggle Control**: `setTalking(true/false)` to start/stop.
+
+## 3. Implementation Steps
+
+1.  **Layout XML**: Add a `io.github.sceneview.SceneView` and a `Button` to your layout.
+2.  **Activity/Fragment**:
+    -   Load your `.glb` model into a `ModelNode`.
+    -   Add the node to the `SceneView`.
+    -   Initialize `LipSyncManager` with the node.
+    -   Set up the Button `OnClickListener` to toggle `lipSyncManager.setTalking(isActive)`.
+    -   **Disable Panning**: Configure the camera manipulator to disallow panning/movement if essentially "locking" the view is desired, or simply don't attach a manipulator that allows it.
+
+## 4. AI Prompt for Integration
+
+If you need to generate the specific Activity/Fragment code for your existing Android project, use the prompt below. Copy and paste this into your AI coding assistant (like Android Studio Bot, ChatGPT, or Cursor).
+
+---
+### 📋 Copy This Prompt:
+
+```text
+I have a Kotlin Android project using Sceneview. 
+I need to implement a 3D Avatar viewer with the following specific requirements:
+
+1. **Load Avatar**: Load a GLB model (e.g., "avatar.glb") into the SceneView.
+2. **Lock Camera**: The user should NOT be able to pan or move the camera. The camera should be fixed on the avatar's head/upper body.
+3. **Lip Sync Feature**:
+   - Use the provided `LipSyncManager` class (I will provide this class definition).
+   - I need a toggle Button on the UI (overlaying the 3D view).
+   - When clicked, it should start the random lip-sync animation (no audio) using `lipSyncManager.setTalking(true)`.
+   - When clicked again, it should stop it.
+4. **UI Layout**: Please provide the XML layout with a SceneView and a styled floating button at the bottom.
+
+Here is the helper class I have:
+
+class LipSyncManager(private val avatarNode: ModelNode, private val scope: CoroutineScope) {
+    // ... (Your AI agent will infer the methods setTalking) ...
+    // It uses standard morph target setting on the node.
+}
+
+Please write the MainActivity.kt and activity_main.xml code.
+```
+---
+
+## 5. Sample Usage Code
+
+Here is a quick preview of how the `MainActivity` might look:
+
+```kotlin
+class MainActivity : AppCompatActivity() {
+
+    private lateinit var sceneView: SceneView
+    private lateinit var lipSyncManager: LipSyncManager
+    private var isTalking = false
+    private val modelUrl = "models/avatar.glb" // In assets folder
+
+    override fun onCreate(savedInstanceState: Bundle?) {
+        super.onCreate(savedInstanceState)
+        setContentView(R.layout.activity_main)
+
+        sceneView = findViewById(R.id.sceneView)
+        val toggleButton = findViewById<Button>(R.id.btnToggleTalk)
+
+        lifecycleScope.launchWhenCreated {
+            val modelInstance = sceneView.modelLoader.loadModelInstance(modelUrl) ?: return@launchWhenCreated
+            val modelNode = ModelNode(
+                modelInstance = modelInstance,
+                scaleToUnits = 1.0f
+            ).apply {
+                // Position avatar so head is visible
+                position = Position(y = -1.0f) 
+            }
+
+            sceneView.addChild(modelNode)
+
+            // Initialize Manager
+            lipSyncManager = LipSyncManager(modelNode, lifecycleScope)
+
+            // Lock Camera (Disable manipulation)
+            sceneView.cameraNode.manipulator = null 
+        }
+
+        toggleButton.setOnClickListener {
+            isTalking = !isTalking
+            lipSyncManager.setTalking(isTalking)
+            toggleButton.text = if (isTalking) "Stop Talking" else "Start Talking"
+        }
+    }
+}
+```

android_reference/LipSyncManager.kt

@@ -0,0 +1,123 @@
+package com.example.avatoon
+
+import android.animation.ValueAnimator
+import android.util.Log
+import io.github.sceneview.SceneView
+import io.github.sceneview.math.MathUtils
+import io.github.sceneview.node.ModelNode
+import kotlinx.coroutines.*
+import kotlin.random.Random
+
+/**
+ * A helper class to manage Lip Sync animation for a 3D Avatar using Sceneview.
+ *
+ * Usage:
+ * 1. Initialize with your ModelNode containing the avatar.
+ * 2. Call `setTalking(true)` to start random lip movement.
+ * 3. Call `setTalking(false)` to stop.
+ *
+ * Dependencies: io.github.sceneview.android:sceneview:2.0.3 (or later)
+ */
+class LipSyncManager(
+    private val avatarNode: ModelNode,
+    private val scope: CoroutineScope = CoroutineScope(Dispatchers.Main)
+) {
+
+    private var isTalking = false
+    private var updateJob: Job? = null
+
+    // Map of logical phonemes to the actual Morph Target names in your GLB
+    // Ensure these match the morph target names exported in your 3D model
+    private val visemeMap = mapOf(
+        "A" to "viseme_aa",
+        "B" to "viseme_PP",
+        "C" to "viseme_CH",
+        "D" to "viseme_DD",
+        "E" to "viseme_E",
+        "F" to "viseme_FF",
+        "I" to "viseme_I",
+        "O" to "viseme_oo",
+        "R" to "viseme_RR"
+    )
+
+    private val activeVisemes = visemeMap.values.toList()
+
+    // State for smooth animation
+    private var currentViseme: String? = null
+    private var currentInfluence = 0f
+    private var targetInfluence = 0f
+
+    // Animation loop speed
+    private val updateIntervalMs = 50L 
+
+    fun setTalking(talking: Boolean) {
+        if (isTalking == talking) return
+        isTalking = talking
+
+        if (talking) {
+            startLoop()
+        } else {
+            stopLoop()
+        }
+    }
+
+    private fun startLoop() {
+        updateJob?.cancel()
+        updateJob = scope.launch {
+            var nextChangeTime = System.currentTimeMillis()
+
+            while (isActive) {
+                val now = System.currentTimeMillis()
+
+                // Pick a new random viseme every ~100-200ms
+                if (now >= nextChangeTime) {
+                    currentViseme = activeVisemes.random()
+                    targetInfluence = if (Random.nextBoolean()) {
+                        Random.nextFloat() * 0.7f + 0.3f // 0.3 to 1.0
+                    } else {
+                        0f // Occasionally pause/silence
+                    }
+
+                    // Schedule next change
+                    nextChangeTime = now + Random.nextLong(50, 200)
+                }
+
+                // Interpolate (Lerp) towards target
+                // In a real game loop, you'd use delta time. Here we approximate with fixed delay.
+                currentInfluence = MathUtils.lerp(currentInfluence, targetInfluence, 0.2f)
+
+                // Apply to model
+                updateMorphTargets()
+
+                delay(updateIntervalMs)
+            }
+        }
+    }
+
+    private fun stopLoop() {
+        updateJob?.cancel()
+        // Smoothly close mouth
+        scope.launch {
+            targetInfluence = 0f
+            while (currentInfluence > 0.01f) {
+                currentInfluence = MathUtils.lerp(currentInfluence, 0f, 0.2f)
+                updateMorphTargets()
+                delay(updateIntervalMs)
+            }
+            currentInfluence = 0f
+            updateMorphTargets()
+        }
+    }
+
+    private fun updateMorphTargets() {
+        // Reset all known visemes to 0 first (or handle blending if your engine supports it)
+        activeVisemes.forEach { visemeName ->
+             avatarNode.setMorphTargetWeight(visemeName, 0f)
+        }
+
+        // Apply current active viseme weight
+        currentViseme?.let { visemeName ->
+            avatarNode.setMorphTargetWeight(visemeName, currentInfluence)
+        }
+    }
+}

bundle-report.html

@@ -4929,7 +4929,7 @@
   </script>
   <script>
     /*<!--*/
-    const data = {"version":2,"tree":{"name":"root","children":[{"name":"Avatoon.umd.js","children":[{"name":"src","children":[{"name":"constants/phonemeToViseme.ts","uid":"8ae8bf35-1"},{"name":"components","children":[{"uid":"8ae8bf35-3","name":"AvatoonModel.tsx"},{"uid":"8ae8bf35-5","name":"CameraFovAnimator.tsx"},{"uid":"8ae8bf35-7","name":"Avatoon.tsx"}]},{"uid":"8ae8bf35-9","name":"index.ts"}]}]}],"isRoot":true},"nodeParts":{"8ae8bf35-1":{"renderedLength":539,"gzipLength":0,"brotliLength":0,"metaUid":"8ae8bf35-0"},"8ae8bf35-3":{"renderedLength":7045,"gzipLength":0,"brotliLength":0,"metaUid":"8ae8bf35-2"},"8ae8bf35-5":{"renderedLength":485,"gzipLength":0,"brotliLength":0,"metaUid":"8ae8bf35-4"},"8ae8bf35-7":{"renderedLength":1779,"gzipLength":0,"brotliLength":0,"metaUid":"8ae8bf35-6"},"8ae8bf35-9":{"renderedLength":0,"gzipLength":0,"brotliLength":0,"metaUid":"8ae8bf35-8"}},"nodeMetas":{"8ae8bf35-0":{"id":"/src/constants/phonemeToViseme.ts","moduleParts":{"Avatoon.umd.js":"8ae8bf35-1"},"imported":[],"importedBy":[{"uid":"8ae8bf35-2"}]},"8ae8bf35-2":{"id":"/src/components/AvatoonModel.tsx","moduleParts":{"Avatoon.umd.js":"8ae8bf35-3"},"imported":[{"uid":"8ae8bf35-10"},{"uid":"8ae8bf35-11"},{"uid":"8ae8bf35-12"},{"uid":"8ae8bf35-13"},{"uid":"8ae8bf35-14"},{"uid":"8ae8bf35-0"}],"importedBy":[{"uid":"8ae8bf35-6"}]},"8ae8bf35-4":{"id":"/src/components/CameraFovAnimator.tsx","moduleParts":{"Avatoon.umd.js":"8ae8bf35-5"},"imported":[{"uid":"8ae8bf35-12"},{"uid":"8ae8bf35-11"},{"uid":"8ae8bf35-14"}],"importedBy":[{"uid":"8ae8bf35-6"}]},"8ae8bf35-6":{"id":"/src/components/Avatoon.tsx","moduleParts":{"Avatoon.umd.js":"8ae8bf35-7"},"imported":[{"uid":"8ae8bf35-10"},{"uid":"8ae8bf35-11"},{"uid":"8ae8bf35-12"},{"uid":"8ae8bf35-13"},{"uid":"8ae8bf35-2"},{"uid":"8ae8bf35-4"}],"importedBy":[{"uid":"8ae8bf35-8"}]},"8ae8bf35-8":{"id":"/src/index.ts","moduleParts":{"Avatoon.umd.js":"8ae8bf35-9"},"imported":[{"uid":"8ae8bf35-6"}],"importedBy":[],"isEntry":true},"8ae8bf35-10":{"id":"react/jsx-runtime","moduleParts":{},"imported":[],"importedBy":[{"uid":"8ae8bf35-6"},{"uid":"8ae8bf35-2"}],"isExternal":true},"8ae8bf35-11":{"id":"react","moduleParts":{},"imported":[],"importedBy":[{"uid":"8ae8bf35-6"},{"uid":"8ae8bf35-2"},{"uid":"8ae8bf35-4"}],"isExternal":true},"8ae8bf35-12":{"id":"@react-three/fiber","moduleParts":{},"imported":[],"importedBy":[{"uid":"8ae8bf35-6"},{"uid":"8ae8bf35-2"},{"uid":"8ae8bf35-4"}],"isExternal":true},"8ae8bf35-13":{"id":"@react-three/drei","moduleParts":{},"imported":[],"importedBy":[{"uid":"8ae8bf35-6"},{"uid":"8ae8bf35-2"}],"isExternal":true},"8ae8bf35-14":{"id":"three","moduleParts":{},"imported":[],"importedBy":[{"uid":"8ae8bf35-2"},{"uid":"8ae8bf35-4"}],"isExternal":true}},"env":{"rollup":"4.40.2"},"options":{"gzip":false,"brotli":false,"sourcemap":false}};
+    const data = {"version":2,"tree":{"name":"root","children":[{"name":"Avatoon.umd.js","children":[{"name":"src","children":[{"name":"constants/phonemeToViseme.ts","uid":"8a07c42f-1"},{"name":"components","children":[{"uid":"8a07c42f-3","name":"AvatoonModel.tsx"},{"uid":"8a07c42f-5","name":"CameraFovAnimator.tsx"},{"uid":"8a07c42f-7","name":"Avatoon.tsx"},{"uid":"8a07c42f-9","name":"LipSyncAvatoon.tsx"}]},{"uid":"8a07c42f-11","name":"index.ts"}]}]}],"isRoot":true},"nodeParts":{"8a07c42f-1":{"renderedLength":539,"gzipLength":0,"brotliLength":0,"metaUid":"8a07c42f-0"},"8a07c42f-3":{"renderedLength":7045,"gzipLength":0,"brotliLength":0,"metaUid":"8a07c42f-2"},"8a07c42f-5":{"renderedLength":485,"gzipLength":0,"brotliLength":0,"metaUid":"8a07c42f-4"},"8a07c42f-7":{"renderedLength":1779,"gzipLength":0,"brotliLength":0,"metaUid":"8a07c42f-6"},"8a07c42f-9":{"renderedLength":4941,"gzipLength":0,"brotliLength":0,"metaUid":"8a07c42f-8"},"8a07c42f-11":{"renderedLength":0,"gzipLength":0,"brotliLength":0,"metaUid":"8a07c42f-10"}},"nodeMetas":{"8a07c42f-0":{"id":"/src/constants/phonemeToViseme.ts","moduleParts":{"Avatoon.umd.js":"8a07c42f-1"},"imported":[],"importedBy":[{"uid":"8a07c42f-8"},{"uid":"8a07c42f-2"}]},"8a07c42f-2":{"id":"/src/components/AvatoonModel.tsx","moduleParts":{"Avatoon.umd.js":"8a07c42f-3"},"imported":[{"uid":"8a07c42f-12"},{"uid":"8a07c42f-13"},{"uid":"8a07c42f-14"},{"uid":"8a07c42f-15"},{"uid":"8a07c42f-16"},{"uid":"8a07c42f-0"}],"importedBy":[{"uid":"8a07c42f-6"}]},"8a07c42f-4":{"id":"/src/components/CameraFovAnimator.tsx","moduleParts":{"Avatoon.umd.js":"8a07c42f-5"},"imported":[{"uid":"8a07c42f-14"},{"uid":"8a07c42f-13"},{"uid":"8a07c42f-16"}],"importedBy":[{"uid":"8a07c42f-6"}]},"8a07c42f-6":{"id":"/src/components/Avatoon.tsx","moduleParts":{"Avatoon.umd.js":"8a07c42f-7"},"imported":[{"uid":"8a07c42f-12"},{"uid":"8a07c42f-13"},{"uid":"8a07c42f-14"},{"uid":"8a07c42f-15"},{"uid":"8a07c42f-2"},{"uid":"8a07c42f-4"}],"importedBy":[{"uid":"8a07c42f-10"}]},"8a07c42f-8":{"id":"/src/components/LipSyncAvatoon.tsx","moduleParts":{"Avatoon.umd.js":"8a07c42f-9"},"imported":[{"uid":"8a07c42f-12"},{"uid":"8a07c42f-13"},{"uid":"8a07c42f-14"},{"uid":"8a07c42f-15"},{"uid":"8a07c42f-16"},{"uid":"8a07c42f-0"}],"importedBy":[{"uid":"8a07c42f-10"}]},"8a07c42f-10":{"id":"/src/index.ts","moduleParts":{"Avatoon.umd.js":"8a07c42f-11"},"imported":[{"uid":"8a07c42f-6"},{"uid":"8a07c42f-8"}],"importedBy":[],"isEntry":true},"8a07c42f-12":{"id":"react/jsx-runtime","moduleParts":{},"imported":[],"importedBy":[{"uid":"8a07c42f-6"},{"uid":"8a07c42f-8"},{"uid":"8a07c42f-2"}],"isExternal":true},"8a07c42f-13":{"id":"react","moduleParts":{},"imported":[],"importedBy":[{"uid":"8a07c42f-6"},{"uid":"8a07c42f-8"},{"uid":"8a07c42f-2"},{"uid":"8a07c42f-4"}],"isExternal":true},"8a07c42f-14":{"id":"@react-three/fiber","moduleParts":{},"imported":[],"importedBy":[{"uid":"8a07c42f-6"},{"uid":"8a07c42f-8"},{"uid":"8a07c42f-2"},{"uid":"8a07c42f-4"}],"isExternal":true},"8a07c42f-15":{"id":"@react-three/drei","moduleParts":{},"imported":[],"importedBy":[{"uid":"8a07c42f-6"},{"uid":"8a07c42f-8"},{"uid":"8a07c42f-2"}],"isExternal":true},"8a07c42f-16":{"id":"three","moduleParts":{},"imported":[],"importedBy":[{"uid":"8a07c42f-8"},{"uid":"8a07c42f-2"},{"uid":"8a07c42f-4"}],"isExternal":true}},"env":{"rollup":"4.40.2"},"options":{"gzip":false,"brotli":false,"sourcemap":false}};
 
     const run = () => {
       const width = window.innerWidth;