diff --git a/FNOL_WORKFLOW_GUIDE.md b/FNOL_WORKFLOW_GUIDE.md new file mode 100644 index 0000000..12a71d9 --- /dev/null +++ b/FNOL_WORKFLOW_GUIDE.md @@ -0,0 +1,598 @@ +# FNOL (First Notice of Loss) Workflow Guide for DialogLab + +This guide structures the insurance claim intake conversation between Alice (Claims Adjuster) and Bob (Elderly Claimant) into organized scenes and conversation nodes. + +--- + +## Recommended Scene Structure + +Create multiple conversation nodes (scenes) following this sequence: + +### Scene 1: Initial Contact & Safety Check +**Title:** "Initial Report and Emergency Assessment" + +**Context:** +Bob is calling to report an accident. Alice answers and immediately assesses if there are safety/medical emergencies. + +**Key Questions Alice Should Ask:** +- Is anyone injured? +- Are all parties safe from traffic hazards? +- Do you need emergency services? +- Are there any hazardous material spills? + +**Information to Capture:** +- Current status of all parties +- Whether emergency services were called +- Current location safety status + +**Dialogue Notes:** +- Alice should: Express concern, be calm, prioritize safety +- Bob should: Report stress level, explain immediate situation + +--- + +### Scene 2: Policy & Policyholder Verification +**Title:** "Policy Information and Contact Details" + +**Context:** +Once safety is confirmed, Alice verifies the policy and gets Bob's full contact information. + +**Key Questions Alice Should Ask:** +- What is your policy number? (or license plate / VIN) +- Are you the policyholder? +- Can you confirm your full name and date of birth? +- What is your telephone number? +- What is your email address? +- How would you prefer we contact you (phone, email, mail)? + +**Information to Capture:** +- [ ] Policy number +- [ ] Policyholder name and DOB +- [ ] Current contact: phone, email, address +- [ ] Preferred communication channel + +**Additional Checks:** +- Verify policy is active and in-force on date of loss +- Check if premium is current (no arrears) +- Confirm vehicle and driver are on policy + +--- + +### Scene 3: Reporter Type & Relationship +**Title:** "Who Is Reporting This Claim?" + +**Context:** +Establish who is making the report and their relationship to the incident. This affects the downstream investigation flow. + +**Key Questions Alice Should Ask:** +- Are you the driver of the vehicle? +- If not you driving: Who was driving? (name, age, driver license info) +- What is your relationship to the driver? +- Is the driver a household member? + +**Information to Capture:** +- Reporter type: ☐ Policyholder ☐ Driver ☐ Other (specify) +- If not driver: Driver's full name, DOB, relationship +- Driver license number and expiry +- Is driver listed on policy? Check driver scope + +**Red Flags to Note:** +- If not the policyholder or named driver calling +- If driver is underage restriction on policy +- If driver recently added to policy + +--- + +### Scene 4: Accident Date, Time & Location +**Title:** "When and Where Did the Accident Happen?" + +**Context:** +Establish the precise timeline and location of the loss. + +**Key Questions Alice Should Ask:** +- What is the exact date of the accident? +- What time did it happen? (as precise as possible) +- What time are you reporting it now? (delay matters for duty-of-disclosure) +- What is the full address/location of the accident? +- What country/city? +- What type of road? (highway, urban, rural, parking lot) +- What were the weather conditions? +- What was the visibility like? +- What were the road conditions? (dry, wet, icy, gravel, etc.) + +**Information to Capture:** +- [ ] Date of loss: __ / __ / ____ +- [ ] Time of loss: __:__ (AM/PM) +- [ ] Time of report: __:__ (AM/PM) — **Note delay** +- [ ] Full address and GPS coordinates if available +- [ ] Country and region +- [ ] Road type: ☐ Highway ☐ Urban ☐ Rural ☐ Parking ☐ Other +- [ ] Weather: ☐ Clear ☐ Rain ☐ Snow ☐ Fog ☐ Other +- [ ] Visibility: ☐ Good ☐ Reduced ☐ Poor +- [ ] Road condition: ☐ Dry ☐ Wet ☐ Icy ☐ Gravel ☐ Other + +**Red Flags:** +- Significant delay between accident and report (>7 days may trigger duty-of-disclosure review) +- Accident in geographic area outside policy coverage + +--- + +### Scene 5: Type of Loss & What Happened +**Title:** "What Type of Accident Was It?" + +**Context:** +Classify the loss and get Bob's account of how the accident happened in his own words. + +**Key Questions Alice Should Ask:** +- What type of loss? (collision, parking damage, animal strike, etc.) +- In your own words, what happened? Please describe the sequence of events. +- Where were you traveling? (direction) +- What speed were you traveling at (estimate)? +- Was there another vehicle involved? +- If yes: Did the other vehicle hit you, or did you hit it? +- Who had right-of-way? +- Any traffic signals or signs involved? +- Did anyone admit fault at the scene? +- Do you have photos or a sketch of the damage? + +**Information to Capture:** +- [ ] Type of loss: + - ☐ Moving traffic collision (2+ vehicles) + - ☐ Stationary vehicle collision (hit parked car) + - ☐ Parking lot collision + - ☐ Wildlife/animal strike + - ☐ Property damage (guardrail, fence, signage, building) + - ☐ Other: __________ +- [ ] Free-form narrative (Bob's account in his own words) +- [ ] Direction of travel +- [ ] Estimated speed +- [ ] Right-of-way situation +- [ ] Traffic signals/signs involved +- [ ] Fault admission at scene (yes/no, by whom) +- [ ] Photos available? ☐ Yes ☐ No +- [ ] Sketch available? ☐ Yes ☐ No + +**Liability Signal Checkpoints:** +- Rear-end collision? (usually liable) +- Lane change involved? +- Reversing involved? +- Right-of-way violation? +- Red light/stop sign run? +- Following distance violation? + +**Bob's Perspective:** +- Bob may struggle to recall exact speeds — accept estimates +- Bob may describe events out of chronological order — gently clarify sequence +- Reassure Bob that details are being documented + +--- + +### Scene 6: Driver Details & Coverage Scope +**Title:** "Driver License and Coverage Check" + +**Context:** +Verify that the driver was legally entitled to drive and falls within the policy's driver scope. + +**Key Questions Alice Should Ask:** +- Do you have a valid driver's license? +- What is your license number and expiry date? +- What class is your license? (B, C1, etc.) +- How long have you held your license? +- Were you under the influence of alcohol or drugs at the time? +- Did you take a breathalyzer test? +- Any medication that might affect driving? +- Any health conditions that affect your ability to drive? + +**Information to Capture:** +- [ ] License number: __________ +- [ ] License expiry: __ / __ / ____ +- [ ] License class: __________ +- [ ] Held since: __ / __ / ____ +- [ ] Valid at time of loss? ☐ Yes ☐ No ☐ Expired +- [ ] Alcohol/drugs involved? ☐ No ☐ Yes (specify) +- [ ] Breathalyzer test? ☐ Not done ☐ Negative ☐ Positive +- [ ] Medications that might affect driving? ☐ No ☐ Yes (list) +- [ ] Health limitations? ☐ No ☐ Yes (describe) + +**Policy Scope Verification:** +- Check: Is this driver listed on the policy? +- Check: Is there an age restriction on the policy? +- Check: If occasional driver, is surcharge applied? +- Check: Any exclusions for this driver? + +**Red Flags:** +- License expired on date of loss +- Driver under minimum age for driving class +- Alcohol/drug involvement +- Hit-and-run situation (immediately triggers investigation) +- DUI or criminal charges pending + +--- + +### Scene 7: Other Party / Third Party Details +**Title:** "Information About the Other Vehicle/Party" + +**Context:** +If this was a collision with another vehicle or party property, gather their identification and insurance details. + +**Key Questions Alice Should Ask:** +- Was anyone else involved? +- Can you provide the other driver's name and contact? +- Do you have their vehicle registration? +- License plate and VIN of their vehicle? +- Make, model, and color of their vehicle? +- Do you have their insurance information? +- What is their insurance company and policy number? +- Is the registered owner the same as the driver? +- Did the other party admit fault? +- Have they already contacted their insurer? +- Have they sent a letter or lawyer involved? + +**Information to Capture:** +- [ ] Other party name: __________ +- [ ] Phone: __________ +- [ ] Email: __________ +- [ ] Address: __________ +- [ ] License plate: __________ +- [ ] VIN: __________ +- [ ] Vehicle make/model/color: __________ +- [ ] Insurance company: __________ +- [ ] Policy number: __________ +- [ ] Is owner same as driver? ☐ Yes ☐ No +- [ ] Other party admits fault? ☐ Unknown ☐ Yes ☐ No +- [ ] Letter received? ☐ No ☐ Yes (date: __) +- [ ] Lawyer involved? ☐ No ☐ Yes (name/firm: __) +- [ ] Already claimed with their insurer? ☐ No ☐ Yes + +**Recourse Indicators:** +- Confirmed other-party insurer = potential for direct recourse +- Other party's liability clear = direct settlement opportunity + +--- + +### Scene 8: Police Report & Authorities +**Title:** "Police Report and Authorities" + +**Context:** +Determine if police were involved and gather reference information. + +**Key Questions Alice Should Ask:** +- Was the police called to the scene? +- Did police attend? +- What police station/department? +- Do you have a police report number or reference? +- When can you obtain a copy of the police report? +- Are there any fines issued? +- Are any criminal charges possible? + +**Information to Capture:** +- [ ] Police attended? ☐ Yes ☐ No +- [ ] Police station: __________ +- [ ] Report number: __________ +- [ ] Reference: __________ +- [ ] Fines issued? ☐ No ☐ Yes (amount: __, cite: __) +- [ ] Criminal charges? ☐ No ☐ Pending ☐ Filed (charges: __) +- [ ] Administrative offense? ☐ No ☐ Yes (type: __) +- [ ] Report available for claim file? ☐ No ☐ Yes (when: __) + +**Flag for Investigation:** +- If criminal charges or hit-and-run: Immediate SIU referral +- If dangerous driving charges: Coverage issue possible + +--- + +### Scene 9: Witnesses +**Title:** "Witnesses and Passengers" + +**Context:** +Identify any independent witnesses who saw the accident. + +**Key Questions Alice Should Ask:** +- Were there any independent witnesses? +- Did anyone stop to help? +- What are their names and contact information? +- Were there passengers in your vehicle? +- What are their names and contact information? +- Any passengers in the other vehicle? + +**Information to Capture:** + +**Independent Witnesses:** +- Witness 1: Name __________, Phone: __________, Brief statement: __________ +- Witness 2: Name __________, Phone: __________, Brief statement: __________ +- Witness 3: (additional as needed) + +**Passengers - Your Vehicle:** +- Passenger 1: Name __________, Relation: __________, Phone: __________ +- Passenger 2: Name __________, Relation: __________, Phone: __________ + +**Passengers - Other Vehicle:** +- Name: __________, Phone: __________, Any injuries? ☐ Yes ☐ No + +**Investigation Note:** +- Independent (non-party) witnesses are highly valuable +- Collect statements if possible before memories fade + +--- + +### Scene 10: Injuries & Medical +**Title:** "Injuries and Medical Treatment" + +**Context:** +Determine if there are any personal injuries requiring medical attention. + +**Key Questions Alice Should Ask:** +- Is anyone injured? +- Yourself? +- Passengers? +- Other party? +- Anyone else (pedestrians, cyclists)? +- What type of injuries? (describe) +- Did anyone receive treatment? +- Hospital or outpatient care? +- Doctor or hospital name? +- Any anticipated time off work? +- Health insurance carrier for injured parties? + +**Information to Capture:** +- [ ] Injuries? ☐ No ☐ Yes + +**If Yes - Injured Parties:** +- Party 1: __________ | Relation: __________ | Injury: __________ +- Party 2: __________ | Relation: __________ | Injury: __________ + +**Treatment Received:** +- [ ] No treatment +- [ ] Outpatient (clinic/doctor) - Where: __________, When: __________ +- [ ] Inpatient (hospital) - Hospital: __________, Duration: __________ + +**Health Insurance for Injured:** +- Injured party: __________ | Health insurer: __________ + +**Time Off Work:** +- Anticipated? ☐ No ☐ Yes | Duration: __________ + +**Medical Investigation Notes:** +- Consider formal medical assessment if significant injuries +- Coordinate with health insurer for subrogation (if applicable in jurisdiction) + +--- + +### Scene 11: Property Damage Assessment +**Title:** "Vehicle Damage and Condition" + +**Context:** +Assess the damage to Bob's vehicle and determine repairability and current location. + +**Key Questions Alice Should Ask:** +- Can you describe the damage to your vehicle? +- Where is the damage located? (front, rear, side, etc.) +- Can the vehicle be driven? +- Where is the vehicle currently? +- Was it towed? By whom and to where? +- Do you have damage photos? +- Has any prior damage to this area that wasn't repaired? + +**Information to Capture:** +- [ ] Damage location: ☐ Front ☐ Rear ☐ Left side ☐ Right side ☐ Top ☐ Interior ☐ Multiple +- [ ] Damage description: __________ +- [ ] Drivable? ☐ Yes ☐ No +- [ ] Current location: __________ +- [ ] Was vehicle towed? ☐ No ☐ Yes +- [ ] Tow company: __________ | Location towed to: __________ +- [ ] Photos available? ☐ No ☐ Yes (how many: __) +- [ ] Prior damage in area? ☐ No ☐ Yes (pre-existing) + +**Third-Party Property:** +- [ ] Damage to other vehicle? ☐ No ☐ Yes (describe: __) +- [ ] Damage to property? (guardrail, fence, sign, building) ☐ No ☐ Yes + - Owner: __________ | Estimated damage: __________ + +--- + +### Scene 12: Coverage Scope & Exclusions Check +**Title:** "Coverage Verification and Policy Exclusions" + +**Context:** +Verify coverage applies and identify any potential exclusions. + +**Key Questions Alice Should Ask:** +- Was the vehicle being used for private personal driving? +- Any commercial use (rideshare, delivery)? +- Any paid passenger transport? +- Was the vehicle modified in any way? +- Are emissions tests current? +- Vehicle registration current? +- Any seasonal plate restrictions? +- Racing or track events involved? + +**Information to Capture:** +- [ ] Use outside scope? ☐ No ☐ Yes (describe: __) +- [ ] Commercial use? ☐ No ☐ Yes +- [ ] Paid passengers? ☐ No ☐ Yes +- [ ] Vehicle modifications? ☐ No ☐ Yes (what: __) +- [ ] Emissions test (HU/AU) current? ☐ Yes ☐ No ☐ Unknown +- [ ] Vehicle registration current? ☐ Yes ☐ No +- [ ] Seasonal plate restrictions? ☐ No applicable ☐ Yes (windows: __) +- [ ] Racing/track event? ☐ No ☐ Yes + +**Exclusion Red Flags:** +- Out-of-scope use = coverage denial likely +- Gross negligence clauses if reckless driving +- Telematics conditions if device was supposed active +- Geographic restrictions if loss outside coverage area + +--- + +### Scene 13: Fraud Signal Assessment +**Title:** "Routine Verification Questions" + +**Context:** +Ask routine questions that help screen for fraud (not accusatory). + +**Key Questions Alice Should Ask:** +- When exactly did you first notice the damage? +- When did you report it to us? +- Do you know the other party involved? +- Are you related to the other party? +- Have you had any similar claims in the last 24 months? +- Is your vehicle currently for sale? +- Any recent owner or major buyer/seller activity? + +**Information to Capture:** +- [ ] Damage noticed: Date __ / __ / ____, Time __:__ +- [ ] Reported to us: Date __ / __ / ____, Time __:__ +- [ ] Known party? ☐ No ☐ Yes (relationship: __) +- [ ] Related to party? ☐ No ☐ Yes (relationship: __) +- [ ] Similar claims last 24 months? ☐ No ☐ Yes (number: __) +- [ ] Vehicle for sale? ☐ No ☐ Yes (listing date: __) +- [ ] Recent transfer? ☐ No ☐ Yes (date: __) + +**Fraud Indicators (flag for SIU review if multiple present):** +- Significant delay before reporting +- Parties known to each other +- Multiple similar claims +- Vehicle listed for sale around loss date +- Damage seems disproportionate to accident description + +--- + +### Scene 14: Settlement Preferences & Next Steps +**Title:** "Immediate Needs and Next Steps" + +**Context:** +Understand Bob's immediate needs and set clear expectations for claim handling. + +**Key Questions Alice Should Ask:** +- Do you need a rental vehicle while yours is repaired? +- Do you have a preferred repair shop? +- Any questions about your coverage? +- What is the best way to reach you? +- Can you provide photos/documents? +- When is a good time to follow up? + +**Information to Capture:** +- [ ] Rental car needed? ☐ No ☐ Yes +- [ ] Preferred repair shop? ☐ No ☐ Yes (name: __) +- [ ] Lawyer involved? ☐ No ☐ Yes (name/firm: __) +- [ ] Preferred contact method: ☐ Phone ☐ Email ☐ Mail +- [ ] Best phone number: __________ +- [ ] Best email: __________ +- [ ] Best time to contact: __________ +- [ ] Can provide photos? ☐ No ☐ Yes (when: __) +- [ ] Can provide statement? ☐ No ☐ Yes (when: __) +- [ ] European Accident Statement? ☐ Not applicable ☐ Has it ☐ Can get from other party + +**Next Steps to Communicate:** +- "Here's what happens next: ..." +- Estimated timeline for claim decision +- Who will be handling the claim +- How they will be updated +- What documents/information is needed + +--- + +## Summary: Question Priority Tiers + +### Tier 1 - **Mandatory (Must Have)** +Must collect before claim can be processed: +1. Policy number (or VIN/license plate) +2. Claimant name and contact +3. Date and location of loss +4. Type of loss +5. Who was driving +6. Other party details (if applicable) +7. Police report info (if obtained) + +### Tier 2 - **Critical (Should Have)** +Needed for coverage determination: +1. Driver license status +2. Is driver in policy scope? +3. Weather/road conditions +4. Accident narration (how it happened) +5. Injuries (if any) +6. Damage scope +7. Coverage exclusion screening + +### Tier 3 - **Important (Nice to Have)** +Valuable for investigation: +1. Witness information +2. Photos +3. Police report +4. Prior damage +5. Fraud signal check + +--- + +## Conversation Flow Tips for Alice + +**Empathy-First Approach:** +- Start each conversation acknowledging Bob's stress +- Use reassuring tone +- Don't rush through information gathering +- Summarize details back to Bob for confirmation + +**Question Sequencing:** +1. Start with immediate safety/emergencies +2. Move to identity/policy verification (builds trust) +3. Then gather accident facts (what, when, where) +4. Include third-party details (other vehicle) +5. Check coverage scope +6. Discuss immediate needs (rental, repair authorization) +7. Explain next steps clearly + +**Handling Bob's Imperfect Recall:** +- "That's okay if you don't remember exactly..." +- Accept approximations (speeds, times) +- Ask clarifying follow-up questions gently +- Confirm details before moving on +- Don't challenge Bob's account unless red flags present + +**Plain Language Rule:** +- Never use: "Vollkasko", "Kasko", "Fahrerkreis", "Obliegenheit" +- Instead say: "comprehensive coverage", "driver scope", "your responsibilities" +- Always explain insurance terms in simple language + +--- + +## Creating Multiple Scenes in DialogLab + +To structure this FNOL workflow across multiple conversation nodes in DialogLab: + +1. **Create separate Scene nodes** for each major topic area (14 scenes recommended) +2. **Connect scenes** in a logical sequence +3. **Set conversation parameters:** + - Alice as Insurer (use updated template) + - Bob as Claimant (use updated template) + - Each scene focuses on one topic area + - Alice asks questions from the checklist + - Bob responds with details of his accident + +4. **Use Node Inspector** to: + - Set `interactionPattern: "supportive"` (Alice) vs `"receptive"` (Bob) + - Set `turns: 4-6` per scene (enough for Q&A exchange) + - Set `derailerMode: false` (keep on-topic for FNOL workflow) + +5. **Scene branching** based on responses: + - If multi-vehicle collision → include Scene 7 (Other Party Details) + - If injuries reported → include Scene 10 (Medical) + - If police involved → include Scene 8 (Police) + - If no injuries/police → skip those scenes + +--- + +## Policy Document Data to Reference + +Alice should mentally reference (or have access to in real system): +- [ ] Policy effective date +- [ ] Coverage type (Vollkasko/Teilkasko/Haftpflicht) +- [ ] Deductible amounts +- [ ] Included drivers +- [ ] Age restrictions +- [ ] Vehicle details on file +- [ ] Geographic coverage area +- [ ] Any active policy exclusions or endorsements + +--- + +This workflow ensures comprehensive FNOL intake while maintaining empathy, clear communication, and systematic information gathering. diff --git a/GEMINI_TTS_SETUP.md b/GEMINI_TTS_SETUP.md new file mode 100644 index 0000000..f9097fa --- /dev/null +++ b/GEMINI_TTS_SETUP.md @@ -0,0 +1,83 @@ +# Gemini TTS Setup for VOX LAB + +Configure your Gemini API key in `.env` using a placeholder format: +``` +GEMINI_API_KEY=your-gemini-api-key-here +``` + +## TTS Provider Support + +VOX LAB now supports **three TTS providers**: + +### 1. **Google Cloud TTS** (default) +- Free tier available +- Large voice selection +- Most stable + +### 2. **Azure Speech Services** +- Enterprise-grade +- Real-time streaming +- Multiple voice styles + +### 3. **Google Gemini TTS** (NEW!) +- Modern LLM-based synthesis +- Natural, expressive output +- Lower latency for research + +## Switching TTS Providers + +You can switch providers via environment variable or API: + +### Environment Variable +```bash +# In .env file +TTS_PROVIDER=gemini # or 'google' or 'azure' +GEMINI_API_KEY=your-api-key +``` + +### API Call (Runtime) +```bash +curl -X POST http://localhost:3010/api/set-tts-provider \ + -H "Content-Type: application/json" \ + -d '{"provider": "gemini"}' +``` + +## Implementation Notes + +The Gemini TTS integration is ready for: +- ✅ Text-to-speech synthesis +- ✅ Multiple voice profiles +- ✅ Streaming audio output +- ✅ Format conversion (WAV, MP3) + +### Example: Using Gemini TTS + +```javascript +// Client-side or API call +const response = await fetch('/api/tts', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + text: "Hello, this is spoken with Gemini TTS", + voice: { name: "en-US-female" }, + audioConfig: { audioEncoding: "MP3" } + }) +}); + +const audio = await response.arrayBuffer(); +``` + +## Next Steps + +1. **Test Gemini TTS**: Run a conversation in VOX LAB with TTS provider set to "gemini" +2. **Regenerate API Key**: For security, [regenerate your Gemini API key](https://console.cloud.google.com/apis/credentials) +3. **Monitor Costs**: Gemini TTS charges per request - monitor your Google Cloud usage + +## Troubleshooting + +If you get "content filter" errors, it means the LLM response violated Azure's content policy. This is separate from TTS and happens during text generation before audio synthesis. + +**Solution**: The prompt content has been simplified to reduce false positives. If issues persist: +1. Check agent.js prompts for problematic content +2. Adjust conversation topics to avoid sensitive content +3. Switch LLM providers if needed diff --git a/LICENSE b/LICENSE deleted file mode 100644 index ca7dc28..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2025 Erzhen Hu - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md index cc8c6e3..bf08d4f 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,22 @@ -# DialogLab +# VOX LAB -DialogLab teaser figure +VOX LAB teaser figure -[Paper](https://dl.acm.org/doi/10.1145/3746059.3747696) | [30s Preview](https://www.youtube.com/watch?v=AdvYhP8A51M) | [Video Figure](https://www.youtube.com/watch?v=U2Ag_Ktobzw) | [Web Demo](https://chatlab.3dvar.com/) +**VOX LAB** is an authoring environment for configuring and orchestrating multi-agent conversations with animated 3D avatars. Built with React, Vite, and Express, it helps researchers, designers, and developers create, visualize, and evaluate complex agent-based dialog systems. -**DialogLab** is an authoring tool for configuring and orchestrating multi-agent conversations with animated 3D avatars. Built with React, Vite, and Express, it enables researchers, designers, and developers to create, visualize, and evaluate complex agent-based dialogue systems. +## Overview + +VOX LAB combines a visual authoring interface, avatar-driven playback, and runtime model integrations so teams can rapidly prototype and test conversational experiences. ## Features -- **Visual Conversation Design**: Configure multi-agent conversations with an intuitive node-based editor -- **3D Avatar Integration**: Animate conversations using Ready Player Me avatars with synchronized speech -- **Multiple LLM Support**: Compatible with OpenAI GPT and Google Gemini models -- **Scene Management**: Create and manage multiple conversation scenarios -- **Real-time Preview**: Test and iterate on conversations in real-time -- **Verification Tools**: Analyze conversation metrics and quality -- **Experience Mode**: Present finished conversations in a polished viewer +- **Visual conversation design**: Build multi-agent conversations with an intuitive node editor. +- **3D avatar integration**: Animate conversations with Ready Player Me avatars and synchronized speech. +- **Multiple LLM providers**: Use either OpenAI or Google Gemini models. +- **Scene management**: Create and organize multiple conversation scenarios. +- **Real-time preview**: Test and iterate quickly while authoring. +- **Verification tooling**: Inspect conversation quality and metrics. +- **Experience mode**: Present completed conversations in a polished viewer. ## Prerequisites @@ -24,7 +26,7 @@ ## Repository Structure ``` -DialogLab/ +VOX-LAB/ ├── client/ # React UI (Vite) - Dev server on port 5173 │ ├── src/ # React components and application logic │ └── public/ # Static assets (avatars, libraries) @@ -47,9 +49,24 @@ cd ../server npm install ``` +Alternatively, from the repository root: + +```bash +npm --prefix client install +npm --prefix server install +``` + ### 2. Configure Environment Variables -Create a `.env` file in the `server/` directory with your API keys: +Copy the template and fill in your own values (never commit secrets): + +```bash +cp server/.env.example server/.env +``` + +Then edit `server/.env` and set your provider keys and related config using placeholders only. + +Example: ```env # Core Configuration @@ -66,10 +83,10 @@ DEFAULT_GEMINI_MODEL=gemini-2.0-flash # Text-to-Speech (optional - for avatar speech synthesis) TTS_API_KEY=your-google-tts-api-key-here -TTS_ENDPOINT=https://eu-texttospeech.googleapis.com/v1beta1/text:synthesize +TTS_ENDPOINT=your-tts-endpoint-here ``` -**Note**: You need at least one LLM provider API key (OpenAI or Gemini) to run conversations. +You must provide at least one LLM provider API key (OpenAI or Gemini) to run conversations. ### 3. Start the Server @@ -78,7 +95,7 @@ cd server node server.js ``` -The server will start at `http://localhost:3010`. You should see a message confirming the server is running. +The server starts on `http://localhost:3010`. ### 4. Start the Client @@ -89,37 +106,15 @@ cd client npm run dev ``` -Open your browser and navigate to `http://localhost:5173`. +Open `http://localhost:5173` in your browser. -## Third-Party Components & Licenses +## Environment Variables (Server) -### TalkingHead (MIT License) -Portions of this project's code are adapted from the open-source project [TalkingHead](https://github.com/met4citizen/TalkingHead), © 2024 Mika Suominen (met4citizen), licensed under the MIT License. - -**Files**: `client/public/libs/talkinghead.mjs` - -### Three.js (MIT License) -This project uses [Three.js](https://threejs.org/) and its example modules, © 2010–present Three.js authors, licensed under the MIT License. - -**License**: [Three.js LICENSE](https://github.com/mrdoob/three.js/blob/dev/LICENSE) - -### Ready Player Me Avatars (Custom License) -Example avatar files (`client/public/assets/*.glb`) were created using [Ready Player Me](https://readyplayer.me/) and are subject to [Ready Player Me's Terms of Use](https://readyplayer.me/terms-of-use). - -**Note**: These assets are provided for demonstration purposes only and are **not covered** by this project's open-source license. For production use, you must obtain your own avatars or comply with Ready Player Me's licensing terms. - -## Citation - -If you use DialogLab in your research, please cite our UIST 2025 paper: - -```bibtex -@inproceedings{dialoglab2025, -author = {Hu, Erzhen and Chen, Yanhe and Li, Mingyi and Phadnis, Vrushank and Xu, Pingmei and Qian, Xun and Olwal, Alex and Kim, David and Heo, Seongkook and Du, Ruofei}, - title = {DialogLab: Configuring and Orchestrating Multi-Agent Conversations}, - booktitle = {Proceedings of the 38th Annual ACM Symposium on User Interface Software and Technology (UIST '25)}, - year = {2025}, - publisher = {Association for Computing Machinery}, - address = {New York, NY, USA}, - doi = {10.1145/3746059.3747696} -} -``` +- `NODE_ENV`: Runtime environment (`development` by default). +- `GEMINI_API_KEY`: Google Gemini API key. +- `API_KEY_LLM`: OpenAI API key. +- `DEFAULT_LLM_PROVIDER`: Default provider (`gemini` or `openai`). +- `DEFAULT_OPENAI_MODEL`: Default OpenAI model. +- `DEFAULT_GEMINI_MODEL`: Default Gemini model. +- `TTS_API_KEY`: Google TTS API key (optional). +- `TTS_ENDPOINT`: Google TTS endpoint (optional). diff --git a/client/index.html b/client/index.html index fab36fd..e8a6add 100644 --- a/client/index.html +++ b/client/index.html @@ -3,8 +3,8 @@ - - Talking Head React App + + VOX LAB diff --git a/client/package-lock.json b/client/package-lock.json index 5ca0a01..4d15c0f 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -24,9 +24,7 @@ "axios": "^1.7.9", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "html2canvas": "^1.4.1", "lucide-react": "^0.475.0", - "rc-slider": "^11.1.8", "react": "^19.0.0", "react-dom": "^19.0.0", "react-draggable": "^4.4.6", @@ -2503,16 +2501,6 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, - "node_modules/@types/node": { - "version": "22.13.4", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.13.4.tgz", - "integrity": "sha512-ywP2X0DYtX3y08eFVx5fNIw7/uIv8hYUKgXoK8oayJlLnKcRfEYCxWMVE1XagUdVtCJlZT1AU4LXEABW+L1Peg==", - "optional": true, - "peer": true, - "dependencies": { - "undici-types": "~6.20.0" - } - }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -3126,14 +3114,6 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, - "node_modules/base64-arraybuffer": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", - "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -3366,11 +3346,6 @@ "url": "https://polar.sh/cva" } }, - "node_modules/classnames": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", - "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==" - }, "node_modules/cli-cursor": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", @@ -3522,14 +3497,6 @@ "node": ">= 8" } }, - "node_modules/css-line-break": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/css-line-break/-/css-line-break-2.1.0.tgz", - "integrity": "sha512-FHcKFCZcAha3LwfVBhCQbW2nCNbkZXn7KVUJcsT5/P8YmfsVja0FMPJr0B903j/E69HUphKiV9iQArX8SDYA4w==", - "dependencies": { - "utrie": "^1.0.2" - } - }, "node_modules/csstype": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", @@ -4910,18 +4877,6 @@ "react-is": "^16.7.0" } }, - "node_modules/html2canvas": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/html2canvas/-/html2canvas-1.4.1.tgz", - "integrity": "sha512-fPU6BHNpsyIhr8yyMpTLLxAbkaK8ArIBcmZIRiBLiDhjeqvXolaEmDGmELFuX9I4xDcaKKcJl+TKZLqruBbmWA==", - "dependencies": { - "css-line-break": "^2.1.0", - "text-segmentation": "^1.0.3" - }, - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/human-signals": { "version": "4.3.1", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-4.3.1.tgz", @@ -6472,41 +6427,6 @@ ], "license": "MIT" }, - "node_modules/rc-slider": { - "version": "11.1.8", - "resolved": "https://registry.npmjs.org/rc-slider/-/rc-slider-11.1.8.tgz", - "integrity": "sha512-2gg/72YFSpKP+Ja5AjC5DPL1YnV8DEITDQrcc1eASrUYjl0esptaBVJBh5nLTXCCp15eD8EuGjwezVGSHhs9tQ==", - "dependencies": { - "@babel/runtime": "^7.10.1", - "classnames": "^2.2.5", - "rc-util": "^5.36.0" - }, - "engines": { - "node": ">=8.x" - }, - "peerDependencies": { - "react": ">=16.9.0", - "react-dom": ">=16.9.0" - } - }, - "node_modules/rc-util": { - "version": "5.44.4", - "resolved": "https://registry.npmjs.org/rc-util/-/rc-util-5.44.4.tgz", - "integrity": "sha512-resueRJzmHG9Q6rI/DfK6Kdv9/Lfls05vzMs1Sk3M2P+3cJa+MakaZyWY8IPfehVuhPJFKrIY1IK4GqbiaiY5w==", - "dependencies": { - "@babel/runtime": "^7.18.3", - "react-is": "^18.2.0" - }, - "peerDependencies": { - "react": ">=16.9.0", - "react-dom": ">=16.9.0" - } - }, - "node_modules/rc-util/node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==" - }, "node_modules/react": { "version": "19.0.0", "resolved": "https://registry.npmjs.org/react/-/react-19.0.0.tgz", @@ -7466,14 +7386,6 @@ "devOptional": true, "license": "MIT" }, - "node_modules/text-segmentation": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/text-segmentation/-/text-segmentation-1.0.3.tgz", - "integrity": "sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw==", - "dependencies": { - "utrie": "^1.0.2" - } - }, "node_modules/three": { "version": "0.170.0", "resolved": "https://registry.npmjs.org/three/-/three-0.170.0.tgz", @@ -7635,13 +7547,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/undici-types": { - "version": "6.20.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", - "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", - "optional": true, - "peer": true - }, "node_modules/universalify": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", @@ -7752,14 +7657,6 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, - "node_modules/utrie": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/utrie/-/utrie-1.0.2.tgz", - "integrity": "sha512-1MLa5ouZiOmQzUbjbu9VmjLzn1QLXBhwpUa7kdLUQK+KQ5KA9I1vk5U4YHe/X2Ch7PYnJfWuWT+VbuxbGwljhw==", - "dependencies": { - "base64-arraybuffer": "^1.0.2" - } - }, "node_modules/victory-vendor": { "version": "36.9.2", "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-36.9.2.tgz", @@ -8030,19 +7927,6 @@ "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", "dev": true }, - "node_modules/yaml": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", - "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", - "optional": true, - "peer": true, - "bin": { - "yaml": "bin.mjs" - }, - "engines": { - "node": ">= 14" - } - }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/client/package.json b/client/package.json index befc409..d8525bc 100644 --- a/client/package.json +++ b/client/package.json @@ -30,9 +30,7 @@ "axios": "^1.7.9", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", - "html2canvas": "^1.4.1", "lucide-react": "^0.475.0", - "rc-slider": "^11.1.8", "react": "^19.0.0", "react-dom": "^19.0.0", "react-draggable": "^4.4.6", diff --git a/client/public/assets/avatar2.glb b/client/public/assets/avatar2.glb deleted file mode 100644 index d687f1b..0000000 Binary files a/client/public/assets/avatar2.glb and /dev/null differ diff --git a/client/public/assets/avatar3.glb b/client/public/assets/avatar3.glb deleted file mode 100644 index b59dcaf..0000000 Binary files a/client/public/assets/avatar3.glb and /dev/null differ diff --git a/client/public/assets/male-avatar6.glb b/client/public/assets/male-avatar6.glb deleted file mode 100644 index cafa09d..0000000 Binary files a/client/public/assets/male-avatar6.glb and /dev/null differ diff --git a/client/public/assets/p-f-a1.glb b/client/public/assets/p-f-a1.glb deleted file mode 100644 index 8551197..0000000 Binary files a/client/public/assets/p-f-a1.glb and /dev/null differ diff --git a/client/public/assets/p-f-a2.glb b/client/public/assets/p-f-a2.glb deleted file mode 100644 index e6db738..0000000 Binary files a/client/public/assets/p-f-a2.glb and /dev/null differ diff --git a/client/public/assets/p-m-a3.glb b/client/public/assets/p-m-a3.glb deleted file mode 100644 index ddf1076..0000000 Binary files a/client/public/assets/p-m-a3.glb and /dev/null differ diff --git a/client/public/assets/p-m-a4.glb b/client/public/assets/p-m-a4.glb deleted file mode 100644 index 95209c4..0000000 Binary files a/client/public/assets/p-m-a4.glb and /dev/null differ diff --git a/client/public/libs/talkinghead.mjs b/client/public/libs/talkinghead.mjs deleted file mode 100644 index 8b5739c..0000000 --- a/client/public/libs/talkinghead.mjs +++ /dev/null @@ -1,3994 +0,0 @@ -/** -* MIT License -* -* Copyright (c) 2024 Mika Suominen -* -* Permission is hereby granted, free of charge, to any person obtaining a copy -* of this software and associated documentation files (the "Software"), to deal -* in the Software without restriction, including without limitation the rights -* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -* copies of the Software, and to permit persons to whom the Software is -* furnished to do so, subject to the following conditions: -* -* The above copyright notice and this permission notice shall be included in all -* copies or substantial portions of the Software. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -import * as THREE from 'three'; -import { OrbitControls } from 'three/addons/controls/OrbitControls.js'; -import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js'; -import { FBXLoader } from 'three/addons/loaders/FBXLoader.js'; -import { RoomEnvironment } from 'three/addons/environments/RoomEnvironment.js'; -import Stats from 'three/addons/libs/stats.module.js'; - -import{ DynamicBones } from './dynamicbones.mjs'; - -// Temporary objects for animation loop -const q = new THREE.Quaternion(); -const e = new THREE.Euler(); -const v = new THREE.Vector3(); -const w = new THREE.Vector3(); -const box = new THREE.Box3(); -const m = new THREE.Matrix4(); -const minv = new THREE.Matrix4(); -const origin = new THREE.Vector3(); -const forward = new THREE.Vector3(0, 0, 1); -const axisx = new THREE.Vector3(1, 0, 0); -const axisy = new THREE.Vector3(0, 1, 0); -const axisz = new THREE.Vector3(0, 0, 1); - -class TalkingHead { - - /** - * Avatar. - * @typedef {Object} Avatar - * @property {string} url URL for the GLB file - * @property {string} [body] Body form 'M' or 'F' - * @property {string} [lipsyncLang] Lip-sync language, e.g. 'fi', 'en' - * @property {string} [ttsLang] Text-to-speech language, e.g. "fi-FI" - * @property {voice} [ttsVoice] Voice name. - * @property {numeric} [ttsRate] Voice rate. - * @property {numeric} [ttsPitch] Voice pitch. - * @property {numeric} [ttsVolume] Voice volume. - * @property {string} [avatarMood] Initial mood. - * @property {boolean} [avatarMute] If true, muted. - * @property {numeric} [avatarIdleEyeContact] Eye contact while idle [0,1] - * @property {numeric} [avatarIdleHeadMove] Eye contact while idle [0,1] - * @property {numeric} [avatarSpeakingEyeContact] Eye contact while speaking [0,1] - * @property {numeric} [avatarSpeakingHeadMove] Eye contact while speaking [0,1] - * @property {Object[]} [modelDynamicBones] Config for Dynamic Bones feature - */ - - /** - * Loading progress. - * @callback progressfn - * @param {string} url URL of the resource - * @param {Object} event Progress event - * @param {boolean} event.lengthComputable If false, total is not known - * @param {number} event.loaded Number of loaded items - * @param {number} event.total Number of total items - */ - - /** - * Callback when new subtitles have been written to the DOM node. - * @callback subtitlesfn - * @param {Object} node DOM node - */ - - /** - * Callback when the speech queue processes this marker item. - * @callback markerfn - */ - - /** - * Audio object. - * @typedef {Object} Audio - * @property {ArrayBuffer|ArrayBuffer[]} audio Audio buffer or array of buffers - * @property {string[]} words Words - * @property {number[]} wtimes Starting times of words - * @property {number[]} wdurations Durations of words - * @property {string[]} [visemes] Oculus lip-sync viseme IDs - * @property {number[]} [vtimes] Starting times of visemes - * @property {number[]} [vdurations] Durations of visemes - * @property {string[]} [markers] Timed callback functions - * @property {number[]} [mtimes] Starting times of markers - */ - - /** - * Lip-sync object. - * @typedef {Object} Lipsync - * @property {string[]} visemes Oculus lip-sync visemes - * @property {number[]} times Starting times in relative units - * @property {number[]} durations Durations in relative units - */ - - /** - * @constructor - * @param {Object} node DOM element of the avatar - * @param {Object} [opt=null] Global/default options - */ - constructor(node, opt = null ) { - this.nodeAvatar = node; - this.opt = { - jwtGet: null, // Function to get JSON Web Token - ttsEndpoint: null, - ttsApikey: null, - ttsTrimStart: 0, - ttsTrimEnd: 400, - ttsLang: "fi-FI", - ttsVoice: "fi-FI-Standard-A", - ttsRate: 1, - ttsPitch: 0, - ttsVolume: 0, - mixerGainSpeech: null, - mixerGainBackground: null, - lipsyncLang: 'fi', - lipsyncModules: ['fi','en','lt'], - pcmSampleRate: 22050, - modelRoot: "Armature", - modelPixelRatio: 1, - modelFPS: 30, - modelMovementFactor: 1, - cameraView: 'full', - cameraDistance: 0, - cameraX: 0, - cameraY: 0, - cameraRotateX: 0, - cameraRotateY: 0, - cameraRotateEnable: true, - cameraPanEnable: false, - cameraZoomEnable: false, - lightAmbientColor: 0xffffff, - lightAmbientIntensity: 2, - lightDirectColor: 0x8888aa, - lightDirectIntensity: 30, - lightDirectPhi: 1, - lightDirectTheta: 2, - lightSpotIntensity: 0, - lightSpotColor: 0x3388ff, - lightSpotPhi: 0.1, - lightSpotTheta: 4, - lightSpotDispersion: 1, - avatarMood: "neutral", - avatarMute: false, - avatarIdleEyeContact: 0.2, - avatarIdleHeadMove: 0.5, - avatarSpeakingEyeContact: 0.5, - avatarSpeakingHeadMove: 0.5, - avatarIgnoreCamera: false, - listeningSilenceThresholdLevel: 40, - listeningSilenceThresholdMs: 2000, - listeningSilenceDurationMax: 10000, - listeningActiveThresholdLevel: 75, - listeningActiveThresholdMs: 300, - listeningActiveDurationMax: 240000, - statsNode: null, - statsStyle: null - }; - Object.assign( this.opt, opt || {} ); - - // Statistics - if ( this.opt.statsNode ) { - this.stats = new Stats(); - if ( this.opt.statsStyle ) { - this.stats.dom.style.cssText = this.opt.statsStyle; - } - this.opt.statsNode.appendChild( this.stats.dom ); - } - - // Pose templates - // NOTE: The body weight on each pose should be on left foot - // for most natural result. - this.poseTemplates = { - 'side': { - standing: true, - props: { - 'Hips.position':{x:0, y:1, z:0}, 'Hips.rotation':{x:-0.003, y:-0.017, z:0.1}, 'Spine.rotation':{x:-0.103, y:-0.002, z:-0.063}, 'Spine1.rotation':{x:0.042, y:-0.02, z:-0.069}, 'Spine2.rotation':{x:0.131, y:-0.012, z:-0.065}, 'Neck.rotation':{x:0.027, y:0.006, z:0}, 'Head.rotation':{x:0.077, y:-0.065, z:0}, 'LeftShoulder.rotation':{x:1.599, y:0.084, z:-1.77}, 'LeftArm.rotation':{x:1.364, y:0.052, z:-0.044}, 'LeftForeArm.rotation':{x:0.002, y:-0.007, z:0.331}, 'LeftHand.rotation':{x:0.104, y:-0.067, z:-0.174}, 'LeftHandThumb1.rotation':{x:0.231, y:0.258, z:0.355}, 'LeftHandThumb2.rotation':{x:-0.106, y:-0.339, z:-0.454}, 'LeftHandThumb3.rotation':{x:-0.02, y:-0.142, z:-0.004}, 'LeftHandIndex1.rotation':{x:0.148, y:0.032, z:-0.069}, 'LeftHandIndex2.rotation':{x:0.326, y:-0.049, z:-0.029}, 'LeftHandIndex3.rotation':{x:0.247, y:-0.053, z:-0.073}, 'LeftHandMiddle1.rotation':{x:0.238, y:-0.057, z:-0.089}, 'LeftHandMiddle2.rotation':{x:0.469, y:-0.036, z:-0.081}, 'LeftHandMiddle3.rotation':{x:0.206, y:-0.015, z:-0.017}, 'LeftHandRing1.rotation':{x:0.187, y:-0.118, z:-0.157}, 'LeftHandRing2.rotation':{x:0.579, y:0.02, z:-0.097}, 'LeftHandRing3.rotation':{x:0.272, y:0.021, z:-0.063}, 'LeftHandPinky1.rotation':{x:0.405, y:-0.182, z:-0.138}, 'LeftHandPinky2.rotation':{x:0.613, y:0.128, z:-0.144}, 'LeftHandPinky3.rotation':{x:0.268, y:0.094, z:-0.081}, 'RightShoulder.rotation':{x:1.541, y:0.192, z:1.775}, 'RightArm.rotation':{x:1.273, y:-0.352, z:-0.067}, 'RightForeArm.rotation':{x:-0.011, y:-0.031, z:-0.357}, 'RightHand.rotation':{x:-0.008, y:0.312, z:-0.028}, 'RightHandThumb1.rotation':{x:0.23, y:-0.258, z:-0.355}, 'RightHandThumb2.rotation':{x:-0.107, y:0.339, z:0.454}, 'RightHandThumb3.rotation':{x:-0.02, y:0.142, z:0.004}, 'RightHandIndex1.rotation':{x:0.148, y:-0.031, z:0.069}, 'RightHandIndex2.rotation':{x:0.326, y:0.049, z:0.029}, 'RightHandIndex3.rotation':{x:0.247, y:0.053, z:0.073}, 'RightHandMiddle1.rotation':{x:0.237, y:0.057, z:0.089}, 'RightHandMiddle2.rotation':{x:0.469, y:0.036, z:0.081}, 'RightHandMiddle3.rotation':{x:0.206, y:0.015, z:0.017}, 'RightHandRing1.rotation':{x:0.204, y:0.086, z:0.135}, 'RightHandRing2.rotation':{x:0.579, y:-0.02, z:0.098}, 'RightHandRing3.rotation':{x:0.272, y:-0.021, z:0.063}, 'RightHandPinky1.rotation':{x:0.404, y:0.182, z:0.137}, 'RightHandPinky2.rotation':{x:0.613, y:-0.128, z:0.144}, 'RightHandPinky3.rotation':{x:0.268, y:-0.094, z:0.081}, 'LeftUpLeg.rotation':{x:0.096, y:0.209, z:2.983}, 'LeftLeg.rotation':{x:-0.053, y:0.042, z:-0.017}, 'LeftFoot.rotation':{x:1.091, y:0.15, z:0.026}, 'LeftToeBase.rotation':{x:0.469, y:-0.07, z:-0.015}, 'RightUpLeg.rotation':{x:-0.307, y:-0.219, z:2.912}, 'RightLeg.rotation':{x:-0.359, y:0.164, z:0.015}, 'RightFoot.rotation':{x:1.035, y:0.11, z:0.005}, 'RightToeBase.rotation':{x:0.467, y:0.07, z:0.015} - } - }, - - 'hip':{ - standing: true, - props: { - 'Hips.position':{x:0,y:1,z:0}, 'Hips.rotation':{x:-0.036,y:0.09,z:0.135}, 'Spine.rotation':{x:0.076,y:-0.035,z:0.01}, 'Spine1.rotation':{x:-0.096,y:0.013,z:-0.094}, 'Spine2.rotation':{x:-0.014,y:0.002,z:-0.097}, 'Neck.rotation':{x:0.034,y:-0.051,z:-0.075}, 'Head.rotation':{x:0.298,y:-0.1,z:0.154}, 'LeftShoulder.rotation':{x:1.694,y:0.011,z:-1.68}, 'LeftArm.rotation':{x:1.343,y:0.177,z:-0.153}, 'LeftForeArm.rotation':{x:-0.049,y:0.134,z:0.351}, 'LeftHand.rotation':{x:0.057,y:-0.189,z:-0.026}, 'LeftHandThumb1.rotation':{x:0.368,y:-0.066,z:0.438}, 'LeftHandThumb2.rotation':{x:-0.156,y:0.029,z:-0.369}, 'LeftHandThumb3.rotation':{x:0.034,y:-0.009,z:0.016}, 'LeftHandIndex1.rotation':{x:0.157,y:-0.002,z:-0.171}, 'LeftHandIndex2.rotation':{x:0.099,y:0,z:0}, 'LeftHandIndex3.rotation':{x:0.1,y:0,z:0}, 'LeftHandMiddle1.rotation':{x:0.222,y:-0.019,z:-0.16}, 'LeftHandMiddle2.rotation':{x:0.142,y:0,z:0}, 'LeftHandMiddle3.rotation':{x:0.141,y:0,z:0}, 'LeftHandRing1.rotation':{x:0.333,y:-0.039,z:-0.174}, 'LeftHandRing2.rotation':{x:0.214,y:0,z:0}, 'LeftHandRing3.rotation':{x:0.213,y:0,z:0}, 'LeftHandPinky1.rotation':{x:0.483,y:-0.069,z:-0.189}, 'LeftHandPinky2.rotation':{x:0.312,y:0,z:0}, 'LeftHandPinky3.rotation':{x:0.309,y:0,z:0}, 'RightShoulder.rotation':{x:1.597,y:0.012,z:1.816}, 'RightArm.rotation':{x:0.618,y:-1.274,z:-0.266}, 'RightForeArm.rotation':{x:-0.395,y:-0.097,z:-1.342}, 'RightHand.rotation':{x:-0.816,y:-0.057,z:-0.976}, 'RightHandThumb1.rotation':{x:0.42,y:0.23,z:-1.172}, 'RightHandThumb2.rotation':{x:-0.027,y:0.361,z:0.122}, 'RightHandThumb3.rotation':{x:0.076,y:0.125,z:-0.371}, 'RightHandIndex1.rotation':{x:-0.158,y:-0.045,z:0.033}, 'RightHandIndex2.rotation':{x:0.391,y:0.051,z:0.025}, 'RightHandIndex3.rotation':{x:0.317,y:0.058,z:0.07}, 'RightHandMiddle1.rotation':{x:0.486,y:0.066,z:0.014}, 'RightHandMiddle2.rotation':{x:0.718,y:0.055,z:0.07}, 'RightHandMiddle3.rotation':{x:0.453,y:0.019,z:0.013}, 'RightHandRing1.rotation':{x:0.591,y:0.241,z:0.11}, 'RightHandRing2.rotation':{x:1.014,y:0.023,z:0.097}, 'RightHandRing3.rotation':{x:0.708,y:0.008,z:0.066}, 'RightHandPinky1.rotation':{x:1.02,y:0.305,z:0.051}, 'RightHandPinky2.rotation':{x:1.187,y:-0.028,z:0.191}, 'RightHandPinky3.rotation':{x:0.872,y:-0.031,z:0.121}, 'LeftUpLeg.rotation':{x:-0.095,y:-0.058,z:-3.338}, 'LeftLeg.rotation':{x:-0.366,y:0.287,z:-0.021}, 'LeftFoot.rotation':{x:1.131,y:0.21,z:0.176}, 'LeftToeBase.rotation':{x:0.739,y:-0.068,z:-0.001}, 'RightUpLeg.rotation':{x:-0.502,y:0.362,z:3.153}, 'RightLeg.rotation':{x:-1.002,y:0.109,z:0.008}, 'RightFoot.rotation':{x:0.626,y:-0.097,z:-0.194}, 'RightToeBase.rotation':{x:1.33,y:0.288,z:-0.078} - } - }, - - 'turn':{ - standing: true, - props: { - 'Hips.position':{x:0,y:1,z:0}, 'Hips.rotation':{x:-0.07,y:-0.604,z:-0.004}, 'Spine.rotation':{x:-0.007,y:0.003,z:0.071}, 'Spine1.rotation':{x:-0.053,y:0.024,z:-0.06}, 'Spine2.rotation':{x:0.074,y:0.013,z:-0.068}, 'Neck.rotation':{x:0.03,y:0.186,z:-0.077}, 'Head.rotation':{x:0.045,y:0.243,z:-0.086}, 'LeftShoulder.rotation':{x:1.717,y:-0.085,z:-1.761}, 'LeftArm.rotation':{x:1.314,y:0.07,z:-0.057}, 'LeftForeArm.rotation':{x:-0.151,y:0.714,z:0.302}, 'LeftHand.rotation':{x:-0.069,y:0.003,z:-0.118}, 'LeftHandThumb1.rotation':{x:0.23,y:0.258,z:0.354}, 'LeftHandThumb2.rotation':{x:-0.107,y:-0.338,z:-0.455}, 'LeftHandThumb3.rotation':{x:-0.015,y:-0.142,z:0.002}, 'LeftHandIndex1.rotation':{x:0.145,y:0.032,z:-0.069}, 'LeftHandIndex2.rotation':{x:0.323,y:-0.049,z:-0.028}, 'LeftHandIndex3.rotation':{x:0.249,y:-0.053,z:-0.074}, 'LeftHandMiddle1.rotation':{x:0.235,y:-0.057,z:-0.088}, 'LeftHandMiddle2.rotation':{x:0.468,y:-0.036,z:-0.081}, 'LeftHandMiddle3.rotation':{x:0.203,y:-0.015,z:-0.017}, 'LeftHandRing1.rotation':{x:0.185,y:-0.118,z:-0.157}, 'LeftHandRing2.rotation':{x:0.578,y:0.02,z:-0.097}, 'LeftHandRing3.rotation':{x:0.27,y:0.021,z:-0.063}, 'LeftHandPinky1.rotation':{x:0.404,y:-0.182,z:-0.138}, 'LeftHandPinky2.rotation':{x:0.612,y:0.128,z:-0.144}, 'LeftHandPinky3.rotation':{x:0.267,y:0.094,z:-0.081}, 'RightShoulder.rotation':{x:1.605,y:0.17,z:1.625}, 'RightArm.rotation':{x:1.574,y:-0.655,z:0.388}, 'RightForeArm.rotation':{x:-0.36,y:-0.849,z:-0.465}, 'RightHand.rotation':{x:0.114,y:0.416,z:-0.069}, 'RightHandThumb1.rotation':{x:0.486,y:0.009,z:-0.492}, 'RightHandThumb2.rotation':{x:-0.073,y:-0.01,z:0.284}, 'RightHandThumb3.rotation':{x:-0.054,y:-0.006,z:0.209}, 'RightHandIndex1.rotation':{x:0.245,y:-0.014,z:0.052}, 'RightHandIndex2.rotation':{x:0.155,y:0,z:0}, 'RightHandIndex3.rotation':{x:0.153,y:0,z:0}, 'RightHandMiddle1.rotation':{x:0.238,y:0.004,z:0.028}, 'RightHandMiddle2.rotation':{x:0.15,y:0,z:0}, 'RightHandMiddle3.rotation':{x:0.149,y:0,z:0}, 'RightHandRing1.rotation':{x:0.267,y:0.012,z:0.007}, 'RightHandRing2.rotation':{x:0.169,y:0,z:0}, 'RightHandRing3.rotation':{x:0.167,y:0,z:0}, 'RightHandPinky1.rotation':{x:0.304,y:0.018,z:-0.021}, 'RightHandPinky2.rotation':{x:0.192,y:0,z:0}, 'RightHandPinky3.rotation':{x:0.19,y:0,z:0}, 'LeftUpLeg.rotation':{x:-0.001,y:-0.058,z:-3.238}, 'LeftLeg.rotation':{x:-0.29,y:0.058,z:-0.021}, 'LeftFoot.rotation':{x:1.288,y:0.168,z:0.183}, 'LeftToeBase.rotation':{x:0.363,y:-0.09,z:-0.01}, 'RightUpLeg.rotation':{x:-0.100,y:0.36,z:3.062}, 'RightLeg.rotation':{x:-0.67,y:-0.304,z:0.043}, 'RightFoot.rotation':{x:1.195,y:-0.159,z:-0.294}, 'RightToeBase.rotation':{x:0.737,y:0.164,z:-0.002} - } - }, - - 'bend':{ - bend: true, standing: true, - props: { - 'Hips.position':{x:-0.007, y:0.943, z:-0.001}, 'Hips.rotation':{x:1.488, y:-0.633, z:1.435}, 'Spine.rotation':{x:-0.126, y:0.007, z:-0.057}, 'Spine1.rotation':{x:-0.134, y:0.009, z:0.01}, 'Spine2.rotation':{x:-0.019, y:0, z:-0.002}, 'Neck.rotation':{x:-0.159, y:0.572, z:-0.108}, 'Head.rotation':{x:-0.064, y:0.716, z:-0.257}, 'RightShoulder.rotation':{x:1.625, y:-0.043, z:1.382}, 'RightArm.rotation':{x:0.746, y:-0.96, z:-1.009}, 'RightForeArm.rotation':{x:-0.199, y:-0.528, z:-0.38}, 'RightHand.rotation':{x:-0.261, y:-0.043, z:-0.027}, 'RightHandThumb1.rotation':{x:0.172, y:-0.138, z:-0.445}, 'RightHandThumb2.rotation':{x:-0.158, y:0.327, z:0.545}, 'RightHandThumb3.rotation':{x:-0.062, y:0.138, z:0.152}, 'RightHandIndex1.rotation':{x:0.328, y:-0.005, z:0.132}, 'RightHandIndex2.rotation':{x:0.303, y:0.049, z:0.028}, 'RightHandIndex3.rotation':{x:0.241, y:0.046, z:0.077}, 'RightHandMiddle1.rotation':{x:0.309, y:0.074, z:0.089}, 'RightHandMiddle2.rotation':{x:0.392, y:0.036, z:0.081}, 'RightHandMiddle3.rotation':{x:0.199, y:0.014, z:0.019}, 'RightHandRing1.rotation':{x:0.239, y:0.143, z:0.091}, 'RightHandRing2.rotation':{x:0.275, y:-0.02, z:0.097}, 'RightHandRing3.rotation':{x:0.248, y:-0.023, z:0.061}, 'RightHandPinky1.rotation':{x:0.211, y:0.154, z:0.029}, 'RightHandPinky2.rotation':{x:0.348, y:-0.128, z:0.144}, 'RightHandPinky3.rotation':{x:0.21, y:-0.091, z:0.065}, 'LeftShoulder.rotation':{x:1.626, y:-0.027, z:-1.367}, 'LeftArm.rotation':{x:1.048, y:0.737, z:0.712}, 'LeftForeArm.rotation':{x:-0.508, y:0.879, z:0.625}, 'LeftHand.rotation':{x:0.06, y:-0.243, z:-0.079}, 'LeftHandThumb1.rotation':{x:0.187, y:-0.072, z:0.346}, 'LeftHandThumb2.rotation':{x:-0.066, y:0.008, z:-0.256}, 'LeftHandThumb3.rotation':{x:-0.085, y:0.014, z:-0.334}, 'LeftHandIndex1.rotation':{x:-0.1, y:0.016, z:-0.058}, 'LeftHandIndex2.rotation':{x:0.334, y:0, z:0}, 'LeftHandIndex3.rotation':{x:0.281, y:0, z:0}, 'LeftHandMiddle1.rotation':{x:-0.056, y:0, z:0}, 'LeftHandMiddle2.rotation':{x:0.258, y:0, z:0}, 'LeftHandMiddle3.rotation':{x:0.26, y:0, z:0}, 'LeftHandRing1.rotation':{x:-0.067, y:-0.002, z:0.008}, 'LeftHandRing2.rotation':{x:0.259, y:0, z:0}, 'LeftHandRing3.rotation':{x:0.276, y:0, z:0}, 'LeftHandPinky1.rotation':{x:-0.128, y:-0.007, z:0.042}, 'LeftHandPinky2.rotation':{x:0.227, y:0, z:0}, 'LeftHandPinky3.rotation':{x:0.145, y:0, z:0}, 'RightUpLeg.rotation':{x:-1.507, y:0.2, z:-3.043}, 'RightLeg.rotation':{x:-0.689, y:-0.124, z:0.017}, 'RightFoot.rotation':{x:0.909, y:0.008, z:-0.093}, 'RightToeBase.rotation':{x:0.842, y:0.075, z:-0.008}, 'LeftUpLeg.rotation':{x:-1.449, y:-0.2, z:3.018}, 'LeftLeg.rotation':{x:-0.74, y:-0.115, z:-0.008}, 'LeftFoot.rotation':{x:1.048, y:-0.058, z:0.117}, 'LeftToeBase.rotation':{x:0.807, y:-0.067, z:0.003} - } - }, - - 'back':{ - standing: true, - props: { - 'Hips.position':{x:0,y:1,z:0}, 'Hips.rotation':{x:-0.732,y:-1.463,z:-0.637}, 'Spine.rotation':{x:-0.171,y:0.106,z:0.157}, 'Spine1.rotation':{x:-0.044,y:0.138,z:-0.059}, 'Spine2.rotation':{x:0.082,y:0.133,z:-0.074}, 'Neck.rotation':{x:0.39,y:0.591,z:-0.248}, 'Head.rotation':{x:-0.001,y:0.596,z:-0.057}, 'LeftShoulder.rotation':{x:1.676,y:0.007,z:-1.892}, 'LeftArm.rotation':{x:-5.566,y:1.188,z:-0.173}, 'LeftForeArm.rotation':{x:-0.673,y:-0.105,z:1.702}, 'LeftHand.rotation':{x:-0.469,y:-0.739,z:0.003}, 'LeftHandThumb1.rotation':{x:0.876,y:0.274,z:0.793}, 'LeftHandThumb2.rotation':{x:0.161,y:-0.23,z:-0.172}, 'LeftHandThumb3.rotation':{x:0.078,y:0.027,z:0.156}, 'LeftHandIndex1.rotation':{x:-0.085,y:-0.002,z:0.009}, 'LeftHandIndex2.rotation':{x:0.176,y:0,z:-0.002}, 'LeftHandIndex3.rotation':{x:-0.036,y:0.001,z:-0.035}, 'LeftHandMiddle1.rotation':{x:0.015,y:0.144,z:-0.076}, 'LeftHandMiddle2.rotation':{x:0.378,y:-0.007,z:-0.077}, 'LeftHandMiddle3.rotation':{x:-0.141,y:-0.001,z:0.031}, 'LeftHandRing1.rotation':{x:0.039,y:0.02,z:-0.2}, 'LeftHandRing2.rotation':{x:0.25,y:-0.002,z:-0.073}, 'LeftHandRing3.rotation':{x:0.236,y:0.006,z:-0.075}, 'LeftHandPinky1.rotation':{x:0.172,y:-0.033,z:-0.275}, 'LeftHandPinky2.rotation':{x:0.216,y:0.043,z:-0.054}, 'LeftHandPinky3.rotation':{x:0.325,y:0.078,z:-0.13}, 'RightShoulder.rotation':{x:2.015,y:-0.168,z:1.706}, 'RightArm.rotation':{x:0.203,y:-1.258,z:-0.782}, 'RightForeArm.rotation':{x:-0.658,y:-0.133,z:-1.401}, 'RightHand.rotation':{x:-1.504,y:0.375,z:-0.005}, 'RightHandThumb1.rotation':{x:0.413,y:-0.158,z:-1.121}, 'RightHandThumb2.rotation':{x:-0.142,y:-0.008,z:0.209}, 'RightHandThumb3.rotation':{x:-0.091,y:0.021,z:0.142}, 'RightHandIndex1.rotation':{x:-0.167,y:0.014,z:-0.072}, 'RightHandIndex2.rotation':{x:0.474,y:0.009,z:0.051}, 'RightHandIndex3.rotation':{x:0.115,y:0.006,z:0.047}, 'RightHandMiddle1.rotation':{x:0.385,y:0.019,z:0.144}, 'RightHandMiddle2.rotation':{x:0.559,y:0.035,z:0.101}, 'RightHandMiddle3.rotation':{x:0.229,y:0,z:0.027}, 'RightHandRing1.rotation':{x:0.48,y:0.026,z:0.23}, 'RightHandRing2.rotation':{x:0.772,y:0.038,z:0.109}, 'RightHandRing3.rotation':{x:0.622,y:0.039,z:0.106}, 'RightHandPinky1.rotation':{x:0.767,y:0.288,z:0.353}, 'RightHandPinky2.rotation':{x:0.886,y:0.049,z:0.122}, 'RightHandPinky3.rotation':{x:0.662,y:0.044,z:0.113}, 'LeftUpLeg.rotation':{x:-0.206,y:-0.268,z:-3.343}, 'LeftLeg.rotation':{x:-0.333,y:0.757,z:-0.043}, 'LeftFoot.rotation':{x:1.049,y:0.167,z:0.287}, 'LeftToeBase.rotation':{x:0.672,y:-0.069,z:-0.004}, 'RightUpLeg.rotation':{x:0.055,y:-0.226,z:3.037}, 'RightLeg.rotation':{x:-0.559,y:0.39,z:-0.001}, 'RightFoot.rotation':{x:1.2,y:0.133,z:0.085}, 'RightToeBase.rotation':{x:0.92,y:0.093,z:-0.013} - } - }, - - 'straight':{ - standing: true, - props: { - 'Hips.position':{x:0, y:0.989, z:0.001}, 'Hips.rotation':{x:0.047, y:0.007, z:-0.007}, 'Spine.rotation':{x:-0.143, y:-0.007, z:0.005}, 'Spine1.rotation':{x:-0.043, y:-0.014, z:0.012}, 'Spine2.rotation':{x:0.072, y:-0.013, z:0.013}, 'Neck.rotation':{x:0.048, y:-0.003, z:0.012}, 'Head.rotation':{x:0.05, y:-0.02, z:-0.017}, 'LeftShoulder.rotation':{x:1.62, y:-0.166, z:-1.605}, 'LeftArm.rotation':{x:1.275, y:0.544, z:-0.092}, 'LeftForeArm.rotation':{x:0, y:0, z:0.302}, 'LeftHand.rotation':{x:-0.225, y:-0.154, z:0.11}, 'LeftHandThumb1.rotation':{x:0.435, y:-0.044, z:0.457}, 'LeftHandThumb2.rotation':{x:-0.028, y:0.002, z:-0.246}, 'LeftHandThumb3.rotation':{x:-0.236, y:-0.025, z:0.113}, 'LeftHandIndex1.rotation':{x:0.218, y:0.008, z:-0.081}, 'LeftHandIndex2.rotation':{x:0.165, y:-0.001, z:-0.017}, 'LeftHandIndex3.rotation':{x:0.165, y:-0.001, z:-0.017}, 'LeftHandMiddle1.rotation':{x:0.235, y:-0.011, z:-0.065}, 'LeftHandMiddle2.rotation':{x:0.182, y:-0.002, z:-0.019}, 'LeftHandMiddle3.rotation':{x:0.182, y:-0.002, z:-0.019}, 'LeftHandRing1.rotation':{x:0.316, y:-0.017, z:0.008}, 'LeftHandRing2.rotation':{x:0.253, y:-0.003, z:-0.026}, 'LeftHandRing3.rotation':{x:0.255, y:-0.003, z:-0.026}, 'LeftHandPinky1.rotation':{x:0.336, y:-0.062, z:0.088}, 'LeftHandPinky2.rotation':{x:0.276, y:-0.004, z:-0.028}, 'LeftHandPinky3.rotation':{x:0.276, y:-0.004, z:-0.028}, 'RightShoulder.rotation':{x:1.615, y:0.064, z:1.53}, 'RightArm.rotation':{x:1.313, y:-0.424, z:0.131}, 'RightForeArm.rotation':{x:0, y:0, z:-0.317}, 'RightHand.rotation':{x:-0.158, y:-0.639, z:-0.196}, 'RightHandThumb1.rotation':{x:0.44, y:0.048, z:-0.549}, 'RightHandThumb2.rotation':{x:-0.056, y:-0.008, z:0.274}, 'RightHandThumb3.rotation':{x:-0.258, y:0.031, z:-0.095}, 'RightHandIndex1.rotation':{x:0.169, y:-0.011, z:0.105}, 'RightHandIndex2.rotation':{x:0.134, y:0.001, z:0.011}, 'RightHandIndex3.rotation':{x:0.134, y:0.001, z:0.011}, 'RightHandMiddle1.rotation':{x:0.288, y:0.014, z:0.092}, 'RightHandMiddle2.rotation':{x:0.248, y:0.003, z:0.02}, 'RightHandMiddle3.rotation':{x:0.249, y:0.003, z:0.02}, 'RightHandRing1.rotation':{x:0.369, y:0.019, z:0.006}, 'RightHandRing2.rotation':{x:0.321, y:0.004, z:0.026}, 'RightHandRing3.rotation':{x:0.323, y:0.004, z:0.026}, 'RightHandPinky1.rotation':{x:0.468, y:0.085, z:-0.03}, 'RightHandPinky2.rotation':{x:0.427, y:0.007, z:0.034}, 'RightHandPinky3.rotation':{x:0.142, y:0.001, z:0.012}, 'LeftUpLeg.rotation':{x:-0.077, y:-0.058, z:3.126}, 'LeftLeg.rotation':{x:-0.252, y:0.001, z:-0.018}, 'LeftFoot.rotation':{x:1.315, y:-0.064, z:0.315}, 'LeftToeBase.rotation':{x:0.577, y:-0.07, z:-0.009}, 'RightUpLeg.rotation':{x:-0.083, y:-0.032, z:3.124}, 'RightLeg.rotation':{x:-0.272, y:-0.003, z:0.021}, 'RightFoot.rotation':{x:1.342, y:0.076, z:-0.222}, 'RightToeBase.rotation':{x:0.44, y:0.069, z:0.016} - } - }, - - 'wide':{ - standing: true, - props: { - 'Hips.position':{x:0, y:1.017, z:0.016}, 'Hips.rotation':{x:0.064, y:-0.048, z:0.059}, 'Spine.rotation':{x:-0.123, y:0, z:-0.018}, 'Spine1.rotation':{x:0.014, y:0.003, z:-0.006}, 'Spine2.rotation':{x:0.04, y:0.003, z:-0.007}, 'Neck.rotation':{x:0.101, y:0.007, z:-0.035}, 'Head.rotation':{x:-0.091, y:-0.049, z:0.105}, 'RightShoulder.rotation':{x:1.831, y:0.017, z:1.731}, 'RightArm.rotation':{x:-1.673, y:-1.102, z:-3.132}, 'RightForeArm.rotation':{x:0.265, y:0.23, z:-0.824}, 'RightHand.rotation':{x:-0.52, y:0.345, z:-0.061}, 'RightHandThumb1.rotation':{x:0.291, y:0.056, z:-0.428}, 'RightHandThumb2.rotation':{x:0.025, y:0.005, z:0.166}, 'RightHandThumb3.rotation':{x:-0.089, y:0.009, z:0.068}, 'RightHandIndex1.rotation':{x:0.392, y:-0.015, z:0.11}, 'RightHandIndex2.rotation':{x:0.391, y:0.001, z:0.004}, 'RightHandIndex3.rotation':{x:0.326, y:0, z:0.003}, 'RightHandMiddle1.rotation':{x:0.285, y:0.068, z:0.081}, 'RightHandMiddle2.rotation':{x:0.519, y:0.004, z:0.011}, 'RightHandMiddle3.rotation':{x:0.252, y:0, z:0.001}, 'RightHandRing1.rotation':{x:0.207, y:0.133, z:0.146}, 'RightHandRing2.rotation':{x:0.597, y:0.004, z:0.004}, 'RightHandRing3.rotation':{x:0.292, y:0.002, z:0.012}, 'RightHandPinky1.rotation':{x:0.338, y:0.182, z:0.136}, 'RightHandPinky2.rotation':{x:0.533, y:0.002, z:0.004}, 'RightHandPinky3.rotation':{x:0.194, y:0, z:0.002}, 'LeftShoulder.rotation':{x:1.83, y:-0.063, z:-1.808}, 'LeftArm.rotation':{x:-1.907, y:1.228, z:-2.959}, 'LeftForeArm.rotation':{x:-0.159, y:0.268, z:0.572}, 'LeftHand.rotation':{x:0.069, y:-0.498, z:-0.025}, 'LeftHandThumb1.rotation':{x:0.738, y:0.123, z:0.178}, 'LeftHandThumb2.rotation':{x:-0.26, y:0.028, z:-0.477}, 'LeftHandThumb3.rotation':{x:-0.448, y:0.093, z:-0.661}, 'LeftHandIndex1.rotation':{x:1.064, y:0.005, z:-0.13}, 'LeftHandIndex2.rotation':{x:1.55, y:-0.143, z:-0.136}, 'LeftHandIndex3.rotation':{x:0.722, y:-0.076, z:-0.127}, 'LeftHandMiddle1.rotation':{x:1.095, y:-0.091, z:0.006}, 'LeftHandMiddle2.rotation':{x:1.493, y:-0.174, z:-0.151}, 'LeftHandMiddle3.rotation':{x:0.651, y:-0.031, z:-0.087}, 'LeftHandRing1.rotation':{x:1.083, y:-0.224, z:0.072}, 'LeftHandRing2.rotation':{x:1.145, y:-0.107, z:-0.195}, 'LeftHandRing3.rotation':{x:1.208, y:-0.134, z:-0.158}, 'LeftHandPinky1.rotation':{x:0.964, y:-0.383, z:0.128}, 'LeftHandPinky2.rotation':{x:1.457, y:-0.146, z:-0.159}, 'LeftHandPinky3.rotation':{x:1.019, y:-0.102, z:-0.141}, 'RightUpLeg.rotation':{x:-0.221, y:-0.233, z:2.87}, 'RightLeg.rotation':{x:-0.339, y:-0.043, z:-0.041}, 'RightFoot.rotation':{x:1.081, y:0.177, z:0.114}, 'RightToeBase.rotation':{x:0.775, y:0, z:0}, 'LeftUpLeg.rotation':{x:-0.185, y:0.184, z:3.131}, 'LeftLeg.rotation':{x:-0.408, y:0.129, z:0.02}, 'LeftFoot.rotation':{x:1.167, y:-0.002, z:-0.007}, 'LeftToeBase.rotation':{x:0.723, y:0, z:0} - } - }, - - 'oneknee':{ - kneeling: true, - props: { - 'Hips.position':{x:-0.005, y:0.415, z:-0.017}, 'Hips.rotation':{x:-0.25, y:0.04, z:-0.238}, 'Spine.rotation':{x:0.037, y:0.043, z:0.047}, 'Spine1.rotation':{x:0.317, y:0.103, z:0.066}, 'Spine2.rotation':{x:0.433, y:0.109, z:0.054}, 'Neck.rotation':{x:-0.156, y:-0.092, z:0.059}, 'Head.rotation':{x:-0.398, y:-0.032, z:0.018}, 'RightShoulder.rotation':{x:1.546, y:0.119, z:1.528}, 'RightArm.rotation':{x:0.896, y:-0.247, z:-0.512}, 'RightForeArm.rotation':{x:0.007, y:0, z:-1.622}, 'RightHand.rotation':{x:1.139, y:-0.853, z:0.874}, 'RightHandThumb1.rotation':{x:0.176, y:0.107, z:-0.311}, 'RightHandThumb2.rotation':{x:-0.047, y:-0.003, z:0.12}, 'RightHandThumb3.rotation':{x:0, y:0, z:0}, 'RightHandIndex1.rotation':{x:0.186, y:0.005, z:0.125}, 'RightHandIndex2.rotation':{x:0.454, y:0.005, z:0.015}, 'RightHandIndex3.rotation':{x:0, y:0, z:0}, 'RightHandMiddle1.rotation':{x:0.444, y:0.035, z:0.127}, 'RightHandMiddle2.rotation':{x:0.403, y:-0.006, z:-0.04}, 'RightHandMiddle3.rotation':{x:0, y:0, z:0}, 'RightHandRing1.rotation':{x:0.543, y:0.074, z:0.121}, 'RightHandRing2.rotation':{x:0.48, y:-0.018, z:-0.063}, 'RightHandRing3.rotation':{x:0, y:0, z:0}, 'RightHandPinky1.rotation':{x:0.464, y:0.086, z:0.113}, 'RightHandPinky2.rotation':{x:0.667, y:-0.06, z:-0.128}, 'RightHandPinky3.rotation':{x:0, y:0, z:0}, 'LeftShoulder.rotation':{x:1.545, y:-0.116, z:-1.529}, 'LeftArm.rotation':{x:0.799, y:0.631, z:0.556}, 'LeftForeArm.rotation':{x:-0.002, y:0.007, z:0.926}, 'LeftHand.rotation':{x:-0.508, y:0.439, z:0.502}, 'LeftHandThumb1.rotation':{x:0.651, y:-0.035, z:0.308}, 'LeftHandThumb2.rotation':{x:-0.053, y:0.008, z:-0.11}, 'LeftHandThumb3.rotation':{x:0, y:0, z:0}, 'LeftHandIndex1.rotation':{x:0.662, y:-0.053, z:-0.116}, 'LeftHandIndex2.rotation':{x:0.309, y:-0.004, z:-0.02}, 'LeftHandIndex3.rotation':{x:0, y:0, z:0}, 'LeftHandMiddle1.rotation':{x:0.501, y:-0.062, z:-0.12}, 'LeftHandMiddle2.rotation':{x:0.144, y:-0.002, z:0.016}, 'LeftHandMiddle3.rotation':{x:0, y:0, z:0}, 'LeftHandRing1.rotation':{x:0.397, y:-0.029, z:-0.143}, 'LeftHandRing2.rotation':{x:0.328, y:0.01, z:0.059}, 'LeftHandRing3.rotation':{x:0, y:0, z:0}, 'LeftHandPinky1.rotation':{x:0.194, y:0.008, z:-0.164}, 'LeftHandPinky2.rotation':{x:0.38, y:0.031, z:0.128}, 'LeftHandPinky3.rotation':{x:0, y:0, z:0}, 'RightUpLeg.rotation':{x:-1.594, y:-0.251, z:2.792}, 'RightLeg.rotation':{x:-2.301, y:-0.073, z:0.055}, 'RightFoot.rotation':{x:1.553, y:-0.207, z:-0.094}, 'RightToeBase.rotation':{x:0.459, y:0.069, z:0.016}, 'LeftUpLeg.rotation':{x:-0.788, y:-0.236, z:-2.881}, 'LeftLeg.rotation':{x:-2.703, y:0.012, z:-0.047}, 'LeftFoot.rotation':{x:2.191, y:-0.102, z:0.019}, 'LeftToeBase.rotation':{x:1.215, y:-0.027, z:0.01} - } - }, - - 'kneel':{ - kneeling: true, lying: true, - props: { - 'Hips.position':{x:0, y:0.532, z:-0.002}, 'Hips.rotation':{x:0.018, y:-0.008, z:-0.017}, 'Spine.rotation':{x:-0.139, y:-0.01, z:0.002}, 'Spine1.rotation':{x:0.002, y:-0.002, z:0.001}, 'Spine2.rotation':{x:0.028, y:-0.002, z:0.001}, 'Neck.rotation':{x:-0.007, y:0, z:-0.002}, 'Head.rotation':{x:-0.02, y:-0.008, z:-0.004}, 'LeftShoulder.rotation':{x:1.77, y:-0.428, z:-1.588}, 'LeftArm.rotation':{x:0.911, y:0.343, z:0.083}, 'LeftForeArm.rotation':{x:0, y:0, z:0.347}, 'LeftHand.rotation':{x:0.033, y:-0.052, z:-0.105}, 'LeftHandThumb1.rotation':{x:0.508, y:-0.22, z:0.708}, 'LeftHandThumb2.rotation':{x:-0.323, y:-0.139, z:-0.56}, 'LeftHandThumb3.rotation':{x:-0.328, y:0.16, z:-0.301}, 'LeftHandIndex1.rotation':{x:0.178, y:0.248, z:0.045}, 'LeftHandIndex2.rotation':{x:0.236, y:-0.002, z:-0.019}, 'LeftHandIndex3.rotation':{x:-0.062, y:0, z:0.005}, 'LeftHandMiddle1.rotation':{x:0.123, y:-0.005, z:-0.019}, 'LeftHandMiddle2.rotation':{x:0.589, y:-0.014, z:-0.045}, 'LeftHandMiddle3.rotation':{x:0.231, y:-0.002, z:-0.019}, 'LeftHandRing1.rotation':{x:0.196, y:-0.008, z:-0.091}, 'LeftHandRing2.rotation':{x:0.483, y:-0.009, z:-0.038}, 'LeftHandRing3.rotation':{x:0.367, y:-0.005, z:-0.029}, 'LeftHandPinky1.rotation':{x:0.191, y:-0.269, z:-0.246}, 'LeftHandPinky2.rotation':{x:0.37, y:-0.006, z:-0.029}, 'LeftHandPinky3.rotation':{x:0.368, y:-0.005, z:-0.029}, 'RightShoulder.rotation':{x:1.73, y:0.434, z:1.715}, 'RightArm.rotation':{x:0.841, y:-0.508, z:-0.155}, 'RightForeArm.rotation':{x:0, y:0, z:-0.355}, 'RightHand.rotation':{x:0.091, y:0.137, z:0.197}, 'RightHandThumb1.rotation':{x:0.33, y:0.051, z:-0.753}, 'RightHandThumb2.rotation':{x:-0.113, y:0.075, z:0.612}, 'RightHandThumb3.rotation':{x:-0.271, y:-0.166, z:0.164}, 'RightHandIndex1.rotation':{x:0.073, y:0.001, z:-0.093}, 'RightHandIndex2.rotation':{x:0.338, y:0.006, z:0.034}, 'RightHandIndex3.rotation':{x:0.131, y:0.001, z:0.013}, 'RightHandMiddle1.rotation':{x:0.13, y:0.005, z:-0.017}, 'RightHandMiddle2.rotation':{x:0.602, y:0.018, z:0.058}, 'RightHandMiddle3.rotation':{x:-0.031, y:0, z:-0.003}, 'RightHandRing1.rotation':{x:0.351, y:0.019, z:0.045}, 'RightHandRing2.rotation':{x:0.19, y:0.002, z:0.019}, 'RightHandRing3.rotation':{x:0.21, y:0.002, z:0.021}, 'RightHandPinky1.rotation':{x:0.256, y:0.17, z:0.118}, 'RightHandPinky2.rotation':{x:0.451, y:0.01, z:0.045}, 'RightHandPinky3.rotation':{x:0.346, y:0.006, z:0.035}, 'LeftUpLeg.rotation':{x:-0.06, y:0.1, z:-2.918}, 'LeftLeg.rotation':{x:-1.933, y:-0.01, z:0.011}, 'LeftFoot.rotation':{x:0.774, y:-0.162, z:-0.144}, 'LeftToeBase.rotation':{x:1.188, y:0, z:0}, 'RightUpLeg.rotation':{x:-0.099, y:-0.057, z:2.922}, 'RightLeg.rotation':{x:-1.93, y:0.172, z:-0.02}, 'RightFoot.rotation':{x:0.644, y:0.251, z:0.212}, 'RightToeBase.rotation':{x:0.638, y:-0.034, z:-0.001} - } - }, - - 'sitting': { - sitting: true, lying: true, - props: { - 'Hips.position':{x:0, y:0.117, z:0.005}, 'Hips.rotation':{x:-0.411, y:-0.049, z:0.056}, 'Spine.rotation':{x:0.45, y:-0.039, z:-0.116}, 'Spine1.rotation':{x:0.092, y:-0.076, z:0.08}, 'Spine2.rotation':{x:0.073, y:0.035, z:0.066}, 'Neck.rotation':{x:0.051, y:0.053, z:-0.079}, 'Head.rotation':{x:-0.169, y:0.009, z:0.034}, 'LeftShoulder.rotation':{x:1.756, y:-0.037, z:-1.301}, 'LeftArm.rotation':{x:-0.098, y:0.016, z:1.006}, 'LeftForeArm.rotation':{x:-0.089, y:0.08, z:0.837}, 'LeftHand.rotation':{x:0.262, y:-0.399, z:0.3}, 'LeftHandThumb1.rotation':{x:0.149, y:-0.043, z:0.452}, 'LeftHandThumb2.rotation':{x:0.032, y:0.006, z:-0.162}, 'LeftHandThumb3.rotation':{x:-0.086, y:-0.005, z:-0.069}, 'LeftHandIndex1.rotation':{x:0.145, y:0.032, z:-0.069}, 'LeftHandIndex2.rotation':{x:0.325, y:-0.001, z:-0.004}, 'LeftHandIndex3.rotation':{x:0.253, y:0, z:-0.003}, 'LeftHandMiddle1.rotation':{x:0.186, y:-0.051, z:-0.091}, 'LeftHandMiddle2.rotation':{x:0.42, y:-0.003, z:-0.011}, 'LeftHandMiddle3.rotation':{x:0.153, y:0.001, z:-0.001}, 'LeftHandRing1.rotation':{x:0.087, y:-0.19, z:-0.078}, 'LeftHandRing2.rotation':{x:0.488, y:-0.004, z:-0.005}, 'LeftHandRing3.rotation':{x:0.183, y:-0.001, z:-0.012}, 'LeftHandPinky1.rotation':{x:0.205, y:-0.262, z:0.051}, 'LeftHandPinky2.rotation':{x:0.407, y:-0.002, z:-0.004}, 'LeftHandPinky3.rotation':{x:0.068, y:0, z:-0.002}, 'RightShoulder.rotation':{x:1.619, y:-0.139, z:1.179}, 'RightArm.rotation':{x:0.17, y:-0.037, z:-1.07}, 'RightForeArm.rotation':{x:-0.044, y:-0.056, z:-0.665}, 'RightHand.rotation':{x:0.278, y:0.454, z:-0.253}, 'RightHandThumb1.rotation':{x:0.173, y:0.089, z:-0.584}, 'RightHandThumb2.rotation':{x:-0.003, y:-0.004, z:0.299}, 'RightHandThumb3.rotation':{x:-0.133, y:-0.002, z:0.235}, 'RightHandIndex1.rotation':{x:0.393, y:-0.023, z:0.108}, 'RightHandIndex2.rotation':{x:0.391, y:0.001, z:0.004}, 'RightHandIndex3.rotation':{x:0.326, y:0, z:0.003}, 'RightHandMiddle1.rotation':{x:0.285, y:0.062, z:0.086}, 'RightHandMiddle2.rotation':{x:0.519, y:0.003, z:0.011}, 'RightHandMiddle3.rotation':{x:0.252, y:-0.001, z:0.001}, 'RightHandRing1.rotation':{x:0.207, y:0.122, z:0.155}, 'RightHandRing2.rotation':{x:0.597, y:0.004, z:0.005}, 'RightHandRing3.rotation':{x:0.292, y:0.001, z:0.012}, 'RightHandPinky1.rotation':{x:0.338, y:0.171, z:0.149}, 'RightHandPinky2.rotation':{x:0.533, y:0.002, z:0.004}, 'RightHandPinky3.rotation':{x:0.194, y:0, z:0.002}, 'LeftUpLeg.rotation':{x:-1.957, y:0.083, z:-2.886}, 'LeftLeg.rotation':{x:-1.46, y:0.123, z:0.005}, 'LeftFoot.rotation':{x:-0.013, y:0.016, z:0.09}, 'LeftToeBase.rotation':{x:0.744, y:0, z:0}, 'RightUpLeg.rotation':{x:-1.994, y:0.125, z:2.905}, 'RightLeg.rotation':{x:-1.5, y:-0.202, z:-0.006}, 'RightFoot.rotation':{x:-0.012, y:-0.065, z:0.081}, 'RightToeBase.rotation':{x:0.758, y:0, z:0} - } - } - }; - - // Gestures - // NOTE: For one hand gestures, use left left - this.gestureTemplates = { - 'handup': { - 'LeftShoulder.rotation':{x:[1.5,2,1,2], y:[0.2,0.4,1,2], z:[-1.5,-1.3,1,2]}, 'LeftArm.rotation':{x:[1.5,1.7,1,2], y:[-0.6,-0.4,1,2], z:[1,1.2,1,2]}, 'LeftForeArm.rotation':{x:-0.815, y:[-0.4,0,1,2], z:1.575}, 'LeftHand.rotation':{x:-0.529, y:-0.2, z:0.022}, 'LeftHandThumb1.rotation':{x:0.745, y:-0.526, z:0.604}, 'LeftHandThumb2.rotation':{x:-0.107, y:-0.01, z:-0.142}, 'LeftHandThumb3.rotation':{x:0, y:0.001, z:0}, 'LeftHandIndex1.rotation':{x:-0.126, y:-0.035, z:-0.087}, 'LeftHandIndex2.rotation':{x:0.255, y:0.007, z:-0.085}, 'LeftHandIndex3.rotation':{x:0, y:0, z:0}, 'LeftHandMiddle1.rotation':{x:-0.019, y:-0.128, z:-0.082}, 'LeftHandMiddle2.rotation':{x:0.233, y:0.019, z:-0.074}, 'LeftHandMiddle3.rotation':{x:0, y:0, z:0}, 'LeftHandRing1.rotation':{x:0.005, y:-0.241, z:-0.122}, 'LeftHandRing2.rotation':{x:0.261, y:0.021, z:-0.076}, 'LeftHandRing3.rotation':{x:0, y:0, z:0}, 'LeftHandPinky1.rotation':{x:0.059, y:-0.336, z:-0.2}, 'LeftHandPinky2.rotation':{x:0.153, y:0.019, z:0.001}, 'LeftHandPinky3.rotation':{x:0, y:0, z:0} - }, - 'index': { - 'LeftShoulder.rotation':{x:[1.5,2,1,2], y:[0.2,0.4,1,2], z:[-1.5,-1.3,1,2]}, 'LeftArm.rotation':{x:[1.5,1.7,1,2], y:[-0.6,-0.4,1,2], z:[1,1.2,1,2]}, 'LeftForeArm.rotation':{x:-0.815, y:[-0.4,0,1,2], z:1.575}, 'LeftHand.rotation':{x:-0.276, y:-0.506, z:-0.208}, 'LeftHandThumb1.rotation':{x:0.579, y:0.228, z:0.363}, 'LeftHandThumb2.rotation':{x:-0.027, y:-0.04, z:-0.662}, 'LeftHandThumb3.rotation':{x:0, y:0.001, z:0}, 'LeftHandIndex1.rotation':{x:0, y:-0.105, z:0.225}, 'LeftHandIndex2.rotation':{x:0.256, y:-0.103, z:-0.213}, 'LeftHandIndex3.rotation':{x:0, y:0, z:0}, 'LeftHandMiddle1.rotation':{x:1.453, y:0.07, z:0.021}, 'LeftHandMiddle2.rotation':{x:1.599, y:0.062, z:0.07}, 'LeftHandMiddle3.rotation':{x:0, y:0, z:0}, 'LeftHandRing1.rotation':{x:1.528, y:-0.073, z:0.052}, 'LeftHandRing2.rotation':{x:1.386, y:0.044, z:0.053}, 'LeftHandRing3.rotation':{x:0, y:0, z:0}, 'LeftHandPinky1.rotation':{x:1.65, y:-0.204, z:0.031}, 'LeftHandPinky2.rotation':{x:1.302, y:0.071, z:0.085}, 'LeftHandPinky3.rotation':{x:0, y:0, z:0} - }, - 'ok': { - 'LeftShoulder.rotation':{x:[1.5,2,1,2], y:[0.2,0.4,1,2], z:[-1.5,-1.3,1,2]}, 'LeftArm.rotation':{x:[1.5,1.7,1,1], y:[-0.6,-0.4,1,2], z:[1,1.2,1,2]}, 'LeftForeArm.rotation':{x:-0.415, y:[-0.4,0,1,2], z:1.575}, 'LeftHand.rotation':{x:-0.476, y:-0.506, z:-0.208}, 'LeftHandThumb1.rotation':{x:0.703, y:0.445, z:0.899}, 'LeftHandThumb2.rotation':{x:-0.312, y:-0.04, z:-0.938}, 'LeftHandThumb3.rotation':{x:-0.37, y:0.024, z:-0.393}, 'LeftHandIndex1.rotation':{x:0.8, y:-0.086, z:-0.091}, 'LeftHandIndex2.rotation':{x:1.123, y:-0.046, z:-0.074}, 'LeftHandIndex3.rotation':{x:0.562, y:-0.013, z:-0.043}, 'LeftHandMiddle1.rotation':{x:-0.019, y:-0.128, z:-0.082}, 'LeftHandMiddle2.rotation':{x:0.233, y:0.019, z:-0.074}, 'LeftHandMiddle3.rotation':{x:0, y:0, z:0}, 'LeftHandRing1.rotation':{x:0.005, y:-0.241, z:-0.122}, 'LeftHandRing2.rotation':{x:0.261, y:0.021, z:-0.076}, 'LeftHandRing3.rotation':{x:0, y:0, z:0}, 'LeftHandPinky1.rotation':{x:0.059, y:-0.336, z:-0.2}, 'LeftHandPinky2.rotation':{x:0.153, y:0.019, z:0.001}, 'LeftHandPinky3.rotation':{x:0, y:0, z:0} - }, - 'thumbup': { - 'LeftShoulder.rotation':{x:[1.5,2,1,2], y:[0.2,0.4,1,2], z:[-1.5,-1.3,1,2]}, 'LeftArm.rotation':{x:[1.5,1.7,1,2], y:[-0.6,-0.4,1,2], z:[1,1.2,1,2]}, 'LeftForeArm.rotation':{x:-0.415, y:0.206, z:1.575}, 'LeftHand.rotation':{x:-0.276, y:-0.506, z:-0.208}, 'LeftHandThumb1.rotation':{x:0.208, y:-0.189, z:0.685}, 'LeftHandThumb2.rotation':{x:0.129, y:-0.285, z:-0.163}, 'LeftHandThumb3.rotation':{x:-0.047, y:0.068, z:0.401}, 'LeftHandIndex1.rotation':{x:1.412, y:-0.102, z:-0.152}, 'LeftHandIndex2.rotation':{x:1.903, y:-0.16, z:-0.114}, 'LeftHandIndex3.rotation':{x:0.535, y:-0.017, z:-0.062}, 'LeftHandMiddle1.rotation':{x:1.424, y:-0.103, z:-0.12}, 'LeftHandMiddle2.rotation':{x:1.919, y:-0.162, z:-0.114}, 'LeftHandMiddle3.rotation':{x:0.44, y:-0.012, z:-0.051}, 'LeftHandRing1.rotation':{x:1.619, y:-0.127, z:-0.053}, 'LeftHandRing2.rotation':{x:1.898, y:-0.16, z:-0.115}, 'LeftHandRing3.rotation':{x:0.262, y:-0.004, z:-0.031}, 'LeftHandPinky1.rotation':{x:1.661, y:-0.131, z:-0.016}, 'LeftHandPinky2.rotation':{x:1.715, y:-0.067, z:-0.13}, 'LeftHandPinky3.rotation':{x:0.627, y:-0.023, z:-0.071} - }, - 'thumbdown': { - 'LeftShoulder.rotation':{x:[1.5,2,1,2], y:[0.2,0.4,1,2], z:[-1.5,-1.3,1,2]}, 'LeftArm.rotation':{x:[1.5,1.7,1,2], y:[-0.6,-0.4,1,2], z:[1,1.2,1,2]}, 'LeftForeArm.rotation':{x:-2.015, y:0.406, z:1.575}, 'LeftHand.rotation':{x:-0.176, y:-0.206, z:-0.208}, 'LeftHandThumb1.rotation':{x:0.208, y:-0.189, z:0.685}, 'LeftHandThumb2.rotation':{x:0.129, y:-0.285, z:-0.163}, 'LeftHandThumb3.rotation':{x:-0.047, y:0.068, z:0.401}, 'LeftHandIndex1.rotation':{x:1.412, y:-0.102, z:-0.152}, 'LeftHandIndex2.rotation':{x:1.903, y:-0.16, z:-0.114}, 'LeftHandIndex3.rotation':{x:0.535, y:-0.017, z:-0.062}, 'LeftHandMiddle1.rotation':{x:1.424, y:-0.103, z:-0.12}, 'LeftHandMiddle2.rotation':{x:1.919, y:-0.162, z:-0.114}, 'LeftHandMiddle3.rotation':{x:0.44, y:-0.012, z:-0.051}, 'LeftHandRing1.rotation':{x:1.619, y:-0.127, z:-0.053}, 'LeftHandRing2.rotation':{x:1.898, y:-0.16, z:-0.115}, 'LeftHandRing3.rotation':{x:0.262, y:-0.004, z:-0.031}, 'LeftHandPinky1.rotation':{x:1.661, y:-0.131, z:-0.016}, 'LeftHandPinky2.rotation':{x:1.715, y:-0.067, z:-0.13}, 'LeftHandPinky3.rotation':{x:0.627, y:-0.023, z:-0.071} - }, - 'side': { - 'LeftShoulder.rotation':{x:1.755, y:-0.035, z:-1.63}, 'LeftArm.rotation':{x:1.263, y:-0.955, z:1.024}, 'LeftForeArm.rotation':{x:0, y:0, z:0.8}, 'LeftHand.rotation':{x:-0.36, y:-1.353, z:-0.184}, 'LeftHandThumb1.rotation':{x:0.137, y:-0.049, z:0.863}, 'LeftHandThumb2.rotation':{x:-0.293, y:0.153, z:-0.193}, 'LeftHandThumb3.rotation':{x:-0.271, y:-0.17, z:0.18}, 'LeftHandIndex1.rotation':{x:-0.018, y:0.007, z:0.28}, 'LeftHandIndex2.rotation':{x:0.247, y:-0.003, z:-0.025}, 'LeftHandIndex3.rotation':{x:0.13, y:-0.001, z:-0.013}, 'LeftHandMiddle1.rotation':{x:0.333, y:-0.015, z:0.182}, 'LeftHandMiddle2.rotation':{x:0.313, y:-0.005, z:-0.032}, 'LeftHandMiddle3.rotation':{x:0.294, y:-0.004, z:-0.03}, 'LeftHandRing1.rotation':{x:0.456, y:-0.028, z:-0.092}, 'LeftHandRing2.rotation':{x:0.53, y:-0.014, z:-0.052}, 'LeftHandRing3.rotation':{x:0.478, y:-0.012, z:-0.047}, 'LeftHandPinky1.rotation':{x:0.647, y:-0.049, z:-0.184}, 'LeftHandPinky2.rotation':{x:0.29, y:-0.004, z:-0.029}, 'LeftHandPinky3.rotation':{x:0.501, y:-0.013, z:-0.049} - }, - 'shrug': { - 'Neck.rotation':{x:[-0.3,0.3,1,2], y:[-0.3,0.3,1,2], z:[-0.1,0.1]}, 'Head.rotation':{x:[-0.3,0.3], y:[-0.3,0.3], z:[-0.1,0.1]}, 'RightShoulder.rotation':{x:1.732, y:-0.058, z:1.407}, 'RightArm.rotation':{x:1.305, y:0.46, z:0.118}, 'RightForeArm.rotation':{x:[0,2.0], y:[-1,0.2], z:-1.637}, 'RightHand.rotation':{x:-0.048, y:0.165, z:-0.39}, 'RightHandThumb1.rotation':{x:1.467, y:0.599, z:-1.315}, 'RightHandThumb2.rotation':{x:-0.255, y:-0.123, z:0.119}, 'RightHandThumb3.rotation':{x:0, y:-0.002, z:0}, 'RightHandIndex1.rotation':{x:-0.293, y:-0.066, z:-0.112}, 'RightHandIndex2.rotation':{x:0.181, y:0.007, z:0.069}, 'RightHandIndex3.rotation':{x:0, y:0, z:0}, 'RightHandMiddle1.rotation':{x:-0.063, y:-0.041, z:0.032}, 'RightHandMiddle2.rotation':{x:0.149, y:0.005, z:0.05}, 'RightHandMiddle3.rotation':{x:0, y:0, z:0}, 'RightHandRing1.rotation':{x:0.152, y:-0.03, z:0.132}, 'RightHandRing2.rotation':{x:0.194, y:0.007, z:0.058}, 'RightHandRing3.rotation':{x:0, y:0, z:0}, 'RightHandPinky1.rotation':{x:0.306, y:-0.015, z:0.257}, 'RightHandPinky2.rotation':{x:0.15, y:-0.003, z:-0.003}, 'RightHandPinky3.rotation':{x:0, y:0, z:0}, 'LeftShoulder.rotation':{x:1.713, y:0.141, z:-1.433}, 'LeftArm.rotation':{x:1.136, y:-0.422, z:-0.416}, 'LeftForeArm.rotation':{x:1.42, y:0.123, z:1.506}, 'LeftHand.rotation':{x:0.073, y:-0.138, z:0.064}, 'LeftHandThumb1.rotation':{x:1.467, y:-0.599, z:1.314}, 'LeftHandThumb2.rotation':{x:-0.255, y:0.123, z:-0.119}, 'LeftHandThumb3.rotation':{x:0, y:0.001, z:0}, 'LeftHandIndex1.rotation':{x:-0.293, y:0.066, z:0.112}, 'LeftHandIndex2.rotation':{x:0.181, y:-0.007, z:-0.069}, 'LeftHandIndex3.rotation':{x:0, y:0, z:0}, 'LeftHandMiddle1.rotation':{x:-0.062, y:0.041, z:-0.032}, 'LeftHandMiddle2.rotation':{x:0.149, y:-0.005, z:-0.05}, 'LeftHandMiddle3.rotation':{x:0, y:0, z:0}, 'LeftHandRing1.rotation':{x:0.152, y:0.03, z:-0.132}, 'LeftHandRing2.rotation':{x:0.194, y:-0.007, z:-0.058}, 'LeftHandRing3.rotation':{x:0, y:0, z:0}, 'LeftHandPinky1.rotation':{x:0.306, y:0.015, z:-0.257}, 'LeftHandPinky2.rotation':{x:0.15, y:0.003, z:0.003}, 'LeftHandPinky3.rotation':{x:0, y:0, z:0} - }, - 'namaste': { - 'RightShoulder.rotation':{x:1.758, y:0.099, z:1.604}, 'RightArm.rotation':{x:0.862, y:-0.292, z:-0.932}, 'RightForeArm.rotation':{x:0.083, y:0.066, z:-1.791}, 'RightHand.rotation':{x:-0.52, y:-0.001, z:-0.176}, 'RightHandThumb1.rotation':{x:0.227, y:0.418, z:-0.776}, 'RightHandThumb2.rotation':{x:-0.011, y:-0.003, z:0.171}, 'RightHandThumb3.rotation':{x:-0.041, y:-0.001, z:-0.013}, 'RightHandIndex1.rotation':{x:-0.236, y:0.003, z:-0.028}, 'RightHandIndex2.rotation':{x:0.004, y:0, z:0.001}, 'RightHandIndex3.rotation':{x:0.002, y:0, z:0}, 'RightHandMiddle1.rotation':{x:-0.236, y:0.003, z:-0.028}, 'RightHandMiddle2.rotation':{x:0.004, y:0, z:0.001}, 'RightHandMiddle3.rotation':{x:0.002, y:0, z:0}, 'RightHandRing1.rotation':{x:-0.236, y:0.003, z:-0.028}, 'RightHandRing2.rotation':{x:0.004, y:0, z:0.001}, 'RightHandRing3.rotation':{x:0.002, y:0, z:0}, 'RightHandPinky1.rotation':{x:-0.236, y:0.003, z:-0.028}, 'RightHandPinky2.rotation':{x:0.004, y:0, z:0.001}, 'RightHandPinky3.rotation':{x:0.002, y:0, z:0}, 'LeftShoulder.rotation':{x:1.711, y:-0.002, z:-1.625}, 'LeftArm.rotation':{x:0.683, y:0.334, z:0.977}, 'LeftForeArm.rotation':{x:0.086, y:-0.066, z:1.843}, 'LeftHand.rotation':{x:-0.595, y:-0.229, z:0.096}, 'LeftHandThumb1.rotation':{x:0.404, y:-0.05, z:0.537}, 'LeftHandThumb2.rotation':{x:-0.02, y:0.004, z:-0.154}, 'LeftHandThumb3.rotation':{x:-0.049, y:0.002, z:-0.019}, 'LeftHandIndex1.rotation':{x:-0.113, y:-0.001, z:0.014}, 'LeftHandIndex2.rotation':{x:0.003, y:0, z:0}, 'LeftHandIndex3.rotation':{x:0.002, y:0, z:0}, 'LeftHandMiddle1.rotation':{x:-0.113, y:-0.001, z:0.014}, 'LeftHandMiddle2.rotation':{x:0.004, y:0, z:0}, 'LeftHandMiddle3.rotation':{x:0.002, y:0, z:0}, 'LeftHandRing1.rotation':{x:-0.113, y:-0.001, z:0.014}, 'LeftHandRing2.rotation':{x:0.003, y:0, z:0}, 'LeftHandRing3.rotation':{x:0.002, y:0, z:0}, 'LeftHandPinky1.rotation':{x:-0.122, y:-0.001, z:-0.057}, 'LeftHandPinky2.rotation':{x:0.012, y:0.001, z:0.07}, 'LeftHandPinky3.rotation':{x:0.002, y:0, z:0} - } - } - - - // Pose deltas - // NOTE: In this object (x,y,z) are always Euler rotations despite the name!! - // NOTE: This object should include all the used delta properties. - this.poseDelta = { - props: { - 'Hips.quaternion':{x:0, y:0, z:0},'Spine.quaternion':{x:0, y:0, z:0}, - 'Spine1.quaternion':{x:0, y:0, z:0}, 'Neck.quaternion':{x:0, y:0, z:0}, - 'Head.quaternion':{x:0, y:0, z:0}, 'Spine1.scale':{x:0, y:0, z:0}, - 'Neck.scale':{x:0, y:0, z:0}, 'LeftArm.scale':{x:0, y:0, z:0}, - 'RightArm.scale':{x:0, y:0, z:0} - } - }; - // Add legs, arms and hands - ['Left','Right'].forEach( x => { - ['Leg','UpLeg','Arm','ForeArm','Hand'].forEach( y => { - this.poseDelta.props[x+y+'.quaternion'] = {x:0, y:0, z:0}; - }); - ['HandThumb', 'HandIndex','HandMiddle','HandRing', 'HandPinky'].forEach( y => { - this.poseDelta.props[x+y+'1.quaternion'] = {x:0, y:0, z:0}; - this.poseDelta.props[x+y+'2.quaternion'] = {x:0, y:0, z:0}; - this.poseDelta.props[x+y+'3.quaternion'] = {x:0, y:0, z:0}; - }); - }) - - // Dynamically pick up all the property names that we need in the code - const names = new Set(); - Object.values(this.poseTemplates).forEach( x => { - Object.keys( this.propsToThreeObjects(x.props) ).forEach( y => names.add(y) ); - }); - Object.keys( this.poseDelta.props ).forEach( x => { - names.add(x) - }); - this.posePropNames = [...names]; - - // Use "side" as the first pose, weight on left leg - this.poseName = "side"; // First pose - this.poseWeightOnLeft = true; // Initial weight on left leg - this.gesture = null; // Values that override pose properties - this.poseCurrentTemplate = this.poseTemplates[this.poseName]; - this.poseBase = this.poseFactory( this.poseCurrentTemplate ); - this.poseTarget = this.poseFactory( this.poseCurrentTemplate ); - this.poseStraight = this.propsToThreeObjects( this.poseTemplates["straight"].props ); // Straight pose used as a reference - this.poseAvatar = null; // Set when avatar has been loaded - - // Avatar height in meters - // NOTE: The actual value is calculated based on the eye level on avatar load - this.avatarHeight = 1.7; - - - // Animation templates - // - // baseline: Describes morph target baseline. Values can be either float or - // an array [start,end,skew] describing a probability distribution. - // speech : Describes voice rate, pitch and volume as deltas to the values - // set as options. - // anims : Animations for breathing, pose, etc. To be used animation - // sequence is selected in the following order: - // 1. State (idle, speaking, listening) - // 2. Mood (moodX, moodY) - // 3. Pose (poseX, poseY) - // 5. View (full, upper, head) - // 6. Body form ('M','F') - // 7. Alt (sequence of objects with propabilities p. If p is not - // specified, the remaining part is shared equivally among - // the rest.) - // 8. Current object - // object : delay, delta times dt and values vs. - // - - this.animTemplateEyes = { name: 'eyes', - idle: { alt: [ - { - p: () => ( this.avatar?.hasOwnProperty('avatarIdleEyeContact') ? this.avatar.avatarIdleEyeContact : this.opt.avatarIdleEyeContact ), - delay: [200,5000], dt: [ 200,[2000,5000],[3000,10000,1,2] ], - vs: { - headMove: [ this.avatar?.hasOwnProperty('avatarIdleHeadMove') ? this.avatar.avatarIdleHeadMove : this.opt.avatarIdleHeadMove ], - eyesRotateY: [[-0.6,0.6]], eyesRotateX: [[-0.2,0.6]], - eyeContact: [null,1] - } - }, - { - delay: [200,5000], dt: [ 200,[2000,5000,1,2] ], vs: { - headMove: [ this.avatar?.hasOwnProperty('avatarIdleHeadMove') ? this.avatar.avatarIdleHeadMove : this.opt.avatarIdleHeadMove ], - eyesRotateY: [[-0.6,0.6]], eyesRotateX: [[-0.2,0.6]] - } - } - ]}, - speaking: { alt: [ - { - p: () => ( this.avatar?.hasOwnProperty('avatarSpeakingEyeContact') ? this.avatar.avatarSpeakingEyeContact : this.opt.avatarSpeakingEyeContact ), - delay: [200,5000], dt: [ 0, [3000,10000,1,2], [2000,5000] ], - vs: { eyeContact: [1,null], - headMove: [null,( this.avatar?.hasOwnProperty('avatarSpeakingHeadMove') ? this.avatar.avatarSpeakingHeadMove : this.opt.avatarSpeakingHeadMove ),null], - eyesRotateY: [null,[-0.6,0.6]], eyesRotateX: [null,[-0.2,0.6]] - } - }, - { - delay: [200,5000], dt: [ 200,[2000,5000,1,2] ], vs: { - headMove: [( this.avatar?.hasOwnProperty('avatarSpeakingHeadMove') ? this.avatar.avatarSpeakingHeadMove : this.opt.avatarSpeakingHeadMove ),null], - eyesRotateY: [[-0.6,0.6]], eyesRotateX: [[-0.2,0.6]] - } - } - ]} - }; - this.animTemplateBlink = { name: 'blink', alt: [ - { p: 0.85, delay: [1000,8000,1,2], dt: [50,[100,300],100], vs: { eyeBlinkLeft: [1,1,0], eyeBlinkRight: [1,1,0] } }, - { delay: [1000,4000,1,2], dt: [50,[100,200],100,[10,400,0],50,[100,200],100], vs: { eyeBlinkLeft: [1,1,0,0,1,1,0], eyeBlinkRight: [1,1,0,0,1,1,0] } } - ]}; - - this.animMoods = { - 'neutral' : { - baseline: { eyesLookDown: 0.1 }, - speech: { deltaRate: 0, deltaPitch: 0, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 1500, dt: [ 1200,500,1000 ], vs: { chestInhale: [0.5,0.5,0] } }, - { name: 'pose', alt: [ - { p: 0.5, delay: [5000,30000], vs: { pose: ['side'] } }, - { p: 0.3, delay: [5000,30000], vs: { pose: ['hip'] }, - 'M': { delay: [5000,30000], vs: { pose: ['wide'] } } - }, - { delay: [5000,30000], vs: { pose: ['straight'] } } - ]}, - { name: 'head', - idle: { delay: [0,1000], dt: [ [200,5000] ], vs: { bodyRotateX: [[-0.04,0.10]], bodyRotateY: [[-0.3,0.3]], bodyRotateZ: [[-0.08,0.08]] } }, - speaking: { dt: [ [0,1000,0] ], vs: { bodyRotateX: [[-0.05,0.15,1,2]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.1,0.1]] } } - }, - this.animTemplateEyes, - this.animTemplateBlink, - { name: 'mouth', delay: [1000,5000], dt: [ [100,500],[100,5000,2] ], vs : { mouthRollLower: [[0,0.3,2]], mouthRollUpper: [[0,0.3,2]], mouthStretchLeft: [[0,0.3]], mouthStretchRight: [[0,0.3]], mouthPucker: [[0,0.3]] } }, - { name: 'misc', delay: [100,5000], dt: [ [100,500],[1000,5000,2] ], vs : { eyeSquintLeft: [[0,0.3,2]], eyeSquintRight: [[0,0.3,2]], browInnerUp: [[0,0.3,2]], browOuterUpLeft: [[0,0.3,2]], browOuterUpRight: [[0,0.3,2]] } } - ] - }, - 'happy' : { - baseline: { mouthSmile: 0.2, eyesLookDown: 0.1 }, - speech: { deltaRate: 0, deltaPitch: 0.1, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 1500, dt: [ 1200,500,1000 ], vs: { chestInhale: [0.5,0.5,0] } }, - { name: 'pose', - idle: { - alt: [ - { p: 0.6, delay: [5000,30000], vs: { pose: ['side'] } }, - { p: 0.2, delay: [5000,30000], vs: { pose: ['hip'] }, - 'M': { delay: [5000,30000], vs: { pose: ['side'] } } - }, - { p: 0.1, delay: [5000,30000], vs: { pose: ['straight'] } }, - { delay: [5000,10000], vs: { pose: ['wide'] } }, - { delay: [1000,3000], vs: { pose: ['turn'] } }, - ] - }, - speaking: { - alt: [ - { p: 0.4, delay: [5000,30000], vs: { pose: ['side'] } }, - { p: 0.4, delay: [5000,30000], vs: { pose: ['straight'] } }, - { delay: [5000,20000], vs: { pose: ['hip'] }, - 'M': { delay: [5000,30000], vs: { pose: ['wide'] } } - }, - ] - } - }, - { name: 'head', - idle: { dt: [ [1000,5000] ], vs: { bodyRotateX: [[-0.04,0.10]], bodyRotateY: [[-0.3,0.3]], bodyRotateZ: [[-0.08,0.08]] } }, - speaking: { dt: [ [0,1000,0] ], vs: { bodyRotateX: [[-0.05,0.15,1,2]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.1,0.1]] } } - }, - this.animTemplateEyes, - this.animTemplateBlink, - { name: 'mouth', delay: [1000,5000], dt: [ [100,500],[100,5000,2] ], vs : { mouthLeft: [[0,0.3,2]], mouthSmile: [[0,0.2,3]], mouthRollLower: [[0,0.3,2]], mouthRollUpper: [[0,0.3,2]], mouthStretchLeft: [[0,0.3]], mouthStretchRight: [[0,0.3]], mouthPucker: [[0,0.3]] } }, - { name: 'misc', delay: [100,5000], dt: [ [100,500],[1000,5000,2] ], vs : { eyeSquintLeft: [[0,0.3,2]], eyeSquintRight: [[0,0.3,2]], browInnerUp: [[0,0.3,2]], browOuterUpLeft: [[0,0.3,2]], browOuterUpRight: [[0,0.3,2]] } } - ] - }, - 'angry' : { - baseline: { eyesLookDown: 0.1, browDownLeft: 0.6, browDownRight: 0.6, jawForward: 0.3, mouthFrownLeft: 0.7, mouthFrownRight: 0.7, mouthRollLower: 0.2, mouthShrugLower: 0.3, handFistLeft: 1, handFistRight: 1 }, - speech: { deltaRate: -0.2, deltaPitch: 0.2, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 500, dt: [ 1000,500,1000 ], vs: { chestInhale: [0.7,0.7,0] } }, - { name: 'pose', alt: [ - { p: 0.4, delay: [5000,30000], vs: { pose: ['side'] } }, - { p: 0.4, delay: [5000,30000], vs: { pose: ['straight'] } }, - { p: 0.2, delay: [5000,30000], vs: { pose: ['hip'] }, - 'M': { delay: [5000,30000], vs: { pose: ['wide'] } } - }, - ]}, - { name: 'head', - idle: { delay: [100,500], dt: [ [200,5000] ], vs: { bodyRotateX: [[-0.04,0.10]], bodyRotateY: [[-0.2,0.2]], bodyRotateZ: [[-0.08,0.08]] } }, - speaking: { dt: [ [0,1000,0] ], vs: { bodyRotateX: [[-0.05,0.15,1,2]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.1,0.1]] } } - }, - this.animTemplateEyes, - this.animTemplateBlink, - { name: 'mouth', delay: [1000,5000], dt: [ [100,500],[100,5000,2] ], vs : { mouthRollLower: [[0,0.3,2]], mouthRollUpper: [[0,0.3,2]], mouthStretchLeft: [[0,0.3]], mouthStretchRight: [[0,0.3]], mouthPucker: [[0,0.3]] } }, - { name: 'misc', delay: [100,5000], dt: [ [100,500],[1000,5000,2] ], vs : { eyeSquintLeft: [[0,0.3,2]], eyeSquintRight: [[0,0.3,2]], browInnerUp: [[0,0.3,2]], browOuterUpLeft: [[0,0.3,2]], browOuterUpRight: [[0,0.3,2]] } } - ] - }, - 'sad' : { - baseline: { eyesLookDown: 0.2, browDownRight: 0.1, browInnerUp: 0.6, browOuterUpRight: 0.2, eyeSquintLeft: 0.7, eyeSquintRight: 0.7, mouthFrownLeft: 0.8, mouthFrownRight: 0.8, mouthLeft: 0.2, mouthPucker: 0.5, mouthRollLower: 0.2, mouthRollUpper: 0.2, mouthShrugLower: 0.2, mouthShrugUpper: 0.2, mouthStretchLeft: 0.4 }, - speech: { deltaRate: -0.2, deltaPitch: -0.2, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 1500, dt: [ 1000,500,1000 ], vs: { chestInhale: [0.3,0.3,0] } }, - { name: 'pose', alt: [ - { p: 0.4, delay: [5000,30000], vs: { pose: ['side'] } }, - { p: 0.4, delay: [5000,30000], vs: { pose: ['straight'] } }, - { delay: [5000,20000], vs: { pose: ['side'] }, - full: { delay: [5000,20000], vs: { pose: ['oneknee'] } } - }, - ]}, - { name: 'head', - idle: { delay: [100,500], dt: [ [200,5000] ], vs: { bodyRotateX: [[-0.04,0.10]], bodyRotateY: [[-0.2,0.2]], bodyRotateZ: [[-0.08,0.08]] } }, - speaking: { dt: [ [0,1000,0] ], vs: { bodyRotateX: [[-0.05,0.15,1,2]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.1,0.1]] } } - }, - this.animTemplateEyes, - this.animTemplateBlink, - { name: 'mouth', delay: [1000,5000], dt: [ [100,500],[100,5000,2] ], vs : { mouthRollLower: [[0,0.3,2]], mouthRollUpper: [[0,0.3,2]], mouthStretchLeft: [[0,0.3]], mouthStretchRight: [[0,0.3]], mouthPucker: [[0,0.3]] } }, - { name: 'misc', delay: [100,5000], dt: [ [100,500],[1000,5000,2] ], vs : { eyeSquintLeft: [[0,0.3,2]], eyeSquintRight: [[0,0.3,2]], browInnerUp: [[0,0.3,2]], browOuterUpLeft: [[0,0.3,2]], browOuterUpRight: [[0,0.3,2]] } } - ] - }, - 'fear' : { - baseline: { browInnerUp: 0.7, eyeSquintLeft: 0.5, eyeSquintRight: 0.5, eyeWideLeft: 0.6, eyeWideRight: 0.6, mouthClose: 0.1, mouthFunnel: 0.3, mouthShrugLower: 0.5, mouthShrugUpper: 0.5 }, - speech: { deltaRate: -0.2, deltaPitch: 0, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 500, dt: [ 1000,500,1000 ], vs: { chestInhale: [0.7,0.7,0] } }, - { name: 'pose', alt: [ - { p: 0.8, delay: [5000,30000], vs: { pose: ['side'] } }, - { delay: [5000,30000], vs: { pose: ['straight'] } }, - { delay: [5000,20000], vs: { pose: ['wide'] } }, - { delay: [5000,20000], vs: { pose: ['side'] }, - full: { delay: [5000,20000], vs: { pose: ['oneknee'] } } - }, - ]}, - { name: 'head', - idle: { delay: [100,500], dt: [ [200,3000] ], vs: { bodyRotateX: [[-0.06,0.12]], bodyRotateY: [[-0.7,0.7]], bodyRotateZ: [[-0.1,0.1]] } }, - speaking: { dt: [ [0,1000,0] ], vs: { bodyRotateX: [[-0.05,0.15,1,2]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.1,0.1]] } } - }, - this.animTemplateEyes, - this.animTemplateBlink, - { name: 'mouth', delay: [1000,5000], dt: [ [100,500],[100,5000,2] ], vs : { mouthRollLower: [[0,0.3,2]], mouthRollUpper: [[0,0.3,2]], mouthStretchLeft: [[0,0.3]], mouthStretchRight: [[0,0.3]], mouthPucker: [[0,0.3]] } }, - { name: 'misc', delay: [100,5000], dt: [ [100,500],[1000,5000,2] ], vs : { eyeSquintLeft: [[0,0.3,2]], eyeSquintRight: [[0,0.3,2]], browInnerUp: [[0,0.3,2]], browOuterUpLeft: [[0,0.3,2]], browOuterUpRight: [[0,0.3,2]] } } - ] - }, - 'disgust' : { - baseline: { browDownLeft: 0.7, browDownRight: 0.1, browInnerUp: 0.3, eyeSquintLeft: 1, eyeSquintRight: 1, eyeWideLeft: 0.5, eyeWideRight: 0.5, eyesRotateX: 0.05, mouthLeft: 0.4, mouthPressLeft: 0.3, mouthRollLower: 0.3, mouthShrugLower: 0.3, mouthShrugUpper: 0.8, mouthUpperUpLeft: 0.3, noseSneerLeft: 1, noseSneerRight: 0.7 }, - speech: { deltaRate: -0.2, deltaPitch: 0, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 1500, dt: [ 1000,500,1000 ], vs: { chestInhale: [0.5,0.5,0] } }, - { name: 'pose', alt: [ - { delay: [5000,20000], vs: { pose: ['side'] } }, - ]}, - { name: 'head', - idle: { delay: [100,500], dt: [ [200,5000] ], vs: { bodyRotateX: [[-0.04,0.10]], bodyRotateY: [[-0.2,0.2]], bodyRotateZ: [[-0.08,0.08]] } }, - speaking: { dt: [ [0,1000,0] ], vs: { bodyRotateX: [[-0.05,0.15,1,2]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.1,0.1]] } } - }, - this.animTemplateEyes, - this.animTemplateBlink, - { name: 'mouth', delay: [1000,5000], dt: [ [100,500],[100,5000,2] ], vs : { mouthRollLower: [[0,0.3,2]], mouthRollUpper: [[0,0.3,2]], mouthStretchLeft: [[0,0.3]], mouthStretchRight: [[0,0.3]], mouthPucker: [[0,0.3]] } }, - { name: 'misc', delay: [100,5000], dt: [ [100,500],[1000,5000,2] ], vs : { eyeSquintLeft: [[0,0.3,2]], eyeSquintRight: [[0,0.3,2]], browInnerUp: [[0,0.3,2]], browOuterUpLeft: [[0,0.3,2]], browOuterUpRight: [[0,0.3,2]] } } - ] - }, - 'love' : { - baseline: { browInnerUp: 0.4, browOuterUpLeft: 0.2, browOuterUpRight: 0.2, mouthSmile: 0.2, eyeBlinkLeft: 0.6, eyeBlinkRight: 0.6, eyeWideLeft: 0.7, eyeWideRight: 0.7, bodyRotateX: 0.1, mouthDimpleLeft: 0.1, mouthDimpleRight: 0.1, mouthPressLeft: 0.2, mouthShrugUpper: 0.2, mouthUpperUpLeft: 0.1, mouthUpperUpRight: 0.1 }, - speech: { deltaRate: -0.1, deltaPitch: -0.7, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 1500, dt: [ 1500,500,1500 ], vs: { chestInhale: [0.8,0.8,0] } }, - { name: 'pose', alt: [ - { p: 0.4, delay: [5000,30000], vs: { pose: ['side'] } }, - { p: 0.2, delay: [5000,30000], vs: { pose: ['straight'] } }, - { p: 0.2, delay: [5000,30000], vs: { pose: ['hip'] }, - 'M': { delay: [5000,30000], vs: { pose: ['side'] } } - }, - { delay: [5000,10000], vs: { pose: ['side'] }, - full: { delay: [5000,10000], vs: { pose: ['kneel'] } } - }, - { delay: [1000,3000], vs: { pose: ['turn'] }, - 'M': { delay: [1000,3000], vs: { pose: ['wide'] } } - }, - { delay: [1000,3000], vs: { pose: ['back'] }, - 'M': { delay: [1000,3000], vs: { pose: ['wide'] } } - }, - { delay: [5000,20000], vs: { pose: ['side'] }, - 'M': { delay: [5000,20000], vs: { pose: ['side'] } }, - full: { delay: [5000,20000], vs: { pose: ['bend'] } } - }, - { delay: [1000,3000], vs: { pose: ['side'] }, - full: { delay: [5000,10000], vs: { pose: ['oneknee'] } } - }, - ]}, - { name: 'head', - idle: { dt: [ [1000,5000] ], vs: { bodyRotateX: [[-0.04,0.10]], bodyRotateY: [[-0.3,0.3]], bodyRotateZ: [[-0.08,0.08]] } }, - speaking: { dt: [ [0,1000,0] ], vs: { bodyRotateX: [[-0.05,0.15,1,2]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.1,0.1]] } } - }, - this.animTemplateEyes, - this.deepCopy(this.animTemplateBlink,(o) => { o.alt[0].delay[0] = o.alt[1].delay[0] = 2000; }), - { name: 'mouth', delay: [1000,5000], dt: [ [100,500],[100,5000,2] ], vs : { mouthLeft: [[0,0.3,2]], mouthRollLower: [[0,0.3,2]], mouthRollUpper: [[0,0.3,2]], mouthStretchLeft: [[0,0.3]], mouthStretchRight: [[0,0.3]], mouthPucker: [[0,0.3]] } }, - { name: 'misc', delay: [100,5000], dt: [ [500,1000],[1000,5000,2] ], vs : { eyeSquintLeft: [[0,0.3,2]], eyeSquintRight: [[0,0.3,2]], browInnerUp: [[0.3,0.6,2]], browOuterUpLeft: [[0.1,0.3,2]], browOuterUpRight: [[0.1,0.3,2]] } } - ] - }, - 'sleep' : { - baseline: { eyeBlinkLeft: 1, eyeBlinkRight: 1, eyesClosed: 0.6 }, - speech: { deltaRate: 0, deltaPitch: -0.2, deltaVolume: 0 }, - anims: [ - { name: 'breathing', delay: 1500, dt: [ 1000,500,1000 ], vs: { chestInhale: [0.6,0.6,0] } }, - { name: 'pose', alt: [ - { delay: [5000,20000], vs: { pose: ['side'] } } - ]}, - { name: 'head', delay: [1000,5000], dt: [ [2000,10000] ], vs: { bodyRotateX: [[0,0.4]], bodyRotateY: [[-0.1,0.1]], bodyRotateZ: [[-0.04,0.04]] } }, - { name: 'eyes', delay: 10010, dt: [], vs: {} }, - { name: 'blink', delay: 10020, dt: [], vs: {} }, - { name: 'mouth', delay: 10030, dt: [], vs: {} }, - { name: 'misc', delay: 10040, dt: [], vs: {} } - ] - } - }; - this.moodName = this.opt.avatarMood || "neutral"; - this.mood = this.animMoods[ this.moodName ]; - if ( !this.mood ) { - this.moodName = "neutral"; - this.mood = this.animMoods["neutral"]; - } - - // Animation templates for emojis - this.animEmojis = { - - '😐': { dt: [300,2000], rescale: [0,1], vs: { pose: ['straight'], browInnerUp: [0.4], eyeWideLeft: [0.7], eyeWideRight: [0.7], mouthPressLeft: [0.6], mouthPressRight: [0.6], mouthRollLower: [0.3], mouthStretchLeft: [1], mouthStretchRight: [1] } }, - '😶': { link: '😐' }, - '😏': { dt: [300,2000], rescale: [0,1], vs: { eyeContact: [0], browDownRight: [0.1], browInnerUp: [0.7], browOuterUpRight: [0.2], eyeLookInRight: [0.7], eyeLookOutLeft: [0.7], eyeSquintLeft: [1], eyeSquintRight: [0.8], eyesRotateY: [0.7], mouthLeft: [0.4], mouthPucker: [0.4], mouthShrugLower: [0.3], mouthShrugUpper: [0.2], mouthSmile: [0.2], mouthSmileLeft: [0.4], mouthSmileRight: [0.2], mouthStretchLeft: [0.5], mouthUpperUpLeft: [0.6], noseSneerLeft: [0.7] } }, - '🙂': { dt: [300,2000], rescale: [0,1], vs: { mouthSmile: [0.5] } }, - '🙃': { link: '🙂' }, - '😊': { dt: [300,2000], rescale: [0,1], vs: { browInnerUp: [0.6], eyeSquintLeft: [1], eyeSquintRight: [1], mouthSmile: [0.7], noseSneerLeft: [0.7], noseSneerRight: [0.7]} }, - '😇': { link: '😊' }, - '😀': { dt: [300,2000], rescale: [0,1], vs: { browInnerUp: [0.6], jawOpen: [0.1], mouthDimpleLeft: [0.2], mouthDimpleRight: [0.2], mouthOpen: [0.3], mouthPressLeft: [0.3], mouthPressRight: [0.3], mouthRollLower: [0.4], mouthShrugUpper: [0.4], mouthSmile: [0.7], mouthUpperUpLeft: [0.3], mouthUpperUpRight: [0.3], noseSneerLeft: [0.4], noseSneerRight: [0.4] }}, - '😃': { dt: [300,2000], rescale: [0,1], vs: { browInnerUp: [0.6], eyeWideLeft: [0.7], eyeWideRight: [0.7], jawOpen: [0.1], mouthDimpleLeft: [0.2], mouthDimpleRight: [0.2], mouthOpen: [0.3], mouthPressLeft: [0.3], mouthPressRight: [0.3], mouthRollLower: [0.4], mouthShrugUpper: [0.4], mouthSmile: [0.7], mouthUpperUpLeft: [0.3], mouthUpperUpRight: [0.3], noseSneerLeft: [0.4], noseSneerRight: [0.4] } }, - '😄': { dt: [300,2000], rescale: [0,1], vs: { browInnerUp: [0.3], eyeSquintLeft: [1], eyeSquintRight: [1], jawOpen: [0.2], mouthDimpleLeft: [0.2], mouthDimpleRight: [0.2], mouthOpen: [0.3], mouthPressLeft: [0.3], mouthPressRight: [0.3], mouthRollLower: [0.4], mouthShrugUpper: [0.4], mouthSmile: [0.7], mouthUpperUpLeft: [0.3], mouthUpperUpRight: [0.3], noseSneerLeft: [0.4], noseSneerRight: [0.4] } }, - '😁': { dt: [300,2000], rescale: [0,1], vs: { browInnerUp: [0.3], eyeSquintLeft: [1], eyeSquintRight: [1], jawOpen: [0.3], mouthDimpleLeft: [0.2], mouthDimpleRight: [0.2], mouthPressLeft: [0.5], mouthPressRight: [0.5], mouthShrugUpper: [0.4], mouthSmile: [0.7], mouthUpperUpLeft: [0.3], mouthUpperUpRight: [0.3], noseSneerLeft: [0.4], noseSneerRight: [0.4] } }, - '😆': { dt: [300,2000], rescale: [0,1], vs: { browInnerUp: [0.3], eyeSquintLeft: [1], eyeSquintRight: [1], eyesClosed: [0.6], jawOpen: [0.3], mouthDimpleLeft: [0.2], mouthDimpleRight: [0.2], mouthPressLeft: [0.5], mouthPressRight: [0.5], mouthShrugUpper: [0.4], mouthSmile: [0.7], mouthUpperUpLeft: [0.3], mouthUpperUpRight: [0.3], noseSneerLeft: [0.4], noseSneerRight: [0.4] } }, - '😝': { dt: [300,100,1500,500,500], rescale: [0,0,1,0,0], vs: { browInnerUp: [0.8], eyesClosed: [1], jawOpen: [0.7], mouthFunnel: [0.5], mouthSmile: [1], tongueOut: [0,1,1,0] } }, - '😋': { link: '😝' }, '😛': { link: '😝' }, '😛': { link: '😝' }, '😜': { link: '😝' }, '🤪': { link: '😝' }, - '😂': { dt: [300,2000], rescale: [0,1], vs: { browInnerUp: [0.3], eyeSquintLeft: [1], eyeSquintRight: [1], eyesClosed: [0.6], jawOpen: [0.3], mouthDimpleLeft: [0.2], mouthDimpleRight: [0.2], mouthPressLeft: [0.5], mouthPressRight: [0.5], mouthShrugUpper: [0.4], mouthSmile: [0.7], mouthUpperUpLeft: [0.3], mouthUpperUpRight: [0.3], noseSneerLeft: [0.4], noseSneerRight: [0.4] } }, - '🤣': { link: '😂' }, '😅': { link: '😂' }, - '😉': { dt: [500,200,500,500], rescale: [0,0,0,1], vs: { mouthSmile: [0.5], mouthOpen: [0.2], mouthSmileLeft: [0,0.5,0], eyeBlinkLeft: [0,0.7,0], eyeBlinkRight: [0,0,0], bodyRotateX: [0.05,0.05,0.05,0], bodyRotateZ: [-0.05,-0.05,-0.05,0], browDownLeft: [0,0.7,0], cheekSquintLeft: [0,0.7,0], eyeSquintLeft: [0,1,0], eyesClosed: [0] } }, - - '😭': { dt: [1000,1000], rescale: [0,1], vs: { browInnerUp: [1], eyeSquintLeft: [1], eyeSquintRight: [1], eyesClosed: [0.1], jawOpen: [0], mouthFrownLeft: [1], mouthFrownRight: [1], mouthOpen: [0.5], mouthPucker: [0.5], mouthUpperUpLeft: [0.6], mouthUpperUpRight: [0.6] } }, - '🥺': { dt: [1000,1000], rescale: [0,1], vs: { browDownLeft: [0.2], browDownRight: [0.2], browInnerUp: [1], eyeWideLeft: [0.9], eyeWideRight: [0.9], eyesClosed: [0.1], mouthClose: [0.2], mouthFrownLeft: [1], mouthFrownRight: [1], mouthPressLeft: [0.4], mouthPressRight: [0.4], mouthPucker: [1], mouthRollLower: [0.6], mouthRollUpper: [0.2], mouthUpperUpLeft: [0.8], mouthUpperUpRight: [0.8] } }, - '😞': { dt: [1000,1000], rescale: [0,1], vs: { browInnerUp: [0.7], eyeSquintLeft: [1], eyeSquintRight: [1], eyesClosed: [0.5], bodyRotateX: [0.3], mouthClose: [0.2], mouthFrownLeft: [1], mouthFrownRight: [1], mouthPucker: [1], mouthRollLower: [1], mouthShrugLower: [0.2], mouthUpperUpLeft: [0.8], mouthUpperUpRight: [0.8] } }, - '😔': { dt: [1000,1000], rescale: [0,1], vs: { browInnerUp: [1], eyeSquintLeft: [1], eyeSquintRight: [1], eyesClosed: [0.5], bodyRotateX: [0.3], mouthClose: [0.2], mouthFrownLeft: [1], mouthFrownRight: [1], mouthPressLeft: [0.4], mouthPressRight: [0.4], mouthPucker: [1], mouthRollLower: [0.6], mouthRollUpper: [0.2], mouthUpperUpLeft: [0.8], mouthUpperUpRight: [0.8] } }, - '😳': { dt: [1000,1000], rescale: [0,1], vs: { browInnerUp: [1], eyeWideLeft: [0.5], eyeWideRight: [0.5], eyesRotateY: [0.05], eyesRotateX: [0.05], mouthClose: [0.2], mouthFunnel: [0.5], mouthPucker: [0.4], mouthRollLower: [0.4], mouthRollUpper: [0.4] } }, - '☹️': { dt: [500,1500], rescale: [0,1], vs: { mouthFrownLeft: [1], mouthFrownRight: [1], mouthPucker: [0.1], mouthRollLower: [0.8] } }, - - '😚': { dt: [500,1000,1000], rescale: [0,1,0], vs: { browInnerUp: [0.6], eyeBlinkLeft: [1], eyeBlinkRight: [1], eyeSquintLeft: [1], eyeSquintRight: [1], mouthPucker: [0,0.5], noseSneerLeft: [0,0.7], noseSneerRight: [0,0.7], viseme_U: [0,1] } }, - '😘': { dt: [500,500,200,500], rescale: [0,0,0,1], vs: { browInnerUp: [0.6], eyeBlinkLeft: [0,0,1,0], eyeBlinkRight: [0], eyesRotateY: [0], bodyRotateY: [0], bodyRotateX: [0,0.05,0.05,0], bodyRotateZ: [0,-0.05,-0.05,0], eyeSquintLeft: [1], eyeSquintRight: [1], mouthPucker: [0,0.5,0], noseSneerLeft: [0,0.7], noseSneerRight: [0.7], viseme_U: [0,1] } }, - '🥰': { dt: [1000,1000], rescale: [0,1], vs: { browInnerUp: [0.6], eyeSquintLeft: [1], eyeSquintRight: [1], mouthSmile: [0.7], noseSneerLeft: [0.7], noseSneerRight: [0.7] } }, - '😍': { dt: [1000,1000], rescale: [0,1], vs: { browInnerUp: [0.6], jawOpen: [0.1], mouthDimpleLeft: [0.2], mouthDimpleRight: [0.2], mouthOpen: [0.3], mouthPressLeft: [0.3], mouthPressRight: [0.3], mouthRollLower: [0.4], mouthShrugUpper: [0.4], mouthSmile: [0.7], mouthUpperUpLeft: [0.3], mouthUpperUpRight: [0.3], noseSneerLeft: [0.4], noseSneerRight: [0.4] } }, - '🤩': { link: '😍' }, - - '😡': { dt: [1000,1500], rescale: [0,1], vs: { browDownLeft: [1], browDownRight: [1], eyesLookUp: [0.2], jawForward: [0.3], mouthFrownLeft: [1], mouthFrownRight: [1], bodyRotateX: [0.15] } }, - '😠': { dt: [1000,1500], rescale: [0,1], vs: { browDownLeft: [1], browDownRight: [1], eyesLookUp: [0.2], jawForward: [0.3], mouthFrownLeft: [1], mouthFrownRight: [1], bodyRotateX: [0.15] } }, - '🤬': { link: '😠' }, - '😒': { dt: [1000,1000], rescale: [0,1], vs: { eyeContact: [0], browDownRight: [0.1], browInnerUp: [0.7], browOuterUpRight: [0.2], eyeLookInRight: [0.7], eyeLookOutLeft: [0.7], eyeSquintLeft: [1], eyeSquintRight: [0.8], eyesRotateY: [0.7], mouthFrownLeft: [1], mouthFrownRight: [1], mouthLeft: [0.2], mouthPucker: [0.5], mouthRollLower: [0.2], mouthRollUpper: [0.2], mouthShrugLower: [0.2], mouthShrugUpper: [0.2], mouthStretchLeft: [0.5] } }, - - '😱': { dt: [500,1500], rescale: [0,1], vs: { browInnerUp: [0.8], eyeWideLeft: [0.5], eyeWideRight: [0.5], jawOpen: [0.7], mouthFunnel: [0.5] } }, - '😬': { dt: [500,1500], rescale: [0,1], vs: { browDownLeft: [1], browDownRight: [1], browInnerUp: [1], mouthDimpleLeft: [0.5], mouthDimpleRight: [0.5], mouthLowerDownLeft: [1], mouthLowerDownRight: [1], mouthPressLeft: [0.4], mouthPressRight: [0.4], mouthPucker: [0.5], mouthSmile: [0.1], mouthSmileLeft: [0.2], mouthSmileRight: [0.2], mouthStretchLeft: [1], mouthStretchRight: [1], mouthUpperUpLeft: [1], mouthUpperUpRight: [1] } }, - '🙄': { dt: [500,1500], rescale: [0,1], vs: { browInnerUp: [0.8], eyeWideLeft: [1], eyeWideRight: [1], eyesRotateX: [-0.8], bodyRotateX: [0.15], mouthPucker: [0.5], mouthRollLower: [0.6], mouthRollUpper: [0.5], mouthShrugLower: [0], mouthSmile: [0] } }, - '🤔': { dt: [500,1500], rescale: [0,1], vs: { - browDownLeft: [1], browOuterUpRight: [1], eyeSquintLeft: [0.6], - mouthFrownLeft: [0.7], mouthFrownRight: [0.7], mouthLowerDownLeft: [0.3], - mouthPressRight: [0.4], mouthPucker: [0.1], mouthRight: [0.5], mouthRollLower: [0.5], - mouthRollUpper: [0.2], handRight: [{ x: 0.1, y: 0.1, z:0.1, d:1000 }, { d:1000 }], - handFistRight: [0.1] - } }, - '👀': { dt: [500,1500], rescale: [0,1], vs: { eyesRotateY: [-0.8] } }, - - '😴': { dt: [5000,5000], rescale: [0,1], vs:{ eyeBlinkLeft: [1], eyeBlinkRight: [1], bodyRotateX: [0.2], bodyRotateZ: [0.1] } }, - - '✋': { dt: [300,2000], rescale: [0,1], vs:{ mouthSmile: [0.5], gesture: [["handup",2,true],null] } }, - '🤚': { dt: [300,2000], rescale: [0,1], vs:{ mouthSmile: [0.5], gesture: [["handup",2],null] } }, - '👋': { link: '✋' }, - '👍': { dt: [300,2000], rescale: [0,1], vs:{ mouthSmile: [0.5], gesture: [["thumbup",2],null] } }, - '👎': { dt: [300,2000], rescale: [0,1], vs:{ browDownLeft: [1], browDownRight: [1], eyesLookUp: [0.2], jawForward: [0.3], mouthFrownLeft: [1], mouthFrownRight: [1], bodyRotateX: [0.15], gesture: [["thumbdown",2],null] } }, - '👌': { dt: [300,2000], rescale: [0,1], vs:{ mouthSmile: [0.5], gesture: [["ok",2],null] } }, - '🤷‍♂️': { dt: [1000,1500], rescale: [0,1], vs:{ gesture: [["shrug",2],null] } }, - '🤷‍♀️': { link: '🤷‍♂️' }, - '🤷': { link: '🤷‍♂️' }, - '🙏': { dt: [1500,300,1000], rescale: [0,1,0], vs:{ eyeBlinkLeft: [0,1], eyeBlinkRight: [0,1], bodyRotateX: [0], bodyRotateZ: [0.1], gesture: [["namaste",2],null] } }, - - 'yes': { dt: [[200,500],[200,500],[200,500],[200,500]], vs:{ headMove: [0], headRotateX: [[0.1,0.2],0.1,[0.1,0.2],0], headRotateZ: [[-0.2,0.2]] } }, - 'no': { dt: [[200,500],[200,500],[200,500],[200,500],[200,500]], vs:{ headMove: [0], headRotateY: [[-0.1,-0.05],[0.05,0.1],[-0.1,-0.05],[0.05,0.1],0], headRotateZ: [[-0.2,0.2]] } } - - }; - - // Morph targets - this.mtAvatar = {}; - this.mtCustoms = [ - "handFistLeft","handFistRight",'bodyRotateX', 'bodyRotateY', - 'bodyRotateZ', 'headRotateX', 'headRotateY', 'headRotateZ','chestInhale' - ]; - this.mtEasingDefault = this.sigmoidFactory(5); // Morph target default ease in/out - this.mtAccDefault = 0.01; // Acceleration [rad / s^2] - this.mtAccExceptions = { - eyeBlinkLeft: 0.1, eyeBlinkRight: 0.1, eyeLookOutLeft: 0.1, - eyeLookInLeft: 0.1, eyeLookOutRight: 0.1, eyeLookInRight: 0.1 - }; - this.mtMaxVDefault = 5; // Maximum velocity [rad / s] - this.mtMaxVExceptions = { - bodyRotateX: 1, bodyRotateY: 1, bodyRotateZ: 1, - // headRotateX: 1, headRotateY: 1, headRotateZ: 1 - }; - this.mtBaselineDefault = 0; // Default baseline value - this.mtBaselineExceptions = { - bodyRotateX: null, bodyRotateY: null, bodyRotateZ: null, - eyeLookOutLeft: null, eyeLookInLeft: null, eyeLookOutRight: null, - eyeLookInRight: null, eyesLookDown: null, eyesLookUp: null - }; - this.mtMinDefault = 0; - this.mtMinExceptions = { - bodyRotateX: -1, bodyRotateY: -1, bodyRotateZ: -1, - headRotateX: -1, headRotateY: -1, headRotateZ: -1 - }; - this.mtMaxDefault = 1; - this.mtMaxExceptions = {}; - this.mtLimits = { - eyeBlinkLeft: (v) => ( Math.max(v, ( this.mtAvatar['eyesLookDown'].value + this.mtAvatar['browDownLeft'].value ) / 2) ), - eyeBlinkRight: (v) => ( Math.max(v, ( this.mtAvatar['eyesLookDown'].value + this.mtAvatar['browDownRight'].value ) / 2 ) ) - }; - this.mtOnchange = { - eyesLookDown: () => { - this.mtAvatar['eyeBlinkLeft'].needsUpdate = true; - this.mtAvatar['eyeBlinkRight'].needsUpdate = true; - }, - browDownLeft: () => { this.mtAvatar['eyeBlinkLeft'].needsUpdate = true; }, - browDownRight: () => { this.mtAvatar['eyeBlinkRight'].needsUpdate = true; } - }; - this.mtRandomized = [ - 'mouthDimpleLeft','mouthDimpleRight', 'mouthLeft', 'mouthPressLeft', - 'mouthPressRight', 'mouthStretchLeft', 'mouthStretchRight', - 'mouthShrugLower', 'mouthShrugUpper', 'noseSneerLeft', 'noseSneerRight', - 'mouthRollLower', 'mouthRollUpper', 'browDownLeft', 'browDownRight', - 'browOuterUpLeft', 'browOuterUpRight', 'cheekPuff', 'cheekSquintLeft', - 'cheekSquintRight' - ]; - - // Anim queues - this.animQueue = []; - this.animClips = []; - this.animPoses = []; - - // Clock - this.animFrameDur = 1000/ this.opt.modelFPS; - this.animClock = 0; - this.animSlowdownRate = 1; - this.animTimeLast = 0; - this.easing = this.sigmoidFactory(5); // Ease in and out - - // Lip-sync extensions, import dynamically - this.lipsync = {}; - this.opt.lipsyncModules.forEach( x => this.lipsyncGetProcessor(x) ); - this.visemeNames = [ - 'aa', 'E', 'I', 'O', 'U', 'PP', 'SS', 'TH', 'DD', 'FF', 'kk', - 'nn', 'RR', 'CH', 'sil' - ]; - - - // Audio context and playlist - this.audioCtx = new AudioContext(); - this.audioSpeechSource = this.audioCtx.createBufferSource(); - this.audioBackgroundSource = this.audioCtx.createBufferSource(); - this.audioBackgroundGainNode = this.audioCtx.createGain(); - this.audioSpeechGainNode = this.audioCtx.createGain(); - this.audioAnalyzerNode = this.audioCtx.createAnalyser(); - this.audioAnalyzerNode.fftSize = 256; - this.audioAnalyzerNode.smoothingTimeConstant = 0.1; - this.audioAnalyzerNode.minDecibels = -70; - this.audioAnalyzerNode.maxDecibels = -10; - this.audioReverbNode = this.audioCtx.createConvolver(); - this.setReverb(null); // Set dry impulse as default - this.audioBackgroundGainNode.connect(this.audioReverbNode); - this.audioAnalyzerNode.connect(this.audioSpeechGainNode); - this.audioSpeechGainNode.connect(this.audioReverbNode); - this.audioReverbNode.connect(this.audioCtx.destination); - this.setMixerGain( this.opt.mixerGainSpeech, this.opt.mixerGainBackground ); // Volume - this.audioPlaylist = []; - - // Volume based head movement - this.volumeFrequencyData = new Uint8Array(16); - this.volumeMax = 0; - this.volumeHeadBase = 0; - this.volumeHeadTarget = 0; - this.volumeHeadCurrent = 0; - this.volumeHeadVelocity = 0.15; - this.volumeHeadEasing = this.sigmoidFactory(3); - - // Listening - this.isListening = false; - this.listeningAnalyzer = null; - this.listeningActive = false; - this.listeningVolume = 0; - this.listeningSilenceThresholdLevel = this.opt.listeningSilenceThresholdLevel; - this.listeningSilenceThresholdMs = this.opt.listeningSilenceThresholdMs; - this.listeningSilenceDurationMax = this.opt.listeningSilenceDurationMax; - this.listeningActiveThresholdLevel = this.opt.listeningActiveThresholdLevel; - this.listeningActiveThresholdMs = this.opt.listeningActiveThresholdMs; - this.listeningActiveDurationMax = this.opt.listeningActiveDurationMax; - this.listeningTimer = 0; - this.listeningTimerTotal = 0; - - // Create a lookup table for base64 decoding - const b64Chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'; - this.b64Lookup = typeof Uint8Array === 'undefined' ? [] : new Uint8Array(256); - for (let i = 0; i < b64Chars.length; i++) this.b64Lookup[b64Chars.charCodeAt(i)] = i; - - // Speech queue - this.stateName = 'idle'; - this.speechQueue = []; - this.isSpeaking = false; - this.isListening = false; - - // Setup Google text-to-speech - if ( this.opt.ttsEndpoint ) { - let audio = new Audio(); - if (audio.canPlayType("audio/ogg")) { - this.ttsAudioEncoding = "OGG-OPUS"; - } else if (audio.canPlayType("audio/mp3")) { - this.ttsAudioEncoding = "MP3"; - } else { - throw new Error("There was no support for either OGG or MP3 audio."); - } - } else { - throw new Error("You must provide some Google-compliant Text-To-Speech Endpoint."); - } - - - // Setup 3D Animation - this.renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true }); - this.renderer.setPixelRatio( this.opt.modelPixelRatio * window.devicePixelRatio ); - this.renderer.setSize(this.nodeAvatar.clientWidth, this.nodeAvatar.clientHeight); - this.renderer.outputColorSpace = THREE.SRGBColorSpace; - this.renderer.toneMapping = THREE.ACESFilmicToneMapping; - this.renderer.shadowMap.enabled = false; - this.nodeAvatar.appendChild( this.renderer.domElement ); - this.camera = new THREE.PerspectiveCamera( 10, this.nodeAvatar.clientWidth / this.nodeAvatar.clientHeight, 0.1, 2000 ); - this.scene = new THREE.Scene(); - this.lightAmbient = new THREE.AmbientLight( - new THREE.Color( this.opt.lightAmbientColor ), - this.opt.lightAmbientIntensity - ); - this.lightDirect = new THREE.DirectionalLight( - new THREE.Color( this.opt.lightDirectColor ), - this.opt.lightDirectIntensity - ); - this.lightSpot = new THREE.SpotLight( - new THREE.Color( this.opt.lightSpotColor ), - this.opt.lightSpotIntensity, - 0, - this.opt.lightSpotDispersion - ); - this.setLighting( this.opt ); - const pmremGenerator = new THREE.PMREMGenerator( this.renderer ); - pmremGenerator.compileEquirectangularShader(); - this.scene.environment = pmremGenerator.fromScene( new RoomEnvironment() ).texture; - this.resizeobserver = new ResizeObserver(this.onResize.bind(this)); - this.resizeobserver.observe(this.nodeAvatar); - - this.controls = new OrbitControls( this.camera, this.renderer.domElement ); - this.controls.enableZoom = this.opt.cameraZoomEnable; - this.controls.enableRotate = this.opt.cameraRotateEnable; - this.controls.enablePan = this.opt.cameraPanEnable; - this.controls.minDistance = 2; - this.controls.maxDistance = 2000; - this.controls.autoRotateSpeed = 0; - this.controls.autoRotate = false; - this.controls.update(); - this.cameraClock = null; - - // IK Mesh - this.ikMesh = new THREE.SkinnedMesh(); - const ikSetup = { - 'LeftShoulder': null, 'LeftArm': 'LeftShoulder', 'LeftForeArm': 'LeftArm', - 'LeftHand': 'LeftForeArm', 'LeftHandMiddle1': 'LeftHand', - 'RightShoulder': null, 'RightArm': 'RightShoulder', 'RightForeArm': 'RightArm', - 'RightHand': 'RightForeArm', 'RightHandMiddle1': 'RightHand' - }; - const ikBones = []; - Object.entries(ikSetup).forEach( (x,i) => { - const bone = new THREE.Bone(); - bone.name = x[0]; - if ( x[1] ) { - this.ikMesh.getObjectByName(x[1]).add(bone); - } else { - this.ikMesh.add(bone); - } - ikBones.push(bone); - }); - this.ikMesh.bind( new THREE.Skeleton( ikBones ) ); - - // Dynamic Bones - this.dynamicbones = new DynamicBones(); - - } - - /** - * Helper that returns the parameter or, if it is a function, its return value. - * @param {Any} x Parameter - * @return {Any} Value - */ - valueFn(x) { - return (typeof x === 'function' ? x() : x); - } - - /** - * Helper to deep copy and edit an object. - * @param {Object} x Object to copy and edit - * @param {function} [editFn=null] Callback function for editing the new object - * @return {Object} Deep copy of the object. - */ - deepCopy(x, editFn=null) { - const o = JSON.parse(JSON.stringify(x)); - if ( editFn && typeof editFn === "function" ) editFn(o); - return o; - } - - /** - * Convert a Base64 MP3 chunk to ArrayBuffer. - * @param {string} chunk Base64 encoded chunk - * @return {ArrayBuffer} ArrayBuffer - */ - b64ToArrayBuffer(chunk) { - - // Calculate the needed total buffer length - let bufLen = 3 * chunk.length / 4; - if (chunk[chunk.length - 1] === '=') { - bufLen--; - if (chunk[chunk.length - 2] === '=') { - bufLen--; - } - } - - // Create the ArrayBuffer - const arrBuf = new ArrayBuffer(bufLen); - const arr = new Uint8Array(arrBuf); - let i, p = 0, c1, c2, c3, c4; - - // Populate the buffer - for (i = 0; i < chunk.length; i += 4) { - c1 = this.b64Lookup[chunk.charCodeAt(i)]; - c2 = this.b64Lookup[chunk.charCodeAt(i+1)]; - c3 = this.b64Lookup[chunk.charCodeAt(i+2)]; - c4 = this.b64Lookup[chunk.charCodeAt(i+3)]; - arr[p++] = (c1 << 2) | (c2 >> 4); - arr[p++] = ((c2 & 15) << 4) | (c3 >> 2); - arr[p++] = ((c3 & 3) << 6) | (c4 & 63); - } - - return arrBuf; - } - - /** - * Concatenate an array of ArrayBuffers. - * @param {ArrayBuffer[]} bufs Array of ArrayBuffers - * @return {ArrayBuffer} Concatenated ArrayBuffer - */ - concatArrayBuffers(bufs) { - if ( bufs.length === 1 ) return bufs[0]; - let len = 0; - for( let i=0; i= 0x8000) ? -(0x10000 - arr[i]) / 0x8000 : arr[i] / 0x7FFF; - } - const audio = this.audioCtx.createBuffer(1, floats.length, this.opt.pcmSampleRate ); - audio.copyToChannel( floats, 0 , 0 ); - return audio; - } - - - /** - * Convert internal notation to THREE objects. - * NOTE: All rotations are converted to quaternions. - * @param {Object} p Pose - * @return {Object} A new pose object. - */ - propsToThreeObjects(p) { - const r = {}; - for( let [key,val] of Object.entries(p) ) { - const ids = key.split('.'); - let x = Array.isArray(val.x) ? this.gaussianRandom(...val.x) : val.x; - let y = Array.isArray(val.y) ? this.gaussianRandom(...val.y) : val.y; - let z = Array.isArray(val.z) ? this.gaussianRandom(...val.z) : val.z; - - if ( ids[1] === 'position' || ids[1] === 'scale' ) { - r[key] = new THREE.Vector3(x,y,z); - } else if ( ids[1] === 'rotation' ) { - key = ids[0] + '.quaternion'; - r[key] = new THREE.Quaternion().setFromEuler(new THREE.Euler(x,y,z,'XYZ')).normalize(); - } else if ( ids[1] === 'quaternion' ) { - r[key] = new THREE.Quaternion(x,y,z,val.w).normalize(); - } - } - - return r; - } - - - /** - * Clear 3D object. - * @param {Object} obj Object - */ - clearThree(obj){ - while( obj.children.length ){ - this.clearThree(obj.children[0]); - obj.remove(obj.children[0]); - } - if ( obj.geometry ) obj.geometry.dispose(); - - if ( obj.material ) { - Object.keys(obj.material).forEach( x => { - if ( obj.material[x] && obj.material[x] !== null && typeof obj.material[x].dispose === 'function' ) { - obj.material[x].dispose(); - } - }); - obj.material.dispose(); - } - } - - /** - * Loader for 3D avatar model. - * @param {string} avatar Avatar object with 'url' property to GLTF/GLB file. - * @param {progressfn} [onprogress=null] Callback for progress - */ - async showAvatar(avatar, onprogress=null ) { - - // Checkt the avatar parameter - if ( !avatar || !avatar.hasOwnProperty('url') ) { - throw new Error("Invalid parameter. The avatar must have at least 'url' specified."); - } - - // Loader - const loader = new GLTFLoader(); - let gltf = await loader.loadAsync( avatar.url, onprogress ); - - // Check the gltf - const required = [ this.opt.modelRoot ]; - this.posePropNames.forEach( x => required.push( x.split('.')[0] ) ); - required.forEach( x => { - if ( !gltf.scene.getObjectByName(x) ) { - throw new Error('Avatar object ' + x + ' not found'); - } - }); - - this.stop(); - this.avatar = avatar; - - // Dispose Dynamic Bones - this.dynamicbones.dispose(); - - // Clear previous scene, if avatar was previously loaded - this.mixer = null; - if ( this.armature ) { - this.clearThree( this.scene ); - } - - // Avatar full-body - this.armature = gltf.scene.getObjectByName( this.opt.modelRoot ); - this.armature.scale.setScalar(1); - - // Morph targets - this.morphs = []; - this.armature.traverse( x => { - if ( x.morphTargetInfluences && x.morphTargetInfluences.length && - x.morphTargetDictionary ) { - this.morphs.push(x); - } - - // Workaround for #40, hands culled from the rendering process - x.frustumCulled = false; - }); - if ( this.morphs.length === 0 ) { - throw new Error('Blend shapes not found'); - } - - // Morph target keys and values - const keys = new Set(this.mtCustoms); - this.morphs.forEach( x => { - Object.keys(x.morphTargetDictionary).forEach( y => keys.add(y) ); - }); - const mtTemp = {}; - keys.forEach( x => { - - // Morph target data structure - mtTemp[x] = { - fixed: null, system: null, systemd: null, newvalue: null, ref: null, - min: (this.mtMinExceptions.hasOwnProperty(x) ? this.mtMinExceptions[x] : this.mtMinDefault), - max: (this.mtMaxExceptions.hasOwnProperty(x) ? this.mtMaxExceptions[x] : this.mtMaxDefault), - easing: this.mtEasingDefault, base: null, v: 0, needsUpdate: true, - acc: (this.mtAccExceptions.hasOwnProperty(x) ? this.mtAccExceptions[x] : this.mtAccDefault) / 1000, - maxv: (this.mtMaxVExceptions.hasOwnProperty(x) ? this.mtMaxVExceptions[x] : this.mtMaxVDefault) / 1000, - limit: this.mtLimits.hasOwnProperty(x) ? this.mtLimits[x] : null, - onchange: this.mtOnchange.hasOwnProperty(x) ? this.mtOnchange[x] : null, - baseline: this.avatar.baseline?.hasOwnProperty(x) ? this.avatar.baseline[x] : (this.mtBaselineExceptions.hasOwnProperty(x) ? this.mtBaselineExceptions[x] : this.mtBaselineDefault ), - ms: [], is: [] - }; - mtTemp[x].value = mtTemp[x].baseline; - mtTemp[x].applied = mtTemp[x].baseline; - - // Copy previous values - const y = this.mtAvatar[x]; - if ( y ) { - [ 'fixed','system','systemd','base','v','value','applied' ].forEach( z => { - mtTemp[x][z] = y[z]; - }); - } - - // Find relevant meshes - this.morphs.forEach( y => { - const ndx = y.morphTargetDictionary[x]; - if ( ndx !== undefined ) { - mtTemp[x].ms.push(y.morphTargetInfluences); - mtTemp[x].is.push(ndx); - y.morphTargetInfluences[ndx] = mtTemp[x].applied; - } - }); - - }); - this.mtAvatar = mtTemp; - - // Objects for needed properties - this.poseAvatar = { props: {} }; - this.posePropNames.forEach( x => { - const ids = x.split('.'); - const o = this.armature.getObjectByName(ids[0]); - this.poseAvatar.props[x] = o[ids[1]]; - if ( this.poseBase.props.hasOwnProperty(x) ) { - this.poseAvatar.props[x].copy( this.poseBase.props[x] ); - } else { - this.poseBase.props[x] = this.poseAvatar.props[x].clone(); - } - - // Make sure the target has the delta properties, because we need it as a basis - if ( this.poseDelta.props.hasOwnProperty(x) && !this.poseTarget.props.hasOwnProperty(x) ) { - this.poseTarget.props[x] = this.poseAvatar.props[x].clone(); - } - - // Take target pose - this.poseTarget.props[x].t = this.animClock; - this.poseTarget.props[x].d = 2000; - }); - - // Reset IK bone positions - this.ikMesh.traverse( x => { - if (x.isBone) { - x.position.copy( this.armature.getObjectByName(x.name).position ); - } - }); - - // Add avatar to scene - this.scene.add(gltf.scene); - - // Add lights - this.scene.add( this.lightAmbient ); - this.scene.add( this.lightDirect ); - this.scene.add( this.lightSpot ); - this.lightSpot.target = this.armature.getObjectByName('Head'); - - // Setup Dynamic Bones - if ( avatar.hasOwnProperty("modelDynamicBones") ) { - try { - this.dynamicbones.setup(this.scene, this.armature, avatar.modelDynamicBones ); - } - catch(error) { - console.error("Dynamic bones setup failed: " + error); - } - } - - // Find objects that we need in the animate function - this.objectLeftToeBase = this.armature.getObjectByName('LeftToeBase'); - this.objectRightToeBase = this.armature.getObjectByName('RightToeBase'); - this.objectLeftEye = this.armature.getObjectByName('LeftEye'); - this.objectRightEye = this.armature.getObjectByName('RightEye'); - this.objectLeftArm = this.armature.getObjectByName('LeftArm'); - this.objectRightArm = this.armature.getObjectByName('RightArm'); - this.objectHips = this.armature.getObjectByName('Hips'); - this.objectHead = this.armature.getObjectByName('Head'); - this.objectNeck = this.armature.getObjectByName('Neck'); - - // Estimate avatar height based on eye level - const plEye = new THREE.Vector3(); - this.objectLeftEye.getWorldPosition(plEye); - this.avatarHeight = plEye.y + 0.2; - - // Set pose, view and start animation - if ( !this.viewName ) this.setView( this.opt.cameraView ); - this.setMood( this.avatar.avatarMood || this.moodName || this.opt.avatarMood ); - this.start(); - - } - - /** - * Get view names. - * @return {string[]} Supported view names. - */ - getViewNames() { - return ['full', 'mid', 'upper', 'head']; - } - - /** - * Get current view. - * @return {string} View name. - */ - getView() { - return this.viewName; - } - - /** - * Fit 3D object to the view. - * @param {string} [view=null] Camera view. If null, reset current view - * @param {Object} [opt=null] Options - */ - setView(view, opt = null) { - if ( view !== 'full' && view !== 'upper' && view !== 'head' && view !== 'mid' ) return; - if ( !this.armature ) { - this.opt.cameraView = view; - return; - } - - this.viewName = view || this.viewName; - opt = opt || {}; - - const fov = this.camera.fov * ( Math.PI / 180 ); - let x = - (opt.cameraX || this.opt.cameraX) * Math.tan( fov / 2 ); - let y = ( 1 - (opt.cameraY || this.opt.cameraY)) * Math.tan( fov / 2 ); - let z = (opt.cameraDistance || this.opt.cameraDistance); - - switch(this.viewName) { - case 'head': - z += 2; - y = y * z + 4 * this.avatarHeight / 5; - break; - case 'upper': - z += 4.5; - y = y * z + 2 * this.avatarHeight / 3; - break; - case 'mid': - z += 8; - y = y * z + this.avatarHeight / 3; - break; - default: - z += 12; - y = y * z; - } - - x = x * z; - - this.controlsEnd = new THREE.Vector3(x, y, 0); - this.cameraEnd = new THREE.Vector3(x, y, z).applyEuler( new THREE.Euler( (opt.cameraRotateX || opt.cameraRotateX), (opt.cameraRotateY || this.opt.cameraRotateY), 0 ) ); - - if ( this.cameraClock === null ) { - this.controls.target.copy( this.controlsEnd ); - this.camera.position.copy( this.cameraEnd ); - } - this.controlsStart = this.controls.target.clone(); - this.cameraStart = this.camera.position.clone(); - this.cameraClock = 0; - - } - - /** - * Change light colors and intensities. - * @param {Object} opt Options - */ - setLighting(opt) { - opt = opt || {}; - - // Ambient light - if ( opt.hasOwnProperty("lightAmbientColor") ) { - this.lightAmbient.color.set( new THREE.Color( opt.lightAmbientColor ) ); - } - if ( opt.hasOwnProperty("lightAmbientIntensity") ) { - this.lightAmbient.intensity = opt.lightAmbientIntensity; - this.lightAmbient.visible = (opt.lightAmbientIntensity !== 0); - } - - // Directional light - if ( opt.hasOwnProperty("lightDirectColor") ) { - this.lightDirect.color.set( new THREE.Color( opt.lightDirectColor ) ); - } - if ( opt.hasOwnProperty("lightDirectIntensity") ) { - this.lightDirect.intensity = opt.lightDirectIntensity; - this.lightDirect.visible = (opt.lightDirectIntensity !== 0); - } - if ( opt.hasOwnProperty("lightDirectPhi") && opt.hasOwnProperty("lightDirectTheta") ) { - this.lightDirect.position.setFromSphericalCoords(2, opt.lightDirectPhi, opt.lightDirectTheta); - } - - // Spot light - if ( opt.hasOwnProperty("lightSpotColor") ) { - this.lightSpot.color.set( new THREE.Color( opt.lightSpotColor ) ); - } - if ( opt.hasOwnProperty("lightSpotIntensity") ) { - this.lightSpot.intensity = opt.lightSpotIntensity; - this.lightSpot.visible = (opt.lightSpotIntensity !== 0); - } - if ( opt.hasOwnProperty("lightSpotPhi") && opt.hasOwnProperty("lightSpotTheta") ) { - this.lightSpot.position.setFromSphericalCoords( 2, opt.lightSpotPhi, opt.lightSpotTheta ); - this.lightSpot.position.add( new THREE.Vector3(0,1.5,0) ); - } - if ( opt.hasOwnProperty("lightSpotDispersion") ) { - this.lightSpot.angle = opt.lightSpotDispersion; - } - } - - /** - * Render scene. - */ - render() { - if ( this.isRunning ) { - this.renderer.render( this.scene, this.camera ); - } - } - - /** - * Resize avatar. - */ - onResize() { - this.camera.aspect = this.nodeAvatar.clientWidth / this.nodeAvatar.clientHeight; - this.camera.updateProjectionMatrix(); - this.renderer.setSize( this.nodeAvatar.clientWidth, this.nodeAvatar.clientHeight ); - this.controls.update(); - this.render(); - } - - /** - * Update avatar pose. - * @param {number} t High precision timestamp in ms. - */ - updatePoseBase(t) { - for( const [key,val] of Object.entries(this.poseTarget.props) ) { - const o = this.poseAvatar.props[key]; - if (o) { - let alpha = (t - val.t) / val.d; - if ( alpha > 1 || !this.poseBase.props.hasOwnProperty(key) ) { - o.copy(val); - } else { - if ( o.isQuaternion ) { - o.copy( this.poseBase.props[key].slerp(val, this.easing(alpha) )); - } else if ( o.isVector3 ) { - o.copy( this.poseBase.props[key].lerp(val, this.easing(alpha) )); - } - } - } - } - } - - /** - * Update avatar pose deltas - */ - updatePoseDelta() { - for( const [key,d] of Object.entries(this.poseDelta.props) ) { - if ( d.x === 0 && d.y === 0 && d.z === 0 ) continue; - e.set(d.x,d.y,d.z); - const o = this.poseAvatar.props[key]; - if ( o.isQuaternion ) { - q.setFromEuler(e); - o.multiply(q); - } else if ( o.isVector3 ) { - o.add( e ); - } - } - } - - /** - * Update morph target values. - * @param {number} dt Delta time in ms. - */ - updateMorphTargets(dt) { - - for( let [mt,o] of Object.entries(this.mtAvatar) ) { - - if ( !o.needsUpdate ) continue; - - // Alternative target (priority order): - // - fixed: Fixed value, typically user controlled - // - system: System value, which overrides animations - // - newvalue: Animation value - // - baseline: Baseline value when none of the above applies - let target = null; - let newvalue = null; - if ( o.fixed !== null ) { - target = o.fixed; - o.system = null; - o.systemd = null; - o.newvalue = null; - if ( o.ref && o.ref.hasOwnProperty(mt) ) delete o.ref[mt]; - o.ref = null; - o.base = null; - if ( o.value === target ) { - o.needsUpdate = false; - continue; - } - } else if ( o.system !== null ) { - target = o.system; - o.newvalue = null; - if ( o.ref && o.ref.hasOwnProperty(mt) ) delete o.ref[mt]; - o.ref = null; - o.base = null; - if ( o.systemd !== null ) { - if ( o.systemd === 0 ) { - target = null; - o.system = null; - o.systemd = null; - } else { - o.systemd -= dt; - if ( o.systemd < 0 ) o.systemd= 0; - if ( o.value === target ) { - target = null; - } - } - } else if ( o.value === target ) { - target = null; - o.system = null; - } - } else if ( o.newvalue !== null ) { - o.ref = null; - o.base = null; - newvalue = o.newvalue; - o.newvalue = null; - } else if ( o.base !== null ) { - target = o.base; - o.ref = null; - if ( o.value === target ) { - target = null; - o.base = null; - o.needsUpdate = false; - } - } else { - o.ref = null; - if ( o.baseline !== null && o.value !== o.baseline ) { - target = o.baseline; - o.base = o.baseline; - } else { - o.needsUpdate = false; - } - } - - // Calculate new value using exponential smoothing - if ( target !== null ) { - let diff = target - o.value; - if ( diff >= 0 ) { - if ( diff < 0.005 ) { - newvalue = target; - o.v = 0; - } else { - if ( o.v < o.maxv ) o.v += o.acc * dt; - if ( o.v >= 0 ) { - newvalue = o.value + diff * ( 1 - Math.exp(- o.v * dt) ); - } else { - newvalue = o.value + o.v * dt * ( 1 - Math.exp(o.v * dt) ); - } - } - } else { - if ( diff > -0.005 ) { - newvalue = target; - o.v = 0; - } else { - if ( o.v > -o.maxv ) o.v -= o.acc * dt; - if ( o.v >= 0 ) { - newvalue = o.value + o.v * dt * ( 1 - Math.exp(- o.v * dt) ); - } else { - newvalue = o.value + diff * ( 1 - Math.exp( o.v * dt) ); - } - } - } - } - - // Check limits and whether we need to actually update the morph target - if ( o.limit !== null ) { - if ( newvalue !== null && newvalue !== o.value ) { - o.value = newvalue; - if ( o.onchange !== null ) o.onchange(newvalue); - } - newvalue = o.limit(o.value); - if ( newvalue === o.applied ) continue; - } else { - if ( newvalue === null || newvalue === o.value ) continue; - o.value = newvalue; - if ( o.onchange !== null ) o.onchange(newvalue); - } - - o.applied = newvalue; - if ( o.applied < o.min ) o.applied = o.min; - if ( o.applied > o.max ) o.applied = o.max; - - // Apply value - switch(mt) { - - case 'headRotateX': - this.poseDelta.props['Head.quaternion'].x = newvalue + this.mtAvatar['bodyRotateX'].applied; - break; - - case 'headRotateY': - this.poseDelta.props['Head.quaternion'].y = newvalue + this.mtAvatar['bodyRotateY'].applied; - break; - - case 'headRotateZ': - this.poseDelta.props['Head.quaternion'].z = newvalue + this.mtAvatar['bodyRotateZ'].applied; - break; - - case 'bodyRotateX': - this.poseDelta.props['Head.quaternion'].x = newvalue + this.mtAvatar['headRotateX'].applied; - this.poseDelta.props['Spine1.quaternion'].x = newvalue/2; - this.poseDelta.props['Spine.quaternion'].x = newvalue/8; - this.poseDelta.props['Hips.quaternion'].x = newvalue/24; - break; - - case 'bodyRotateY': - this.poseDelta.props['Head.quaternion'].y = newvalue + this.mtAvatar['headRotateY'].applied; - this.poseDelta.props['Spine1.quaternion'].y = newvalue/2; - this.poseDelta.props['Spine.quaternion'].y = newvalue/2; - this.poseDelta.props['Hips.quaternion'].y = newvalue/4; - this.poseDelta.props['LeftUpLeg.quaternion'].y = newvalue/2; - this.poseDelta.props['RightUpLeg.quaternion'].y = newvalue/2; - this.poseDelta.props['LeftLeg.quaternion'].y = newvalue/4; - this.poseDelta.props['RightLeg.quaternion'].y = newvalue/4; - break; - - case 'bodyRotateZ': - this.poseDelta.props['Head.quaternion'].z = newvalue + this.mtAvatar['headRotateZ'].applied; - this.poseDelta.props['Spine1.quaternion'].z = newvalue/12; - this.poseDelta.props['Spine.quaternion'].z = newvalue/12; - this.poseDelta.props['Hips.quaternion'].z = newvalue/24; - break; - - case 'handFistLeft': - case 'handFistRight': - const side = mt.substring(8); - ['HandThumb', 'HandIndex','HandMiddle', - 'HandRing', 'HandPinky'].forEach( (x,i) => { - if ( i === 0 ) { - this.poseDelta.props[side+x+'1.quaternion'].x = 0; - this.poseDelta.props[side+x+'2.quaternion'].z = (side === 'Left' ? -1 : 1) * newvalue; - this.poseDelta.props[side+x+'3.quaternion'].z = (side === 'Left' ? -1 : 1) * newvalue; - } else { - this.poseDelta.props[side+x+'1.quaternion'].x = newvalue; - this.poseDelta.props[side+x+'2.quaternion'].x = 1.5 * newvalue; - this.poseDelta.props[side+x+'3.quaternion'].x = 1.5 * newvalue; - } - }); - break; - - case 'chestInhale': - const scale = newvalue/20; - const d = { x: scale, y: (scale/2), z: (3 * scale) }; - const dneg = { x: (1/(1+scale) - 1), y: (1/(1 + scale/2) - 1), z: (1/(1 + 3 * scale) - 1) }; - this.poseDelta.props['Spine1.scale'] = d; - this.poseDelta.props['Neck.scale'] = dneg; - this.poseDelta.props['LeftArm.scale'] = dneg; - this.poseDelta.props['RightArm.scale'] = dneg; - break; - - default: - for( let i=0,l=o.ms.length; i { - const ids = x[0].split('.'); - if ( ids[1] === 'position' || ids[1] === 'rotation' || ids[1] === 'quaternion' ) { - const key = (ids[1] === 'quaternion' ? (ids[0]+'.rotation') : x[0]); - const val = (x[1].isQuaternion ? new THREE.Euler().setFromQuaternion(x[1]) : x[1]); - s += (i?", ":"") + "'" + key + "':{"; - s += 'x:' + Math.round(val.x * prec) / prec; - s += ', y:' + Math.round(val.y * prec) / prec; - s += ', z:' + Math.round(val.z * prec) / prec; - s += '}'; - } - }); - s += '}'; - return s; - } - - - /** - * Return pose template property taking into account mirror pose and gesture. - * @param {string} key Property key - * @return {Quaternion|Vector3} Position or rotation - */ - getPoseTemplateProp(key) { - - const ids = key.split('.'); - let target = ids[0] + '.' + (ids[1] === 'rotation' ? 'quaternion' : ids[1]); - - if ( this.gesture && this.gesture.hasOwnProperty(target) ) { - return this.gesture[target].clone(); - } else { - let source = ids[0] + '.' + (ids[1] === 'quaternion' ? 'rotation' : ids[1]); - if ( !this.poseWeightOnLeft ) { - if ( source.startsWith('Left') ) { - source = 'Right' + source.substring(4); - target = 'Right' + target.substring(4); - } else if ( source.startsWith('Right') ) { - source = 'Left' + source.substring(5); - target = 'Left' + target.substring(5); - } - } - - // Get value - let val; - if ( this.poseTarget.template.props.hasOwnProperty(target) ) { - const o = {}; - o[target] = this.poseTarget.template.props[target]; - val = this.propsToThreeObjects( o )[target]; - } else if ( this.poseTarget.template.props.hasOwnProperty(source) ) { - const o = {}; - o[source] = this.poseTarget.template.props[source]; - val = this.propsToThreeObjects( o )[target]; - } - - // Mirror - if ( val && !this.poseWeightOnLeft && val.isQuaternion ) { - val.x *= -1; - val.w *= -1; - } - - return val; - } - } - - /** - * Change body weight from current leg to another. - * @param {Object} p Pose properties - * @return {Object} Mirrored pose. - */ - mirrorPose(p) { - const r = {}; - for( let [key,val] of Object.entries(p) ) { - - // Create a mirror image - if ( val.isQuaternion ) { - if ( key.startsWith('Left') ) { - key = 'Right' + key.substring(4); - } else if ( key.startsWith('Right') ) { - key = 'Left' + key.substring(5); - } - val.x *= -1; - val.w *= -1; - } - - r[key] = val.clone(); - - // Custom properties - r[key].t = val.t; - r[key].d = val.d; - } - return r; - } - - /** - * Create a new pose. - * @param {Object} template Pose template - * @param {numeric} [ms=2000] Transition duration in ms - * @return {Object} A new pose object. - */ - poseFactory(template, ms=2000) { - - // Pose object - const o = { - template: template, - props: this.propsToThreeObjects( template.props ) - }; - - for( const [p,val] of Object.entries(o.props) ) { - - // Restrain movement when standing - if ( this.opt.modelMovementFactor < 1 && template.standing && - (p === 'Hips.quaternion' || p === 'Spine.quaternion' || - p === 'Spine1.quaternion' || p === 'Spine2.quaternion' || - p === 'Neck.quaternion' || p === 'LeftUpLeg.quaternion' || - p === 'LeftLeg.quaternion' || p === 'RightUpLeg.quaternion' || - p === 'RightLeg.quaternion') ) { - const ref = this.poseStraight[p]; - const angle = val.angleTo( ref ); - val.rotateTowards( ref, (1 - this.opt.modelMovementFactor) * angle ); - } - - // Custom properties - val.t = this.animClock; // timestamp - val.d = ms; // Transition duration - - } - return o; - } - - /** - * Set a new pose and start transition timer. - * @param {Object} template Pose template, if null update current pose - * @param {number} [ms=2000] Transition time in milliseconds - */ - setPoseFromTemplate(template, ms=2000) { - - // Special cases - const isIntermediate = template && this.poseTarget && this.poseTarget.template && ((this.poseTarget.template.standing && template.lying) || (this.poseTarget.template.lying && template.standing)); - const isSameTemplate = template && (template === this.poseCurrentTemplate); - const isWeightOnLeft = this.poseWeightOnLeft; - let duration = isIntermediate ? 1000 : ms; - - // New pose template - if ( isIntermediate) { - this.poseCurrentTemplate = this.poseTemplates['oneknee']; - setTimeout( () => { - this.setPoseFromTemplate(template,ms); - }, duration); - } else { - this.poseCurrentTemplate = template || this.poseCurrentTemplate; - } - - // Set target - this.poseTarget = this.poseFactory(this.poseCurrentTemplate, duration); - this.poseWeightOnLeft = true; - - // Mirror properties, if necessary - if ( (!isSameTemplate && !isWeightOnLeft) || (isSameTemplate && isWeightOnLeft ) ) { - this.poseTarget.props = this.mirrorPose(this.poseTarget.props); - this.poseWeightOnLeft = !this.poseWeightOnLeft; - } - - // Gestures - if ( this.gesture ) { - for( let [p,val] of Object.entries(this.gesture) ) { - if ( this.poseTarget.props.hasOwnProperty(p) ) { - this.poseTarget.props[p].copy(val); - this.poseTarget.props[p].t = val.t; - this.poseTarget.props[p].d = val.d; - } - } - } - - // Make sure deltas are included in the target - Object.keys(this.poseDelta.props).forEach( key => { - if ( !this.poseTarget.props.hasOwnProperty(key) ) { - this.poseTarget.props[key] = this.poseBase.props[key].clone(); - this.poseTarget.props[key].t = this.animClock; - this.poseTarget.props[key].d = duration; - } - }); - - } - - /** - * Get morph target value. - * @param {string} mt Morph target - * @return {number} Value - */ - getValue(mt) { - return this.mtAvatar[mt]?.value; - } - - /** - * Set morph target value. - * @param {string} mt Morph target - * @param {number} val Value - * @param {number} [ms=null] Transition time in milliseconds. - */ - setValue(mt,val,ms=null) { - if ( this.mtAvatar.hasOwnProperty(mt) ) { - Object.assign(this.mtAvatar[mt],{ system: val, systemd: ms, needsUpdate: true }); - } - } - - - /** - * Get mood names. - * @return {string[]} Mood names. - */ - getMoodNames() { - return Object.keys(this.animMoods); - } - - /** - * Get current mood. - * @return {string[]} Mood name. - */ - getMood() { - return this.opt.avatarMood; - } - - /** - * Set mood. - * @param {string} s Mood name. - */ - setMood(s) { - s = (s || '').trim().toLowerCase(); - if ( !this.animMoods.hasOwnProperty(s) ) throw new Error("Unknown mood."); - this.moodName = s; - this.mood = this.animMoods[this.moodName]; - - // Reset morph target baseline - for( let mt of Object.keys(this.mtAvatar) ) { - let val = this.mtBaselineExceptions.hasOwnProperty(mt) ? this.mtBaselineExceptions[mt] : this.mtBaselineDefault; - if ( this.mood.baseline.hasOwnProperty(mt) ) { - val = this.mood.baseline[mt]; - } else if ( this.avatar.baseline?.hasOwnProperty(mt) ) { - val = this.avatar.baseline[mt]; - } - this.setBaselineValue( mt, val ); - } - - // Set/replace animations - this.mood.anims.forEach( x => { - let i = this.animQueue.findIndex( y => y.template.name === x.name ); - if ( i !== -1 ) { - this.animQueue.splice(i, 1); - } - this.animQueue.push( this.animFactory( x, -1 ) ); - }); - - } - - - /** - * Get morph target names. - * @return {string[]} Morph target names. - */ - getMorphTargetNames() { - return [ 'eyesRotateX', 'eyesRotateY', ...Object.keys(this.mtAvatar)].sort(); - } - - /** - * Get baseline value for the morph target. - * @param {string} mt Morph target name - * @return {number} Value, null if not in baseline - */ - getBaselineValue( mt ) { - if ( mt === 'eyesRotateY' ) { - const ll = this.getBaselineValue('eyeLookOutLeft'); - if ( ll === undefined ) return undefined; - const lr = this.getBaselineValue('eyeLookInLeft'); - if ( lr === undefined ) return undefined; - const rl = this.getBaselineValue('eyeLookOutRight'); - if ( rl === undefined ) return undefined; - const rr = this.getBaselineValue('eyeLookInRight'); - if ( rr === undefined ) return undefined; - return ll - lr; - } else if ( mt === 'eyesRotateX' ) { - const d = this.getBaselineValue('eyesLookDown'); - if ( d === undefined ) return undefined; - const u = this.getBaselineValue('eyesLookUp'); - if ( u === undefined ) return undefined; - return d - u; - } else { - return this.mtAvatar[mt]?.baseline; - } - } - - /** - * Set baseline for morph target. - * @param {string} mt Morph target name - * @param {number} val Value, null if to be removed from baseline - */ - setBaselineValue( mt, val ) { - if ( mt === 'eyesRotateY' ) { - this.setBaselineValue('eyeLookOutLeft', (val === null) ? null : (val>0 ? val : 0) ); - this.setBaselineValue('eyeLookInLeft', (val === null) ? null : (val>0 ? 0 : -val) ); - this.setBaselineValue('eyeLookOutRight', (val === null) ? null : (val>0 ? 0 : -val) ); - this.setBaselineValue('eyeLookInRight', (val === null) ? null : (val>0 ? val : 0) ); - } else if ( mt === 'eyesRotateX' ) { - this.setBaselineValue('eyesLookDown', (val === null) ? null : (val>0 ? val : 0) ); - this.setBaselineValue('eyesLookUp', (val === null) ? null : (val>0 ? 0 : -val) ); - } else { - if ( this.mtAvatar.hasOwnProperty(mt) ) { - Object.assign(this.mtAvatar[mt],{ base: null, baseline: val, needsUpdate: true }); - } - } - } - - /** - * Get fixed value for the morph target. - * @param {string} mt Morph target name - * @return {number} Value, null if not fixed - */ - getFixedValue( mt ) { - if ( mt === 'eyesRotateY' ) { - const ll = this.getFixedValue('eyeLookOutLeft'); - if ( ll === null ) return null; - const lr = this.getFixedValue('eyeLookInLeft'); - if ( lr === null ) return null; - const rl = this.getFixedValue('eyeLookOutRight'); - if ( rl === null ) return null; - const rr = this.getFixedValue('eyeLookInRight'); - if ( rr === null ) return null; - return ll - lr; - } else if ( mt === 'eyesRotateX' ) { - const d = this.getFixedValue('eyesLookDown'); - if ( d === null ) return null; - const u = this.getFixedValue('eyesLookUp'); - if ( u === null ) return null; - return d - u; - } else { - return this.mtAvatar[mt]?.fixed; - } - } - - /** - * Fix morph target. - * @param {string} mt Morph target name - * @param {number} val Value, null if to be removed - */ - setFixedValue( mt, val, ms=null ) { - if ( mt === 'eyesRotateY' ) { - this.setFixedValue('eyeLookOutLeft', (val === null) ? null : (val>0 ? val : 0 ), ms ); - this.setFixedValue('eyeLookInLeft', (val === null) ? null : (val>0 ? 0 : -val ), ms ); - this.setFixedValue('eyeLookOutRight', (val === null) ? null : (val>0 ? 0 : -val ), ms ); - this.setFixedValue('eyeLookInRight', (val === null) ? null : (val>0 ? val : 0 ), ms ); - } else if ( mt === 'eyesRotateX' ) { - this.setFixedValue('eyesLookDown', (val === null) ? null : (val>0 ? val : 0 ), ms ); - this.setFixedValue('eyesLookUp', (val === null) ? null : (val>0 ? 0 : -val ), ms ); - } else { - if ( this.mtAvatar.hasOwnProperty(mt) ) { - Object.assign(this.mtAvatar[mt],{ fixed: val, needsUpdate: true }); - } - } - } - - - /** - * Create a new animation based on an animation template. - * @param {Object} t Animation template - * @param {number} [loop=false] Number of loops, false if not looped - * @param {number} [scaleTime=1] Scale template times - * @param {number} [scaleValue=1] Scale template values - * @return {Object} New animation object. - */ - animFactory( t, loop = false, scaleTime = 1, scaleValue = 1 ) { - const o = { template: t, ts: [0], vs: {} }; - - // Follow the hierarchy of objects - let a = t; - while(1) { - if ( a.hasOwnProperty(this.stateName) ) { - a = a[this.stateName]; - } else if ( a.hasOwnProperty(this.moodName) ) { - a = a[this.moodName]; - } else if ( a.hasOwnProperty(this.poseName) ) { - a = a[this.poseName]; - } else if ( a.hasOwnProperty(this.viewName) ) { - a = a[this.viewName]; - } else if ( this.avatar.body && a.hasOwnProperty(this.avatar.body) ) { - a = a[this.avatar.body]; - } else if ( a.hasOwnProperty('alt') ) { - - // Go through alternatives with probabilities - let b = a.alt[0]; - if ( a.alt.length > 1 ) { - // Flip a coin - const coin = Math.random(); - let p = 0; - for( let i=0; i { - let val = this.valueFn(x); - if ( Array.isArray(val) ) { - val = this.gaussianRandom(...val); - } - o.ts[i+1] = o.ts[i] + val; - }); - } else { - let l = Object.values(a.vs).reduce( (acc,val) => (val.length > acc) ? val.length : acc, 0); - o.ts = Array(l+1).fill(0); - } - o.ts = o.ts.map( x => this.animClock + delay + x * scaleTime ); - - // Values - for( let [mt,vs] of Object.entries(a.vs) ) { - const base = this.getBaselineValue(mt); - const vals = vs.map( x => { - x = this.valueFn(x); - if ( x === null ) { - return null; - } else if ( typeof x === 'function' ) { - return x; - } else if ( typeof x === 'string' || x instanceof String ) { - return x.slice(); - } else if ( Array.isArray(x) ) { - if ( mt === 'gesture' ) { - return x.slice(); - } else { - return (base === undefined ? 0 : base) + scaleValue * this.gaussianRandom(...x); - } - } else if (typeof x == "boolean") { - return x; - } else if ( x instanceof Object && x.constructor === Object ) { - return Object.assign( {}, x ); - } else { - return (base === undefined ? 0 : base) + scaleValue * x; - } - }); - - if ( mt === 'eyesRotateY' ) { - o.vs['eyeLookOutLeft'] = [null, ...vals.map( x => (x>0) ? x : 0 ) ]; - o.vs['eyeLookInLeft'] = [null, ...vals.map( x => (x>0) ? 0 : -x ) ]; - o.vs['eyeLookOutRight'] = [null, ...vals.map( x => (x>0) ? 0 : -x ) ]; - o.vs['eyeLookInRight'] = [null, ...vals.map( x => (x>0) ? x : 0 ) ]; - } else if ( mt === 'eyesRotateX' ) { - o.vs['eyesLookDown'] = [null, ...vals.map( x => (x>0) ? x : 0 ) ]; - o.vs['eyesLookUp'] = [null, ...vals.map( x => (x>0) ? 0 : -x ) ]; - } else { - o.vs[mt] = [null, ...vals]; - } - } - for( let mt of Object.keys(o.vs) ) { - while( o.vs[mt].length <= o.ts.length ) o.vs[mt].push( o.vs[mt][ o.vs[mt].length - 1 ]); - } - - // Mood - if ( t.hasOwnProperty("mood") ) o.mood = this.valueFn(t.mood).slice(); - - // Loop - if ( loop ) o.loop = loop; - - return o; - } - - /** - * Calculate the correct value based on a given time using the given function. - * @param {number[]} vstart Start value - * @param {number[]} vend End value - * @param {number[]} tstart Start time - * @param {number[]} tend End time - * @param {number[]} t Current time - * @param {function} [fun=null] Ease in/out function, null = linear - * @return {number} Value based on the given time. - */ - valueAnimationSeq(vstart,vend,tstart,tend,t,fun=null) { - vstart = this.valueFn(vstart); - vend = this.valueFn(vend); - if ( t < tstart ) t = tstart; - if ( t > tend ) t = tend; - let k = (vend - vstart) / (tend - tstart); - if ( fun ) { - k *= fun( ( t - tstart ) / (tend - tstart) ); - } - return k * t + (vstart - k * tstart); - } - - /** - * Return gaussian distributed random value between start and end with skew. - * @param {number} start Start value - * @param {number} end End value - * @param {number} [skew=1] Skew - * @param {number} [samples=5] Number of samples, 1 = uniform distribution. - * @return {number} Gaussian random value. - */ - gaussianRandom(start,end,skew=1,samples=5) { - let r = 0; - for( let i=0; i vol) { - vol = this.volumeFrequencyData[i]; - } - } - - this.listeningVolume = (this.listeningVolume + vol) / 2; - if ( this.listeningActive ) { - this.listeningTimerTotal += dt; - if ( this.listeningVolume < this.listeningSilenceThresholdLevel ) { - this.listeningTimer += dt; - if ( this.listeningTimer > this.listeningSilenceThresholdMs ) { - if ( this.listeningOnchange ) this.listeningOnchange('stop',this.listeningTimer); - this.listeningActive = false; - this.listeningTimer = 0; - this.listeningTimerTotal = 0; - } - } else { - this.listeningTimer *= 0.5; - } - if ( this.listeningTimerTotal > this.listeningActiveDurationMax ) { - if ( this.listeningOnchange ) this.listeningOnchange('maxactive'); - this.listeningTimerTotal = 0; - } - } else { - this.listeningTimerTotal += dt; - if ( this.listeningVolume > this.listeningActiveThresholdLevel ) { - this.listeningTimer += dt; - if ( this.listeningTimer > this.listeningActiveThresholdMs ) { - if ( this.listeningOnchange ) this.listeningOnchange('start'); - this.listeningActive = true; - this.listeningTimer = 0; - this.listeningTimerTotal = 0; - } - } else { - this.listeningTimer *= 0.5; - } - if ( this.listeningTimerTotal > this.listeningSilenceDurationMax ) { - if ( this.listeningOnchange ) this.listeningOnchange('maxsilence'); - this.listeningTimerTotal = 0; - } - } - } - - // Speaking - if ( this.isSpeaking ) { - vol = 0; - this.audioAnalyzerNode.getByteFrequencyData(this.volumeFrequencyData); - for (i=2, l=10; i vol) { - vol = this.volumeFrequencyData[i]; - } - } - } - - // Animation loop - let isEyeContact = null; - let isHeadMove = null; - const tasks = []; - for( i=0, l=this.animQueue.length; i 0.0001 ) alpha = (this.animClock - x.ts[j]) / tdiff; - if ( alpha < 1 ) { - if ( m.easing ) alpha = m.easing(alpha); - m.newvalue = ( 1 - alpha ) * vs[j] + alpha * m.newvalue; - } - if ( m.ref && m.ref !== x.vs && m.ref.hasOwnProperty(mt) ) delete m.ref[mt]; - m.ref = x.vs; - } - - // Volume effect - if ( vol ) { - switch(mt){ - case 'viseme_aa': - case 'viseme_E': - case 'viseme_I': - case 'viseme_O': - case 'viseme_U': - m.newvalue *= 1 + vol / 255 - 0.5; - } - } - - // Update - m.needsUpdate = true; - - } else if ( mt === 'eyeContact' && vs[j] !== null && isEyeContact !== false ) { - isEyeContact = Boolean(vs[j]); - } else if ( mt === 'headMove' && vs[j] !== null && isHeadMove !== false ) { - if ( vs[j] === 0 ) { - isHeadMove = false; - } else { - if ( Math.random() < vs[j] ) isHeadMove = true; - vs[j] = null; - } - } else if ( vs[j] !== null ) { - tasks.push({ mt: mt, val: vs[j] }); - vs[j] = null; - } - - } - - } - - // If end timeslot, loop or remove the animation, otherwise keep at it - if ( j === k ) { - if ( x.hasOwnProperty('mood') ) this.setMood(x.mood); - if ( x.loop ) { - k = ( this.isSpeaking && (x.template.name === 'head' || x.template.name === 'eyes') ) ? 4 : 1; // Restrain - this.animQueue[i] = this.animFactory( x.template, (x.loop > 0 ? x.loop - 1 : x.loop), 1, 1/k ); - } else { - this.animQueue.splice(i--, 1); - l--; - } - } else { - x.ndx = j - 1; - } - - } - - // Tasks - for( let i=0, l=tasks.length; i { - if ( y[1] ) { - this.poseTarget.props[y[0]].copy( y[1] ); - } else { - this.poseTarget.props[y[0]].copy( this.getPoseTemplateProp(y[0]) ); - } - this.poseTarget.props[y[0]].t = this.animClock; - this.poseTarget.props[y[0]].d = (y[1] && y[1].d) ? y[1].d : (y.duration || 2000); - }); - break; - - case 'handLeft': - this.ikSolve( { - iterations: 20, root: "LeftShoulder", effector: "LeftHandMiddle1", - links: [ - { link: "LeftHand", minx: -0.5, maxx: 0.5, miny: -1, maxy: 1, minz: -0.5, maxz: 0.5 }, - { link: "LeftForeArm", minx: -0.5, maxx: 1.5, miny: -1.5, maxy: 1.5, minz: -0.5, maxz: 3 }, - { link: "LeftArm", minx: -1.5, maxx: 1.5, miny: 0, maxy: 0, minz: -1, maxz: 3 } - ] - }, j.x ? new THREE.Vector3(j.x,j.y,j.z) : null, true, j.d ); - break; - - - case 'handRight': - this.ikSolve( { - iterations: 20, root: "RightShoulder", effector: "RightHandMiddle1", - links: [ - { link: "RightHand", minx: -0.5, maxx: 0.5, miny: -1, maxy: 1, minz: -0.5, maxz: 0.5, maxAngle: 0.1 }, - { link: "RightForeArm", minx: -0.5, maxx: 1.5, miny: -1.5, maxy: 1.5, minz: -3, maxz: 0.5, maxAngle: 0.2 }, - { link: "RightArm", minx: -1.5, maxx: 1.5, miny: 0, maxy: 0, minz: -1, maxz: 3 } - ] - }, j.x ? new THREE.Vector3(j.x,j.y,j.z) : null, true, j.d ); - break; - } - } - - // Eye contact - if ( isEyeContact || isHeadMove ) { - - // Get head position - e.setFromQuaternion( this.poseAvatar.props['Head.quaternion'] ); - e.x = Math.max(-0.9,Math.min(0.9, 2 * e.x - 0.5 )); - e.y = Math.max(-0.9,Math.min(0.9, -2.5 * e.y)); - - if ( isEyeContact ) { - Object.assign( this.mtAvatar['eyesLookDown'], { system: e.x < 0 ? -e.x : 0, needsUpdate: true }); - Object.assign( this.mtAvatar['eyesLookUp'], { system: e.x < 0 ? 0 : e.x, needsUpdate: true }); - Object.assign( this.mtAvatar['eyeLookInLeft'], { system: e.y < 0 ? -e.y : 0, needsUpdate: true }); - Object.assign( this.mtAvatar['eyeLookOutLeft'], { system: e.y < 0 ? 0 : e.y, needsUpdate: true }); - Object.assign( this.mtAvatar['eyeLookInRight'], { system: e.y < 0 ? 0 : e.y, needsUpdate: true }); - Object.assign( this.mtAvatar['eyeLookOutRight'], { system: e.y < 0 ? -e.y : 0, needsUpdate: true }); - - // Head move - if ( isHeadMove ) { - i = - this.mtAvatar['bodyRotateY'].value; - j = this.gaussianRandom(-0.2,0.2); - this.animQueue.push( this.animFactory({ name: "headmove", - dt: [[1000,2000],[1000,2000,1,2],[1000,2000],[1000,2000,1,2]], vs: { - headRotateY: [i,i,0], headRotateX: [j,j,0], headRotateZ: [-i/4,-i/4,0] - } - })); - } - - } else { - i = this.mtAvatar['eyeLookInLeft'].value + this.mtAvatar['eyeLookOutLeft'].value; - j = this.gaussianRandom(-0.2,0.2); - this.animQueue.push( this.animFactory({ name: "headmove", - dt: [[1000,2000],[1000,2000,1,2],[1000,2000],[1000,2000,1,2]], vs: { - headRotateY: [null,i,i,0], headRotateX: [null,j,j,0], headRotateZ: [null,-i/4,-i/4,0], - eyeLookInLeft: [null,0], eyeLookOutLeft: [null,0], eyeLookInRight: [null,0], eyeLookOutRight: [null,0], - eyeContact: [0] - } - })); - } - - } - - // Make sure we do not overshoot - if ( dt > 2 * this.animFrameDur ) dt = 2 * this.animFrameDur; - - // Randomize facial expression by changing baseline - if ( this.viewName !== 'full' ) { - i = this.mtRandomized[ Math.floor( Math.random() * this.mtRandomized.length ) ]; - j = this.mtAvatar[i]; - if ( !j.needsUpdate ) { - Object.assign(j,{ base: (this.mood.baseline[i] || 0) + ( 1 + vol/255 ) * Math.random() / 5, needsUpdate: true }); - } - } - - // Animate - this.updatePoseBase(this.animClock); - if ( this.mixer ) { - this.mixer.update(dt / 1000 * this.mixer.timeScale); - } - this.updatePoseDelta(); - - - // Volume based head movement, set targets - if ( (this.isSpeaking || this.isListening) && isEyeContact ) { - if ( vol > this.volumeMax ) { - this.volumeHeadBase = 0.05; - if ( Math.random() > 0.6 ) { - this.volumeHeadTarget = - 0.05 - Math.random() / 15; - } - this.volumeMax = vol; - } else { - this.volumeMax *= 0.92; - this.volumeHeadTarget = this.volumeHeadBase - 0.9 * (this.volumeHeadBase - this.volumeHeadTarget); - } - } else { - this.volumeHeadTarget = 0; - this.volumeMax = 0; - } - i = this.volumeHeadTarget - this.volumeHeadCurrent; - j = Math.abs(i); - if ( j > 0.0001 ) { - k = j * (this.volumeHeadEasing( Math.min(1, this.volumeHeadVelocity * dt / 1000 / j ) / 2 + 0.5 ) - 0.5 ); - this.volumeHeadCurrent += Math.sign(i) * Math.min(j,k); - } - if ( Math.abs(this.volumeHeadCurrent) > 0.0001 ) { - q.setFromAxisAngle(axisx, this.volumeHeadCurrent ); - this.objectNeck.quaternion.multiply(q); - } - - // Hip-feet balance - box.setFromObject( this.armature ); - this.objectLeftToeBase.getWorldPosition(v); - this.objectRightToeBase.getWorldPosition(w); - this.objectHips.position.y -= box.min.y / 2; - this.objectHips.position.x -= (v.x+w.x)/4; - this.objectHips.position.z -= (v.z+w.z)/2; - - // Update Dynamic Bones - this.dynamicbones.update(dt); - - // Update morph targets - this.updateMorphTargets(dt); - - // Camera - if ( this.cameraClock !== null && this.cameraClock < 1000 ) { - this.cameraClock += dt; - if ( this.cameraClock > 1000 ) this.cameraClock = 1000; - let s = new THREE.Spherical().setFromVector3(this.cameraStart); - let sEnd = new THREE.Spherical().setFromVector3(this.cameraEnd); - s.phi += this.easing(this.cameraClock / 1000) * (sEnd.phi - s.phi); - s.theta += this.easing(this.cameraClock / 1000) * (sEnd.theta - s.theta); - s.radius += this.easing(this.cameraClock / 1000) * (sEnd.radius - s.radius); - s.makeSafe(); - this.camera.position.setFromSpherical( s ); - if ( this.controlsStart.x !== this.controlsEnd.x ) { - this.controls.target.copy( this.controlsStart.lerp( this.controlsEnd, this.easing(this.cameraClock / 1000) ) ); - } else { - s.setFromVector3(this.controlsStart); - sEnd.setFromVector3(this.controlsEnd); - s.phi += this.easing(this.cameraClock / 1000) * (sEnd.phi - s.phi); - s.theta += this.easing(this.cameraClock / 1000) * (sEnd.theta - s.theta); - s.radius += this.easing(this.cameraClock / 1000) * (sEnd.radius - s.radius); - s.makeSafe(); - this.controls.target.setFromSpherical( s ); - } - this.controls.update(); - } - - // Autorotate - if ( this.controls.autoRotate ) this.controls.update(); - - // Statistics end - if ( this.stats ) { - this.stats.end(); - } - - this.render(); - - } - - /** - * Reset all the visemes - */ - resetLips() { - this.visemeNames.forEach( x => { - this.morphs.forEach( y => { - const ndx = y.morphTargetDictionary['viseme_'+x]; - if ( ndx !== undefined ) { - y.morphTargetInfluences[ndx] = 0; - } - }); - }); - } - - /** - * Get lip-sync processor based on language. Import module dynamically. - * @param {string} lang Language - * @param {string} [path="./"] Module path - */ - lipsyncGetProcessor(lang, path="./") { - if ( !this.lipsync.hasOwnProperty(lang) ) { - const moduleName = path + 'lipsync-' + lang.toLowerCase() + '.mjs'; - const className = 'Lipsync' + lang.charAt(0).toUpperCase() + lang.slice(1); - import(moduleName).then( module => { - this.lipsync[lang] = new module[className]; - }); - } - } - - /** - * Preprocess text for tts/lipsync, including: - * - convert symbols/numbers to words - * - filter out characters that should be left unspoken - * @param {string} s Text - * @param {string} lang Language - * @return {string} Pre-processsed text. - */ - lipsyncPreProcessText(s,lang) { - const o = this.lipsync[lang] || Object.values(this.lipsync)[0]; - return o.preProcessText(s); - } - - /** - * Convert words to Oculus LipSync Visemes. - * @param {string} word Word - * @param {string} lang Language - * @return {Lipsync} Lipsync object. - */ - lipsyncWordsToVisemes(word,lang) { - const o = this.lipsync[lang] || Object.values(this.lipsync)[0]; - return o.wordsToVisemes(word); - } - - - /** - * Add text to the speech queue. - * @param {string} s Text. - * @param {Options} [opt=null] Text-specific options for lipsync/TTS language, voice, rate and pitch, mood and mute - * @param {subtitlesfn} [onsubtitles=null] Callback when a subtitle is written - * @param {number[][]} [excludes=null] Array of [start, end] index arrays to not speak - */ - speakText(s, opt = null, onsubtitles = null, excludes = null ) { - opt = opt || {}; - - // Classifiers - const dividersSentence = /[!\.\?\n\p{Extended_Pictographic}]/ug; - const dividersWord = /[ ]/ug; - const speakables = /[\p{L}\p{N},\.\p{Quotation_Mark}!€\$\+\p{Dash_Punctuation}%&\?]/ug; - const emojis = /[\p{Extended_Pictographic}]/ug; - const lipsyncLang = opt.lipsyncLang || this.avatar.lipsyncLang || this.opt.lipsyncLang; - - let markdownWord = ''; // markdown word - let textWord = ''; // text-to-speech word - let markId = 0; // SSML mark id - let ttsSentence = []; // Text-to-speech sentence - let lipsyncAnim = []; // Lip-sync animation sequence - const letters = [...s]; - for( let i=0; i (i < x[0]) || (i > x[1]) ) ) { - textWord += letters[i]; - } - } - - // Add words to sentence and animations - if ( isEndOfWord || isEndOfSentence || isLast ) { - - // Add to text-to-speech sentence - if ( textWord.length ) { - textWord = this.lipsyncPreProcessText(textWord, lipsyncLang); - if ( textWord.length ) { - ttsSentence.push( { - mark: markId, - word: textWord - }); - } - } - - // Push subtitles to animation queue - if ( markdownWord.length ) { - lipsyncAnim.push( { - mark: markId, - template: { name: 'subtitles' }, - ts: [0], - vs: { - subtitles: [markdownWord] - }, - }); - markdownWord = ''; - } - - // Push visemes to animation queue - if ( textWord.length ) { - const val = this.lipsyncWordsToVisemes(textWord, lipsyncLang); - if ( val && val.visemes && val.visemes.length ) { - const d = val.times[ val.visemes.length-1 ] + val.durations[ val.visemes.length-1 ]; - for( let j=0; j 0 ) { - for( let j=0; j setTimeout(() => rej("p2"), 1000)); - try { - await Promise.race([resume, timeout]); - } catch(e) { - console.log("Can't play audio. Web Audio API suspended. This is often due to calling some speak method before the first user action, which is typically prevented by the browser."); - this.playAudio(true); - return; - } - } - - // AudioBuffer - let audio; - if ( Array.isArray(item.audio) ) { - // Convert from PCM samples - let buf = this.concatArrayBuffers( item.audio ); - audio = this.pcmToAudioBuffer(buf); - } else { - audio = item.audio; - } - - // Create audio source - this.audioSpeechSource = this.audioCtx.createBufferSource(); - this.audioSpeechSource.buffer = audio; - this.audioSpeechSource.playbackRate.value = 1 / this.animSlowdownRate; - this.audioSpeechSource.connect(this.audioAnalyzerNode); - this.audioSpeechSource.addEventListener('ended', () => { - this.audioSpeechSource.disconnect(); - this.playAudio(true); - }, { once: true }); - - // Rescale lipsync and push to queue - let delay = 0; - if ( item.anim ) { - // Find the lowest negative time point, if any - delay = Math.abs(Math.max(0, Math.min( ...item.anim.map( x => Math.min(...x.ts) ) ) ) ); - item.anim.forEach( x => { - for(let i=0; i a+b,0); - this.animQueue.push( this.animFactory( line.emoji ) ); - setTimeout( this.startSpeaking.bind(this), duration, true ); - } else if ( line.break ) { - // Break - setTimeout( this.startSpeaking.bind(this), line.break, true ); - } else if ( line.audio ) { - - // Look at the camera - this.lookAtCamera(500); - this.speakWithHands(); - - // Make a playlist - this.audioPlaylist.push({ anim: line.anim, audio: line.audio }); - this.onSubtitles = line.onSubtitles || null; - this.resetLips(); - if ( line.mood ) this.setMood( line.mood ); - this.playAudio(); - - } else if ( line.text ) { - - // Look at the camera - this.lookAtCamera(500); - - // Spoken text - try { - // Convert text to SSML - let ssml = ""; - line.text.forEach( (x,i) => { - // Add mark - if (i > 0) { - ssml += " "; - } - - // Add word - ssml += x.word.replaceAll('&','&') - .replaceAll('<','<') - .replaceAll('>','>') - .replaceAll('"','"') - .replaceAll('\'',''') - .replace(/^\p{Dash_Punctuation}$/ug,''); - - }); - ssml += ""; - - - const o = { - method: "POST", - headers: { - "Content-Type": "application/json; charset=utf-8" - }, - body: JSON.stringify({ - "input": { - "ssml": ssml - }, - "voice": { - "languageCode": line.lang || this.avatar.ttsLang || this.opt.ttsLang, - "name": line.voice || this.avatar.ttsVoice || this.opt.ttsVoice - }, - "audioConfig": { - "audioEncoding": this.ttsAudioEncoding, - "speakingRate": (line.rate || this.avatar.ttsRate || this.opt.ttsRate) + this.mood.speech.deltaRate, - "pitch": (line.pitch || this.avatar.ttsPitch || this.opt.ttsPitch) + this.mood.speech.deltaPitch, - "volumeGainDb": (line.volume || this.avatar.ttsVolume || this.opt.ttsVolume) + this.mood.speech.deltaVolume - }, - "enableTimePointing": [ 1 ] // Timepoint information for mark tags - }) - }; - - // JSON Web Token - if ( this.opt.jwtGet && typeof this.opt.jwtGet === "function" ) { - o.headers["Authorization"] = "Bearer " + await this.opt.jwtGet(); - } - - const res = await fetch( this.opt.ttsEndpoint + (this.opt.ttsApikey ? "?key=" + this.opt.ttsApikey : ''), o); - const data = await res.json(); - - if ( res.status === 200 && data && data.audioContent ) { - - // Audio data - const buf = this.b64ToArrayBuffer(data.audioContent); - const audio = await this.audioCtx.decodeAudioData( buf ); - this.speakWithHands(); - - // Workaround for Google TTS not providing all timepoints - const times = [ 0 ]; - let markIndex = 0; - line.text.forEach( (x,i) => { - if ( i > 0 ) { - let ms = times[ times.length - 1 ]; - if ( data.timepoints[markIndex] ) { - ms = data.timepoints[markIndex].timeSeconds * 1000; - if ( data.timepoints[markIndex].markName === ""+x.mark ) { - markIndex++; - } - } - times.push( ms ); - } - }); - - // Word-to-audio alignment - const timepoints = [ { mark: 0, time: 0 } ]; - times.forEach( (x,i) => { - if ( i>0 ) { - let prevDuration = x - times[i-1]; - if ( prevDuration > 150 ) prevDuration - 150; // Trim out leading space - timepoints[i-1].duration = prevDuration; - timepoints.push( { mark: i, time: x }); - } - }); - let d = 1000 * audio.duration; // Duration in ms - if ( d > this.opt.ttsTrimEnd ) d = d - this.opt.ttsTrimEnd; // Trim out silence at the end - timepoints[timepoints.length-1].duration = d - timepoints[timepoints.length-1].time; - - // Re-set animation starting times and rescale durations - line.anim.forEach( x => { - const timepoint = timepoints[x.mark]; - if ( timepoint ) { - for(let i=0; i { - for(let j=0; j x.template.name !== 'viseme' && x.template.name !== 'subtitles' ); - if ( this.armature ) { - this.resetLips(); - this.render(); - } - } - - /** - * Stop speaking and clear the speech queue. - */ - stopSpeaking() { - try { this.audioSpeechSource.stop(); } catch(error) {} - this.audioPlaylist.length = 0; - this.speechQueue.length = 0; - this.animQueue = this.animQueue.filter( x => x.template.name !== 'viseme' && x.template.name !== 'subtitles' ); - this.stateName = 'idle'; - this.isSpeaking = false; - this.isAudioPlaying = false; - if ( this.armature ) { - this.resetLips(); - this.render(); - } - } - - /** - * Make eye contact. - * @param {number} t Time in milliseconds - */ - makeEyeContact(t) { - this.animQueue.push( this.animFactory( { - name: 'eyecontact', dt: [0,t], vs: { eyeContact: [1] } - })); - } - - /** - * Look ahead. - * @param {number} t Time in milliseconds - */ - lookAhead(t) { - - if ( t ) { - - // Randomize head/eyes ratio - let drotx = (Math.random() - 0.5) / 4; - let droty = (Math.random() - 0.5) / 4; - - // Remove old, if any - let old = this.animQueue.findIndex( y => y.template.name === 'lookat' ); - if ( old !== -1 ) { - this.animQueue.splice(old, 1); - } - - // Add new anim - const templateLookAt = { - name: 'lookat', - dt: [750,t], - vs: { - bodyRotateX: [ drotx ], - bodyRotateY: [ droty ], - eyesRotateX: [ - 3 * drotx + 0.1 ], - eyesRotateY: [ - 5 * droty ], - browInnerUp: [[0,0.7]], - mouthLeft: [[0,0.7]], - mouthRight: [[0,0.7]], - eyeContact: [0], - headMove: [0] - } - }; - this.animQueue.push( this.animFactory( templateLookAt ) ); - } - - } - - /** - * Turn head and eyes to look at the camera. - * @param {number} t Time in milliseconds - */ - lookAtCamera(t) { - - if ( this.avatar.hasOwnProperty('avatarIgnoreCamera') ) { - if ( this.avatar.avatarIgnoreCamera ) { - this.lookAhead(t); - } else { - this.lookAt( null, null, t ); - } - } else if ( this.opt.avatarIgnoreCamera ) { - this.lookAhead(t); - } else { - this.lookAt( null, null, t ); - } - - } - - /** - * Turn head and eyes to look at the point (x,y). - * @param {number} x X-coordinate relative to visual viewport - * @param {number} y Y-coordinate relative to visual viewport - * @param {number} t Time in milliseconds - */ - lookAt(x,y,t) { - - // Eyes position - const rect = this.nodeAvatar.getBoundingClientRect(); - this.objectLeftEye.updateMatrixWorld(true); - this.objectRightEye.updateMatrixWorld(true); - const plEye = new THREE.Vector3().setFromMatrixPosition(this.objectLeftEye.matrixWorld); - const prEye = new THREE.Vector3().setFromMatrixPosition(this.objectRightEye.matrixWorld); - const pEyes = new THREE.Vector3().addVectors( plEye, prEye ).divideScalar( 2 ); - - pEyes.project(this.camera); - let eyesx = (pEyes.x + 1) / 2 * rect.width + rect.left; - let eyesy = -(pEyes.y - 1) / 2 * rect.height + rect.top; - - // if coordinate not specified, look at the camera - if ( x === null ) x = eyesx; - if ( y === null ) y = eyesy; - - // Use body/camera rotation to determine the required head rotation - q.copy( this.poseTarget.props['Hips.quaternion'] ); - q.multiply( this.poseTarget.props['Spine.quaternion'] ); - q.multiply( this.poseTarget.props['Spine1.quaternion'] ); - q.multiply( this.poseTarget.props['Spine2.quaternion'] ); - q.multiply( this.poseTarget.props['Neck.quaternion'] ); - q.multiply( this.poseTarget.props['Head.quaternion'] ); - e.setFromQuaternion(q); - let rx = e.x / (40/24); // Refer to setValue(bodyRotateX) - let ry = e.y / (9/4); // Refer to setValue(bodyRotateY) - let camerarx = Math.min(0.4, Math.max(-0.4,this.camera.rotation.x)); - let camerary = Math.min(0.4, Math.max(-0.4,this.camera.rotation.y)); - - // Calculate new delta - let maxx = Math.max( window.innerWidth - eyesx, eyesx ); - let maxy = Math.max( window.innerHeight - eyesy, eyesy ); - let rotx = this.convertRange(y,[eyesy-maxy,eyesy+maxy],[-0.3,0.6]) - rx + camerarx; - let roty = this.convertRange(x,[eyesx-maxx,eyesx+maxx],[-0.8,0.8]) - ry + camerary; - rotx = Math.min(0.6,Math.max(-0.3,rotx)); - roty = Math.min(0.8,Math.max(-0.8,roty)); - - // Randomize head/eyes ratio - let drotx = (Math.random() - 0.5) / 4; - let droty = (Math.random() - 0.5) / 4; - - if ( t ) { - - // Remove old, if any - let old = this.animQueue.findIndex( y => y.template.name === 'lookat' ); - if ( old !== -1 ) { - this.animQueue.splice(old, 1); - } - - // Add new anim - const templateLookAt = { - name: 'lookat', - dt: [750,t], - vs: { - bodyRotateX: [ rotx + drotx ], - bodyRotateY: [ roty + droty ], - eyesRotateX: [ - 3 * drotx + 0.1 ], - eyesRotateY: [ - 5 * droty ], - browInnerUp: [[0,0.7]], - mouthLeft: [[0,0.7]], - mouthRight: [[0,0.7]], - eyeContact: [0], - headMove: [0] - } - }; - this.animQueue.push( this.animFactory( templateLookAt ) ); - } - } - - - /** - * Set the closest hand to touch at (x,y). - * @param {number} x X-coordinate relative to visual viewport - * @param {number} y Y-coordinate relative to visual viewport - * @return {Boolean} If true, (x,y) touch the avatar - */ - touchAt(x,y) { - - const rect = this.nodeAvatar.getBoundingClientRect(); - const pointer = new THREE.Vector2( - ( (x - rect.left) / rect.width ) * 2 - 1, - - ( (y - rect.top) / rect.height ) * 2 + 1 - ); - const raycaster = new THREE.Raycaster(); - raycaster.setFromCamera(pointer,this.camera); - const intersects = raycaster.intersectObject(this.armature); - if ( intersects.length > 0 ) { - const target = intersects[0].point; - const LeftArmPos = new THREE.Vector3(); - const RightArmPos = new THREE.Vector3(); - this.objectLeftArm.getWorldPosition(LeftArmPos); - this.objectRightArm.getWorldPosition(RightArmPos); - const LeftD2 = LeftArmPos.distanceToSquared(target); - const RightD2 = RightArmPos.distanceToSquared(target); - if ( LeftD2 < RightD2 ) { - this.ikSolve( { - iterations: 20, root: "LeftShoulder", effector: "LeftHandMiddle1", - links: [ - { link: "LeftHand", minx: -0.5, maxx: 0.5, miny: -1, maxy: 1, minz: -0.5, maxz: 0.5, maxAngle: 0.1 }, - { link: "LeftForeArm", minx: -0.5, maxx: 1.5, miny: -1.5, maxy: 1.5, minz: -0.5, maxz: 3, maxAngle: 0.2 }, - { link: "LeftArm", minx: -1.5, maxx: 1.5, miny: 0, maxy: 0, minz: -1, maxz: 3 } - ] - }, target, false, 1000 ); - this.setValue("handFistLeft",0); - } else { - this.ikSolve( { - iterations: 20, root: "RightShoulder", effector: "RightHandMiddle1", - links: [ - { link: "RightHand", minx: -0.5, maxx: 0.5, miny: -1, maxy: 1, minz: -0.5, maxz: 0.5, maxAngle: 0.1 }, - { link: "RightForeArm", minx: -0.5, maxx: 1.5, miny: -1.5, maxy: 1.5, minz: -3, maxz: 0.5, maxAngle: 0.2 }, - { link: "RightArm", minx: -1.5, maxx: 1.5, miny: 0, maxy: 0, minz: -1, maxz: 3 } - ] - }, target, false, 1000 ); - this.setValue("handFistRight",0); - } - } else { - ["LeftArm","LeftForeArm","LeftHand","RightArm","RightForeArm","RightHand"].forEach( x => { - let key = x + ".quaternion"; - this.poseTarget.props[key].copy( this.getPoseTemplateProp(key) ); - this.poseTarget.props[key].t = this.animClock; - this.poseTarget.props[key].d = 1000; - }); - } - - return ( intersects.length > 0 ); - } - - /** - * Talk with hands. - * @param {number} [delay=0] Delay in milliseconds - * @param {number} [prob=1] Probability of hand movement - */ - speakWithHands(delay=0,prob=0.5) { - - // Only if we are standing and not bending and probabilities match up - if ( this.mixer || this.gesture || !this.poseTarget.template.standing || this.poseTarget.template.bend || Math.random()>prob ) return; - - // Random targets for left hand - this.ikSolve( { - root: "LeftShoulder", effector: "LeftHandMiddle1", - links: [ - { link: "LeftHand", minx: -0.5, maxx: 0.5, miny: -1, maxy: 1, minz: -0.5, maxz: 0.5 }, - { link: "LeftForeArm", minx: -0.5, maxx: 1.5, miny: -1.5, maxy: 1.5, minz: -0.5, maxz: 3 }, - { link: "LeftArm", minx: -1.5, maxx: 1.5, miny: -1.5, maxy: 1.5, minz: -1, maxz: 3 } - ] - }, new THREE.Vector3( - this.gaussianRandom(0,0.5), - this.gaussianRandom(-0.8,-0.2), - this.gaussianRandom(0,0.5) - ), true); - - // Random target for right hand - this.ikSolve( { - root: "RightShoulder", effector: "RightHandMiddle1", - links: [ - { link: "RightHand", minx: -0.5, maxx: 0.5, miny: -1, maxy: 1, minz: -0.5, maxz: 0.5 }, - { link: "RightForeArm", minx: -0.5, maxx: 1.5, miny: -1.5, maxy: 1.5, minz: -3, maxz: 0.5 }, - { link: "RightArm" } - ] - }, new THREE.Vector3( - this.gaussianRandom(-0.5,0), - this.gaussianRandom(-0.8,-0.2), - this.gaussianRandom(0,0.5) - ), true); - - // Moveto - const dt = []; - const moveto = []; - - // First move - dt.push( 100 + Math.round( Math.random() * 500 ) ); - moveto.push( { duration: 1000, props: { - "LeftHand.quaternion": new THREE.Quaternion().setFromEuler( new THREE.Euler( 0, -1 - Math.random(), 0 ) ), - "RightHand.quaternion": new THREE.Quaternion().setFromEuler( new THREE.Euler( 0, 1 + Math.random(), 0 ) ) - } } ); - ["LeftArm","LeftForeArm","RightArm","RightForeArm"].forEach( x => { - moveto[0].props[x+'.quaternion'] = this.ikMesh.getObjectByName(x).quaternion.clone(); - }); - - // Return to original target - dt.push( 1000 + Math.round( Math.random() * 500 ) ); - moveto.push( { duration: 2000, props: {} } ); - ["LeftArm","LeftForeArm","RightArm","RightForeArm","LeftHand","RightHand"].forEach( x => { - moveto[1].props[x+'.quaternion'] = null; - }); - - // Make an animation - const anim = this.animFactory( { - name: 'talkinghands', - delay: delay, - dt: dt, - vs: { moveto: moveto } - }); - this.animQueue.push( anim ); - - } - - /** - * Get slowdown. - * @return {numeric} Slowdown factor. - */ - getSlowdownRate(k) { - return this.animSlowdownRate; - } - - /** - * Set slowdown. - * @param {numeric} k Slowdown factor. - */ - setSlowdownRate(k) { - this.animSlowdownRate = k; - this.audioSpeechSource.playbackRate.value = 1 / this.animSlowdownRate; - this.audioBackgroundSource.playbackRate.value = 1 / this.animSlowdownRate; - } - - /** - * Get autorotate speed. - * @return {numeric} Autorotate speed. - */ - getAutoRotateSpeed(k) { - return this.controls.autoRotateSpeed; - } - - /** - * Set autorotate. - * @param {numeric} speed Autorotate speed, e.g. value 2 = 30 secs per orbit at 60fps. - */ - setAutoRotateSpeed(speed) { - this.controls.autoRotateSpeed = speed; - this.controls.autoRotate = (speed > 0); - } - - /** - * Start animation cycle. - */ - start() { - if ( this.armature && this.isRunning === false ) { - this.audioCtx.resume(); - this.animTimeLast = performance.now(); - this.isRunning = true; - requestAnimationFrame( this.animate.bind(this) ); - } - } - - /** - * Stop animation cycle. - */ - stop() { - this.isRunning = false; - this.audioCtx.suspend(); - } - - /** - * Start listening incoming audio. - * @param {AnalyserNode} analyzer Analyzer node for incoming audio - * @param {Object} [opt={}] Options - * @param {function} [onchange=null] Callback function for start - */ - startListening(analyzer, opt = {}, onchange = null) { - this.listeningAnalyzer = analyzer; - this.listeningAnalyzer.fftSize = 256; - this.listeningAnalyzer.smoothingTimeConstant = 0.1; - this.listeningAnalyzer.minDecibels = -70; - this.listeningAnalyzer.maxDecibels = -10; - this.listeningOnchange = (onchange && typeof onchange === 'function') ? onchange : null; - - this.listeningSilenceThresholdLevel = opt?.hasOwnProperty('listeningSilenceThresholdLevel') ? opt.listeningSilenceThresholdLevel : this.opt.listeningSilenceThresholdLevel; - this.listeningSilenceThresholdMs = opt?.hasOwnProperty('listeningSilenceThresholdMs') ? opt.listeningSilenceThresholdMs : this.opt.listeningSilenceThresholdMs; - this.listeningSilenceDurationMax = opt?.hasOwnProperty('listeningSilenceDurationMax') ? opt.listeningSilenceDurationMax : this.opt.listeningSilenceDurationMax; - this.listeningActiveThresholdLevel = opt?.hasOwnProperty('listeningActiveThresholdLevel') ? opt.listeningActiveThresholdLevel : this.opt.listeningActiveThresholdLevel; - this.listeningActiveThresholdMs = opt?.hasOwnProperty('listeningActiveThresholdMs') ? opt.listeningActiveThresholdMs : this.opt.listeningActiveThresholdMs; - this.listeningActiveDurationMax = opt?.hasOwnProperty('listeningActiveDurationMax') ? opt.listeningActiveDurationMax : this.opt.listeningActiveDurationMax; - - this.listeningActive = false; - this.listeningVolume = 0; - this.listeningTimer = 0; - this.listeningTimerTotal = 0; - this.isListening = true; - } - - /** - * Stop animation cycle. - */ - stopListening() { - this.isListening = false; - } - - /** - * Play RPM/Mixamo animation clip. - * @param {string|Object} url URL to animation file FBX - * @param {progressfn} [onprogress=null] Callback for progress - * @param {number} [dur=10] Duration in seconds, but at least once - * @param {number} [ndx=0] Index of the clip - * @param {number} [scale=0.01] Position scale factor - */ - async playAnimation(url, onprogress=null, dur=10, ndx=0, scale=0.01) { - if ( !this.armature ) return; - - let item = this.animClips.find( x => x.url === url+'-'+ndx ); - if ( item ) { - - // Reset pose update - let anim = this.animQueue.find( x => x.template.name === 'pose' ); - if ( anim ) { - anim.ts[0] = Infinity; - } - - // Set new pose - Object.entries(item.pose.props).forEach( x => { - this.poseBase.props[x[0]] = x[1].clone(); - this.poseTarget.props[x[0]] = x[1].clone(); - this.poseTarget.props[x[0]].t = 0; - this.poseTarget.props[x[0]].d = 1000; - }); - - // Create a new mixer - this.mixer = new THREE.AnimationMixer(this.armature); - this.mixer.addEventListener( 'finished', this.stopAnimation.bind(this), { once: true }); - - // Play action - const repeat = Math.ceil(dur / item.clip.duration); - const action = this.mixer.clipAction(item.clip); - action.setLoop( THREE.LoopRepeat, repeat ); - action.clampWhenFinished = true; - action.fadeIn(0.5).play(); - - } else { - - // Load animation - const loader = new FBXLoader(); - - let fbx = await loader.loadAsync( url, onprogress ); - - if ( fbx && fbx.animations && fbx.animations[ndx] ) { - let anim = fbx.animations[ndx]; - - // Rename and scale Mixamo tracks, create a pose - const props = {}; - anim.tracks.forEach( t => { - t.name = t.name.replaceAll('mixamorig',''); - const ids = t.name.split('.'); - if ( ids[1] === 'position' ) { - for(let i=0; i x.template.name === 'pose' ); - if ( anim ) { - anim.ts[0] = this.animClock; - } - this.setPoseFromTemplate( null ); - - } - - - /** - * Play RPM/Mixamo pose. - * @param {string|Object} url Pose name | URL to FBX - * @param {progressfn} [onprogress=null] Callback for progress - * @param {number} [dur=5] Duration of the pose in seconds - * @param {number} [ndx=0] Index of the clip - * @param {number} [scale=0.01] Position scale factor - */ - async playPose(url, onprogress=null, dur=5, ndx=0, scale=0.01) { - - if ( !this.armature ) return; - - // Check if we already have the pose template ready - let pose = this.poseTemplates[url]; - if ( !pose ) { - const item = this.animPoses.find( x => x.url === url+'-'+ndx ); - if ( item ) { - pose = item.pose; - } - } - - // If we have the template, use it, otherwise try to load it - if ( pose ) { - - this.poseName = url; - - this.mixer = null; - let anim = this.animQueue.find( x => x.template.name === 'pose' ); - if ( anim ) { - anim.ts[0] = this.animClock + (dur * 1000) + 2000; - } - this.setPoseFromTemplate( pose ); - - } else { - - // Load animation - const loader = new FBXLoader(); - - let fbx = await loader.loadAsync( url, onprogress ); - - if ( fbx && fbx.animations && fbx.animations[ndx] ) { - let anim = fbx.animations[ndx]; - - // Create a pose - const props = {}; - anim.tracks.forEach( t => { - - // Rename and scale Mixamo tracks - t.name = t.name.replaceAll('mixamorig',''); - const ids = t.name.split('.'); - if ( ids[1] === 'position' ) { - props[t.name] = new THREE.Vector3( t.values[0] * scale, t.values[1] * scale, t.values[2] * scale); - } else if ( ids[1] === 'quaternion' ) { - props[t.name] = new THREE.Quaternion( t.values[0], t.values[1], t.values[2], t.values[3] ); - } else if ( ids[1] === 'rotation' ) { - props[ids[0]+".quaternion"] = new THREE.Quaternion().setFromEuler(new THREE.Euler( t.values[0], t.values[1], t.values[2],'XYZ' )).normalize(); - } - }); - - // Add to pose - const newPose = { props: props }; - if ( props['Hips.position'] ) { - if ( props['Hips.position'].y < 0.5 ) { - newPose.lying = true; - } else { - newPose.standing = true; - } - } - this.animPoses.push({ - url: url+'-'+ndx, - pose: newPose - }); - - // Play - this.playPose(url, onprogress, dur, ndx, scale); - - } else { - const msg = 'Pose ' + url + ' (ndx=' + ndx + ') not found'; - console.error(msg); - } - } - } - - /** - * Stop the pose. (Functionality is the same as in stopAnimation.) - */ - stopPose() { - this.stopAnimation(); - } - - /** - * Play a gesture, which is either a hand gesture, an emoji animation or their - * combination. - * @param {string} name Gesture name - * @param {number} [dur=3] Duration of the gesture in seconds - * @param {boolean} [mirror=false] Mirror gesture - * @param {number} [ms=1000] Transition time in milliseconds - */ - playGesture(name, dur=3, mirror=false, ms=1000) { - - if ( !this.armature ) return; - - // Hand gesture, if any - let g = this.gestureTemplates[name]; - if ( g ) { - - // New gesture always overrides the existing one - if ( this.gestureTimeout ) { - clearTimeout( this.gestureTimeout ); - this.gestureTimeout = null; - } - - // Stop talking hands animation - let ndx = this.animQueue.findIndex( y => y.template.name === "talkinghands" ); - if ( ndx !== -1 ) { - this.animQueue[ndx].ts = this.animQueue[ndx].ts.map( x => 0 ); - } - - // Set gesture - this.gesture = this.propsToThreeObjects( g ); - if ( mirror ) { - this.gesture = this.mirrorPose( this.gesture ); - } - if ( name === "namaste" && this.avatar.body === 'M' ) { - // Work-a-round for male model so that the hands meet - this.gesture["RightArm.quaternion"].rotateTowards( new THREE.Quaternion(0,1,0,0), -0.25); - this.gesture["LeftArm.quaternion"].rotateTowards( new THREE.Quaternion(0,1,0,0), -0.25); - } - - // Apply to target - for( let [p,val] of Object.entries(this.gesture) ) { - val.t = this.animClock; - val.d = ms; - if ( this.poseTarget.props.hasOwnProperty(p) ) { - this.poseTarget.props[p].copy(val); - this.poseTarget.props[p].t = this.animClock; - this.poseTarget.props[p].d = ms; - } - } - - // Timer - if ( dur && Number.isFinite(dur) ) { - this.gestureTimeout = setTimeout( this.stopGesture.bind(this,ms), 1000 * dur); - } - } - - // Animated emoji, if any - let em = this.animEmojis[name]; - if ( em ) { - - // Follow link - if ( em && em.link ) { - em = this.animEmojis[em.link]; - } - - if ( em ) { - // Look at the camera for 500 ms - this.lookAtCamera(500); - - // Create animation and tag as gesture - const anim = this.animFactory( em ); - anim.gesture = true; - - // Rescale duration - if ( dur && Number.isFinite(dur) ) { - const first = anim.ts[0]; - const last = anim.ts[ anim.ts.length -1 ]; - const total = last - first; - const excess = (dur * 1000) - total; - - // If longer, increase longer parts; if shorter, scale everything - if ( excess > 0 ) { - const dt = []; - for( let i=1; i x / total ); - const excess = dur * 1000 - total; - anim.ts = anim.ts.map( (x,i,arr) => { - return (i===0) ? first : (arr[i-1] + dt[i-1] + rescale[i-1] * excess); - }); - } else { - const scale = (dur * 1000) / total; - anim.ts = anim.ts.map( x => first + scale * (x - first) ); - } - } - - this.animQueue.push( anim ); - } - } - - } - - /** - * Stop the gesture. - * @param {number} [ms=1000] Transition time in milliseconds - */ - stopGesture(ms=1000) { - - // Stop gesture timer - if ( this.gestureTimeout ) { - clearTimeout( this.gestureTimeout ); - this.gestureTimeout = null; - } - - // Stop hand gesture, if any - if ( this.gesture ) { - const gs = Object.entries(this.gesture); - this.gesture = null; - for( const [p,val] of gs ) { - if ( this.poseTarget.props.hasOwnProperty(p) ) { - this.poseTarget.props[p].copy( this.getPoseTemplateProp(p) ); - this.poseTarget.props[p].t = this.animClock; - this.poseTarget.props[p].d = ms; - } - } - } - - // Stop animated emoji gesture, if any - let i = this.animQueue.findIndex( y => y.gesture ); - if ( i !== -1 ) { - this.animQueue.splice(i, 1); - } - - } - - /** - * Cyclic Coordinate Descent (CCD) Inverse Kinematic (IK) algorithm. - * Adapted from: - * https://github.com/mrdoob/three.js/blob/master/examples/jsm/animation/CCDIKSolver.js - * @param {Object} ik IK configuration object - * @param {Vector3} [target=null] Target coordinate, if null return to template - * @param {Boolean} [relative=false] If true, target is relative to root - * @param {numeric} [d=null] If set, apply in d milliseconds - */ - ikSolve(ik, target=null, relative=false, d=null) { - const targetVec = new THREE.Vector3(); - const effectorPos = new THREE.Vector3(); - const effectorVec = new THREE.Vector3(); - const linkPos = new THREE.Vector3(); - const invLinkQ = new THREE.Quaternion(); - const linkScale = new THREE.Vector3(); - const axis = new THREE.Vector3(); - const vector = new THREE.Vector3(); - - // Reset IK setup positions and rotations - const root = this.ikMesh.getObjectByName(ik.root); - root.position.setFromMatrixPosition( this.armature.getObjectByName(ik.root).matrixWorld ); - root.quaternion.setFromRotationMatrix( this.armature.getObjectByName(ik.root).matrixWorld ); - if ( target && relative ) { - target.add( root.position ); - } - const effector = this.ikMesh.getObjectByName(ik.effector); - const links = ik.links; - links.forEach( x => { - x.bone = this.ikMesh.getObjectByName(x.link); - x.bone.quaternion.copy( this.getPoseTemplateProp(x.link+'.quaternion') ); - }); - root.updateMatrixWorld(true); - const iterations = ik.iterations || 10; - - // Iterate - if ( target ) { - for ( let i = 0; i < iterations; i ++ ) { - let rotated = false; - for ( let j = 0, jl = links.length; j < jl; j++ ) { - const bone = links[j].bone; - bone.matrixWorld.decompose( linkPos, invLinkQ, linkScale ); - invLinkQ.invert(); - effectorPos.setFromMatrixPosition( effector.matrixWorld ); - effectorVec.subVectors( effectorPos, linkPos ); - effectorVec.applyQuaternion( invLinkQ ); - effectorVec.normalize(); - targetVec.subVectors( target, linkPos ); - targetVec.applyQuaternion( invLinkQ ); - targetVec.normalize(); - let angle = targetVec.dot( effectorVec ); - if ( angle > 1.0 ) { - angle = 1.0; - } else if ( angle < - 1.0 ) { - angle = - 1.0; - } - angle = Math.acos( angle ); - if ( angle < 1e-5 ) continue; - if ( links[j].minAngle !== undefined && angle < links[j].minAngle ) { - angle = links[j].minAngle; - } - if ( links[j].maxAngle !== undefined && angle > links[j].maxAngle ) { - angle = links[j].maxAngle; - } - axis.crossVectors( effectorVec, targetVec ); - axis.normalize(); - q.setFromAxisAngle( axis, angle ); - bone.quaternion.multiply( q ); - - // Constraints - bone.rotation.setFromVector3( vector.setFromEuler( bone.rotation ).clamp( new THREE.Vector3( - links[j].minx !== undefined ? links[j].minx : -Infinity, - links[j].miny !== undefined ? links[j].miny : -Infinity, - links[j].minz !== undefined ? links[j].minz : -Infinity - ), new THREE.Vector3( - links[j].maxx !== undefined ? links[j].maxx : Infinity, - links[j].maxy !== undefined ? links[j].maxy : Infinity, - links[j].maxz !== undefined ? links[j].maxz : Infinity - )) ); - - bone.updateMatrixWorld( true ); - rotated = true; - } - if ( !rotated ) break; - } - } - - // Apply - if ( d ) { - links.forEach( x => { - this.poseTarget.props[x.link+".quaternion"].copy( x.bone.quaternion ); - this.poseTarget.props[x.link+".quaternion"].t = this.animClock; - this.poseTarget.props[x.link+".quaternion"].d = d; - }); - } - } - -} - -export { TalkingHead }; \ No newline at end of file diff --git a/client/src/components/ApiKeyModal.jsx b/client/src/components/ApiKeyModal.jsx index 0a09e30..a92aff1 100644 --- a/client/src/components/ApiKeyModal.jsx +++ b/client/src/components/ApiKeyModal.jsx @@ -82,12 +82,31 @@ const ApiKeyModal = ({ missing, provider = 'gemini', onSelectProvider, onClose, }; useEffect(() => { - const storedOpenAI = localStorage.getItem('OPENAI_API_KEY') || ''; - const storedGemini = localStorage.getItem('GEMINI_API_KEY') || ''; - const storedTts = localStorage.getItem('TTS_API_KEY') || ''; - setOpenaiKey(storedOpenAI); - setGeminiKey(storedGemini); - setTtsKey(storedTts); + // Try to load from server first, then fall back to localStorage + const loadKeys = async () => { + try { + const response = await fetch(`${API_CONFIG.BASE_URL}/api/llm-keys`); + if (response.ok) { + const data = await response.json(); + setOpenaiKey(data.openai || localStorage.getItem('OPENAI_API_KEY') || ''); + setGeminiKey(data.gemini || localStorage.getItem('GEMINI_API_KEY') || ''); + setTtsKey(data.tts || localStorage.getItem('TTS_API_KEY') || ''); + return; + } + } catch (error) { + console.log('Could not fetch keys from server, using localStorage'); + } + + // Fallback to localStorage + const storedOpenAI = localStorage.getItem('OPENAI_API_KEY') || ''; + const storedGemini = localStorage.getItem('GEMINI_API_KEY') || ''; + const storedTts = localStorage.getItem('TTS_API_KEY') || ''; + setOpenaiKey(storedOpenAI); + setGeminiKey(storedGemini); + setTtsKey(storedTts); + }; + + loadKeys(); }, [provider]); return ( diff --git a/client/src/components/Header.jsx b/client/src/components/Header.jsx index a7cc74c..4a04a81 100644 --- a/client/src/components/Header.jsx +++ b/client/src/components/Header.jsx @@ -162,7 +162,7 @@ const Header = ({ mode, setMode, onOpenKeys }) => {
{/* Left section */}
- DialogLab + VOX LAB | v1.0.0
diff --git a/client/src/components/Home.jsx b/client/src/components/Home.jsx index d36ca20..9169d67 100644 --- a/client/src/components/Home.jsx +++ b/client/src/components/Home.jsx @@ -134,6 +134,11 @@ const TalkingHeadComponent = () => { } }; + // Expose avatar instances ref globally for audio context management + useEffect(() => { + window.avatarInstancesRef = avatarInstancesRef; + }, []); + const handleTopicChange = (topic) => { setCurrentTopic(topic); @@ -951,6 +956,38 @@ const TalkingHeadComponent = () => { } }; + // Resume AudioContext on user interaction (required by browser policies) + useEffect(() => { + const handleUserInteraction = () => { + // Find all audio contexts and resume them + const audioContexts = document.querySelectorAll('[data-audio-context]'); + + // Also try to resume any TalkingHead instances + if (window.avatarInstancesRef?.current) { + Object.values(window.avatarInstancesRef.current).forEach(instance => { + if (instance?.audioCtx && instance.audioCtx.state === 'suspended') { + instance.audioCtx.resume().catch(e => console.log('AudioContext resume failed:', e)); + } + }); + } + + // Remove listener after first interaction + document.removeEventListener('click', handleUserInteraction); + document.removeEventListener('touchstart', handleUserInteraction); + document.removeEventListener('keydown', handleUserInteraction); + }; + + document.addEventListener('click', handleUserInteraction, { once: true }); + document.addEventListener('touchstart', handleUserInteraction, { once: true }); + document.addEventListener('keydown', handleUserInteraction, { once: true }); + + return () => { + document.removeEventListener('click', handleUserInteraction); + document.removeEventListener('touchstart', handleUserInteraction); + document.removeEventListener('keydown', handleUserInteraction); + }; + }, []); + return ( <>
{ diff --git a/client/src/components/avatarconfig/ConversationGroup.jsx b/client/src/components/avatarconfig/ConversationGroup.jsx index 1812685..60bf726 100644 --- a/client/src/components/avatarconfig/ConversationGroup.jsx +++ b/client/src/components/avatarconfig/ConversationGroup.jsx @@ -601,13 +601,13 @@ const ConversationGroup = ({ {/* Avatar container - Add speaking highlight here */}
{ + e.stopPropagation(); + // Open the avatar inspector when clicked + if (element.avatarData) { + setSelectedItem({ + avatarConfig: element.avatarData, + type: "avatar", + id: element.id + }); + } + }} + title="Click to edit avatar properties" /> {loadingBoxIds.has(element.id) && (
diff --git a/client/src/components/avatarconfig/PartcipantModal.jsx b/client/src/components/avatarconfig/PartcipantModal.jsx deleted file mode 100644 index aaec8c1..0000000 --- a/client/src/components/avatarconfig/PartcipantModal.jsx +++ /dev/null @@ -1,49 +0,0 @@ -import React from 'react'; - -const ParticipantModal = ({ - participantCount, - setParticipantCount, - onCancel, - onSubmit, - isEditing = false -}) => { - return ( -
-
-

- {isEditing ? 'Modify Conversation Group' : 'Create Conversation Group'} -

-
- - setParticipantCount(Math.min(6, Math.max(2, parseInt(e.target.value))))} - className="w-full px-3 py-2 border border-gray-300 rounded-md" - /> -

Min: 2, Max: 6 participants

-
-
- - -
-
-
- ); -}; - -export default ParticipantModal; \ No newline at end of file diff --git a/client/src/components/avatarconfig/SceneSetup.jsx b/client/src/components/avatarconfig/SceneSetup.jsx index d003cb4..928a545 100644 --- a/client/src/components/avatarconfig/SceneSetup.jsx +++ b/client/src/components/avatarconfig/SceneSetup.jsx @@ -5,7 +5,7 @@ import React, { forwardRef, useImperativeHandle, } from "react"; -import { Plus, X, Trash } from "lucide-react"; +import { Plus, X, Trash, ZoomIn, ZoomOut, RotateCcw } from "lucide-react"; import VerticalToolbar from "./VerticalToolbar"; import ConversationGroup from "./ConversationGroup"; import useEditorStore from "../inspector/store"; @@ -139,6 +139,11 @@ const SceneSetup = forwardRef( const [show360View, setShow360View] = useState(false); const [availableParties, setAvailableParties] = useState([]); + + // Zoom state for scene canvas + const [zoom, setZoom] = useState(1); + const sceneContentRef = useRef(null); + const sceneContainerRef = useRef(null); // Add effect to load and sync availableParties with savedParties useEffect(() => { @@ -316,12 +321,45 @@ const SceneSetup = forwardRef( }, [activeSceneIndex, scenes, setScenes, selectedBoxId]); // Add dependencies to handle scene changes // Various refs - const sceneContainerRef = useRef(null); const fileInputRef = useRef(null); // const avatarInstancesRef = useRef({}); const previousActiveSceneRef = useRef(null); const initializedScenesRef = useRef(new Set()); + // Zoom handlers for scene canvas + const handleZoom = (direction) => { + setZoom(prevZoom => { + const newZoom = direction === 'in' + ? Math.min(prevZoom + 0.2, 3) + : Math.max(prevZoom - 0.2, 0.5); + return newZoom; + }); + }; + + const handleMouseWheel = (e) => { + if (!e.ctrlKey && !e.metaKey) return; + + e.preventDefault(); + const delta = e.deltaY > 0 ? -0.2 : 0.2; + setZoom(prevZoom => { + const newZoom = Math.min(Math.max(prevZoom + delta, 0.5), 3); + return newZoom; + }); + }; + + const resetZoom = () => { + setZoom(1); + }; + + // Add zoom event listener + useEffect(() => { + const container = sceneContainerRef.current; + if (container) { + container.addEventListener('wheel', handleMouseWheel, { passive: false }); + return () => container.removeEventListener('wheel', handleMouseWheel); + } + }, []); + // Add handler function for emoji state changes const handleEmojiStateChange = (newEmojiStates) => { setEmojiStates(newEmojiStates); @@ -1245,7 +1283,7 @@ const SceneSetup = forwardRef( {/* Scene background */}
{currentScene && ( <> - {/* Toggle 360 view button */} - {/* {currentScene.backgroundImage && ( + {/* Zoom Controls */} +
- )} */} - - {/* Render conversation boxes */} - {currentScene.boxes.map((box) => ( - - ))} +
+ {Math.round(zoom * 100)}% +
+ + +
+ + {/* Zoomable content container */} +
+ {/* Render conversation boxes */} + {currentScene.boxes.map((box) => ( + + ))} +
{/* 360 Viewer */} diff --git a/client/src/components/avatarconfig/utils/AvatarHandler.js b/client/src/components/avatarconfig/utils/AvatarHandler.js index 85552a5..8ba6195 100644 --- a/client/src/components/avatarconfig/utils/AvatarHandler.js +++ b/client/src/components/avatarconfig/utils/AvatarHandler.js @@ -168,9 +168,11 @@ export const initializeAvatar = async (boxId, persona, avatarInstancesRef) => { const { TalkingHead } = await import("talkinghead"); console.log("Creating TalkingHead with URL:", persona.url); + const ttsSpeaker = encodeURIComponent(persona?.name || boxId || "unknown"); + const instance = new TalkingHead(container, { height: boxHeight, - ttsEndpoint: `${API_CONFIG.BASE_URL}${API_CONFIG.ENDPOINTS.TTS}`, + ttsEndpoint: `${API_CONFIG.BASE_URL}${API_CONFIG.ENDPOINTS.TTS}/${ttsSpeaker}`, ttsApikey: localStorage.getItem('TTS_API_KEY') || null, lipsyncModules: ["en"], }); diff --git a/client/src/components/events/SceneEventHandler.js b/client/src/components/events/SceneEventHandler.js index cee10b4..2301b58 100644 --- a/client/src/components/events/SceneEventHandler.js +++ b/client/src/components/events/SceneEventHandler.js @@ -111,12 +111,23 @@ export const setupSceneEventListeners = ( initializeSceneAvatars(event.detail.sceneId, scenes); } } else if (event.detail.source === 'nodeeditor' && !event.detail.context) { - // Only switch to experience mode when coming from nodeeditor without explicit context - setMode('experience'); - setShowTopicPanel(false); - setShowPreview(false); - setShowSceneView(false); - console.log('Setting experience mode based on nodeeditor source'); + // Check if we're coming from the scene library/avatar config panel + // If so, stay in authoring mode. Only switch to experience mode when playing from the node editor + if (event.detail.fromSceneLibrary) { + // Coming from DraggableScenes with a scene setup context + console.log('Scene selected from library - maintaining authoring mode'); + setMode('authoring'); + setShowAvatarPanel(true); + setEditAvatar(true); + setShowSceneView(false); + } else { + // Coming from node editor to play - switch to experience mode + setMode('experience'); + setShowTopicPanel(false); + setShowPreview(false); + setShowSceneView(false); + console.log('Setting experience mode based on nodeeditor source'); + } } else if (event.detail.source === 'experience-modal') { // Always switch to experience mode for experience-modal source setMode('experience'); diff --git a/client/src/components/experience/ExperienceMode.jsx b/client/src/components/experience/ExperienceMode.jsx index 4ea18dc..836337f 100644 --- a/client/src/components/experience/ExperienceMode.jsx +++ b/client/src/components/experience/ExperienceMode.jsx @@ -222,10 +222,12 @@ const ExperienceMode = () => { console.log("TalkingHead library imported successfully"); console.log("Creating TalkingHead with URL:", avatarConfig.url || "/assets/avatar1.glb"); + + const ttsSpeaker = encodeURIComponent(avatarConfig?.name || containerId || "unknown"); const avatar = new TalkingHead(containerElement, { height: boxHeight, - ttsEndpoint: `${API_CONFIG.BASE_URL}${API_CONFIG.ENDPOINTS.TTS}`, + ttsEndpoint: `${API_CONFIG.BASE_URL}${API_CONFIG.ENDPOINTS.TTS}/${ttsSpeaker}`, ttsApikey: localStorage.getItem('TTS_API_KEY') || null, lipsyncModules: ["en"], }); @@ -466,7 +468,7 @@ const ExperienceMode = () => { // Fallback to speakText if playAudio is not available else if (typeof avatar.speakText === 'function') { console.log("Using speakText fallback with:", audioEvent.text); - avatar.speakText(audioEvent.text || ""); + avatar.speakText((audioEvent.text || "").replace(/\s+/g, ' ').replace(/\.\s+(?=[A-Z])/g, ', ').replace(/[;:](?=\s)/g, ',').trim()); } // Last resort fallback else if (typeof avatar.speak === 'function') { @@ -481,7 +483,7 @@ const ExperienceMode = () => { // If no audio URL but we have text, use speakText else if (audioEvent.text && typeof avatar.speakText === 'function') { console.log("Speaking text only:", audioEvent.text); - avatar.speakText(audioEvent.text); + avatar.speakText(audioEvent.text.replace(/\s+/g, ' ').replace(/\.\s+(?=[A-Z])/g, ', ').replace(/[;:](?=\s)/g, ',').trim()); } // Add gesture animation if applicable diff --git a/client/src/components/inspector/AvatarInspector.jsx b/client/src/components/inspector/AvatarInspector.jsx index 95ceede..9a2c9f3 100644 --- a/client/src/components/inspector/AvatarInspector.jsx +++ b/client/src/components/inspector/AvatarInspector.jsx @@ -2,18 +2,81 @@ import React, { useState, useEffect, useCallback } from 'react'; import useEditorStore from './store'; import { avatarTemplates } from '../topic/avatarTemplates'; +// Gemini TTS voices +const VOICE_CATALOG = { + 'en-US-female-aria': { + label: 'en-US-AriaNeural (F)', + region: 'US', + gender: 'female', + styles: ['cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious', 'shouting', 'unfriendly', 'whispering'] + }, + 'en-US-female-eve': { + label: 'en-US-EveNeural (F)', + region: 'US', + gender: 'female', + styles: ['cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious'] + }, + 'Kore': { + label: 'Kore (Gemini Female)', + region: 'US', + gender: 'female', + styles: ['cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious'] + }, + 'en-US-female-jenny': { + label: 'en-US-JennyNeural (F)', + region: 'US', + gender: 'female', + styles: ['cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious'] + }, + 'en-US-male-brian': { + label: 'en-US-BrianNeural (M)', + region: 'US', + gender: 'male', + styles: ['cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious', 'shouting', 'unfriendly', 'whispering'] + }, + 'en-US-male-christopher': { + label: 'en-US-ChristopherNeural (M)', + region: 'US', + gender: 'male', + styles: ['cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious'] + }, + 'en-US-male-guy': { + label: 'en-US-GuyNeural (M)', + region: 'US', + gender: 'male', + styles: ['cheerful', 'sad', 'calm', 'fearful', 'disgruntled'] + }, + 'en-GB-female-sonia': { + label: 'en-GB-SoniaNeural (F)', + region: 'GB', + gender: 'female', + styles: ['cheerful', 'sad', 'calm', 'fearful', 'disgruntled'] + }, + 'en-GB-male-ryan': { + label: 'en-GB-RyanNeural (M)', + region: 'GB', + gender: 'male', + styles: ['cheerful', 'sad', 'calm', 'fearful', 'disgruntled', 'serious', 'unfriendly', 'whispering'] + }, + 'en-GB-male-thomas': { + label: 'en-GB-ThomasNeural (M)', + region: 'GB', + gender: 'male', + styles: ['cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious'] + } +}; + +// Voice groups by gender for filter const voiceMapByGender = { - male: [ - { value: 'en-US-Standard-B', label: 'American Voice (M)' }, - { value: 'en-GB-Standard-B', label: 'British Voice (M)' }, - { value: 'en-GB-Standard-D', label: 'British Voice (M2)' } - ], - female: [ - { value: 'en-US-Standard-C', label: 'American Voice (F)' }, - { value: 'en-GB-Standard-A', label: 'British Voice (F)' } - ] + male: Object.keys(VOICE_CATALOG).filter(key => VOICE_CATALOG[key].gender === 'male'), + female: Object.keys(VOICE_CATALOG).filter(key => VOICE_CATALOG[key].gender === 'female') }; +// Voice styles +const VOICE_STYLES = [ + 'neutral', 'cheerful', 'sad', 'angry', 'calm', 'fearful', 'disgruntled', 'serious', 'shouting', 'unfriendly', 'whispering' +]; + // Predefined options for personality and interaction pattern const personalityOptions = [ "friendly", @@ -45,6 +108,25 @@ const fillerWordsFrequencyOptions = [ function AvatarInspector({ avatar }) { const { selectedItem, updateSelectedItem } = useEditorStore(); + const normalizedAvatarConfig = { + ...(avatar?.avatarConfig || {}), + settings: { + ...(avatar?.avatarConfig?.settings || {}) + } + }; + + const formatAttributeValue = (value) => { + if (value === null || value === undefined) return ''; + if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + try { + return JSON.stringify(value); + } catch (error) { + return String(value); + } + }; + // Add state for collapsible sections const [activeTab, setActiveTab] = useState('basic'); @@ -53,9 +135,13 @@ function AvatarInspector({ avatar }) { const [attributeKey, setAttributeKey] = useState(''); const [attributeValue, setAttributeValue] = useState(''); + // Add state for voice style and intensity + const [voiceStyle, setVoiceStyle] = useState('neutral'); + const [styleDegree, setStyleDegree] = useState(1.0); + const getDefaultVoiceByGender = (gender) => { const genderVoices = voiceMapByGender[gender] || voiceMapByGender.male; - return genderVoices[0].value; // Return first voice in the list for the gender + return genderVoices[0]; // Return first voice key from the list for the gender }; // Guard against missing avatar data @@ -70,19 +156,19 @@ function AvatarInspector({ avatar }) { ); } - const avatarGender = avatar.avatarConfig.gender || 'male'; + const avatarGender = normalizedAvatarConfig.gender || 'male'; // Modify the useEffect hook that loads configurations useEffect(() => { if (avatar && avatar.avatarConfig) { // Create a unique key for this avatar based on name or id - const storageKey = `avatar-config-${avatar.avatarConfig.name || selectedItem.id}`; + const storageKey = `avatar-config-${normalizedAvatarConfig.name || selectedItem?.id || 'unknown'}`; try { const savedConfig = localStorage.getItem(storageKey); // Check if we have a template for this avatar - const template = avatarTemplates[avatar.avatarConfig.name]; + const template = avatarTemplates[normalizedAvatarConfig.name]; if (savedConfig) { // If there's a saved config, use it @@ -107,7 +193,7 @@ function AvatarInspector({ avatar }) { updateSelectedItem({ ...selectedItem, avatarConfig: { - ...avatar.avatarConfig, + ...normalizedAvatarConfig, ...mergedConfig } }); @@ -115,12 +201,16 @@ function AvatarInspector({ avatar }) { // Set custom attributes setCustomAttributes(mergedConfig.customAttributes || {}); + // Set voice style and degree from settings + setVoiceStyle(mergedConfig.settings?.voiceStyle || 'neutral'); + setStyleDegree(mergedConfig.settings?.styleDegree || 1.0); + } else if (template) { // If no saved config but template exists, use template updateSelectedItem({ ...selectedItem, avatarConfig: { - ...avatar.avatarConfig, + ...normalizedAvatarConfig, ...template } }); @@ -128,6 +218,10 @@ function AvatarInspector({ avatar }) { // Set custom attributes from template setCustomAttributes(template.customAttributes || {}); + // Set voice style from template or default + setVoiceStyle(template.settings?.voiceStyle || 'neutral'); + setStyleDegree(template.settings?.styleDegree || 1.0); + // Save template to localStorage localStorage.setItem(storageKey, JSON.stringify(template)); @@ -137,7 +231,7 @@ function AvatarInspector({ avatar }) { updateSelectedItem({ ...selectedItem, avatarConfig: { - ...avatar.avatarConfig, + ...normalizedAvatarConfig, voice: defaultVoice, personality: "friendly", interactionPattern: "neutral", @@ -145,13 +239,21 @@ function AvatarInspector({ avatar }) { proactiveThreshold: 0.3, fillerWordsFrequency: "none", roleDescription: "", - customAttributes: {} + customAttributes: {}, + settings: { + voiceStyle: 'neutral', + styleDegree: 1.0 + } } }); + + // Set default voice style state + setVoiceStyle('neutral'); + setStyleDegree(1.0); } // Apply settings to avatar instance - applySettingsToAvatar(selectedItem.avatarConfig); + applySettingsToAvatar(selectedItem?.avatarConfig || normalizedAvatarConfig); } catch (error) { console.error('Error loading avatar configuration:', error); @@ -261,14 +363,14 @@ function AvatarInspector({ avatar }) { const saveAvatarConfig = (config) => { try { // Create a unique key for this avatar - const storageKey = `avatar-config-${avatar.avatarConfig.name || selectedItem.id}`; + const storageKey = `avatar-config-${normalizedAvatarConfig.name || selectedItem?.id || 'unknown'}`; // Determine if we're working with a full config object or just an avatarConfig const avatarConfigToSave = config.avatarConfig || config; // Include all settings in the saved config using a consistent structure const configToSave = { - name: avatar.avatarConfig.name || selectedItem.id, + name: normalizedAvatarConfig.name || selectedItem?.id || 'unknown', // Save all settings directly at the root level for simplicity and consistency personality: avatarConfigToSave.personality || selectedItem.avatarConfig?.personality || "friendly", interactionPattern: avatarConfigToSave.interactionPattern || selectedItem.avatarConfig?.interactionPattern || "neutral", @@ -298,14 +400,14 @@ function AvatarInspector({ avatar }) { // Find the participant with matching name and update their settings const participantIndex = participants.findIndex(p => - p.name === avatar.avatarConfig.name || p.id === selectedItem.id + p.name === normalizedAvatarConfig.name || p.id === selectedItem?.id ); // Create a new participant entry with proper structure if not found const participantToAdd = { id: selectedItem.id || `generated-${Date.now()}`, - name: avatar.avatarConfig.name || selectedItem.id, - gender: avatar.avatarConfig.gender || selectedItem.avatarConfig?.gender || "neutral", + name: normalizedAvatarConfig.name || selectedItem?.id || 'unknown', + gender: normalizedAvatarConfig.gender || selectedItem?.avatarConfig?.gender || "neutral", voice: configToSave.voice || "", personality: configToSave.personality, interactionPattern: configToSave.interactionPattern, @@ -412,18 +514,96 @@ function AvatarInspector({ avatar }) { }, 500); }; + const handleVoiceStyleChange = (e) => { + const newStyle = e.target.value; + setVoiceStyle(newStyle); + + // Update settings with voice style + const updatedConfig = { + ...avatar.avatarConfig, + settings: { + ...(avatar.avatarConfig?.settings || {}), + voiceStyle: newStyle, + styleDegree: styleDegree + } + }; + + updateSelectedItem({ + ...selectedItem, + avatarConfig: updatedConfig + }); + + saveAvatarConfig(updatedConfig); + + // Dispatch event for style change + const customEvent = new CustomEvent('avatarConfigChanged', { + detail: { + id: selectedItem.id, + config: updatedConfig + } + }); + document.dispatchEvent(customEvent); + }; + + const handleStyleDegreeChange = (e) => { + const newDegree = parseFloat(e.target.value); + setStyleDegree(newDegree); + + // Update settings with style degree + const updatedConfig = { + ...avatar.avatarConfig, + settings: { + ...(avatar.avatarConfig?.settings || {}), + voiceStyle: voiceStyle, + styleDegree: newDegree + } + }; + + updateSelectedItem({ + ...selectedItem, + avatarConfig: updatedConfig + }); + + saveAvatarConfig(updatedConfig); + + // Dispatch event for style change + const customEvent = new CustomEvent('avatarConfigChanged', { + detail: { + id: selectedItem.id, + config: updatedConfig + } + }); + document.dispatchEvent(customEvent); + }; + // Function to get appropriate test text based on voice type const getVoiceTestText = (voice) => { // Shorter test phrases to avoid TTS rate limiting - if (voice.includes('en-GB')) { + if (voice.includes('en-GB') || voice.includes('GB')) { return "Hello, this is my British voice."; - } else if (voice.includes('en-US')) { + } else if (voice.includes('en-US') || voice.includes('US')) { return "Hello, this is my American voice."; } else { return "Hello, this is my new voice."; } }; + // Function to test voice + const handleTestVoice = (voice) => { + const avatarInstance = window.avatarInstancesRef?.current?.[selectedItem.id]; + if (avatarInstance && typeof avatarInstance.say === 'function') { + try { + const testText = getVoiceTestText(voice); + console.log(`Testing voice ${voice} with text: "${testText}"`); + avatarInstance.say(testText); + } catch (error) { + console.error('Error testing voice:', error); + } + } else { + console.warn('Avatar instance not available or does not have say method'); + } + }; + const handleMoodChange = (e) => { const newMood = e.target.value; @@ -880,7 +1060,7 @@ function AvatarInspector({ avatar }) {
  • {key}: - {value} + {formatAttributeValue(value)}
  • -
    + {/* Voice Style selection */} +
    + + +
    + + {/* Style Degree/Intensity Slider */} +
    + + + 0.5 (subtle) to 2.0 (very expressive) +
    +
    + {/* Mood Settings */}

    Emotional State

    diff --git a/client/src/components/inspector/DraggableScenes.tsx b/client/src/components/inspector/DraggableScenes.tsx index e768d7d..c30fbc7 100644 --- a/client/src/components/inspector/DraggableScenes.tsx +++ b/client/src/components/inspector/DraggableScenes.tsx @@ -119,6 +119,20 @@ const DraggableScenes: React.FC = () => { // Dispatch the event on the document so other components can listen for it document.dispatchEvent(event); + + // Also dispatch a show-scene-in-editor event with context to preserve authoring mode + setTimeout(() => { + window.dispatchEvent(new CustomEvent('show-scene-in-editor', { + detail: { + sceneId: scene.id, + sceneName: scene.name, + source: 'draggable-scenes', + context: 'authoring', + fromSceneLibrary: true, + maintainMode: false + } + })); + }, 100); }; const handleDoubleClick = (scene: Scene, e: React.MouseEvent) => { diff --git a/client/src/components/inspector/SceneSelector.tsx b/client/src/components/inspector/SceneSelector.tsx index c0a7679..ae9b92a 100644 --- a/client/src/components/inspector/SceneSelector.tsx +++ b/client/src/components/inspector/SceneSelector.tsx @@ -35,6 +35,22 @@ interface PartyInfo { turnMode: string | null; } +const DEFAULT_SCENE_DESCRIPTION = + "Single-scene FNOL intake where Alice asks structured insurance questions and Bob reports the accident details. Capture policy and vehicle identifiers, reporter identity/contact, accident circumstances, driver scope, other-party/police/witness information, injuries, property damage, liability indicators, exclusions and fraud signals, then close with missing documents and clear next steps."; + +const normalizeSceneDescription = (text: string, sceneName?: string): string => { + const source = String(text || "").trim(); + if (!source) return source; + + if (sceneName) { + const escaped = sceneName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const scenePrefix = new RegExp(`^Conversation context for\\s+${escaped}\\.\\s*`, "i"); + return source.replace(scenePrefix, "").trim(); + } + + return source.replace(/^Conversation context for\s+[^.]+\.\s*/i, "").trim(); +}; + const SceneSelector: React.FC = ({ snippetNode, onSceneChange, onDescriptionGenerated }) => { const { scenes, activeSceneId, setActiveSceneId, updateNode, updateSelectedItem, updateSnippetNode } = useEditorStore() as EditorState; const [description, setDescription] = useState(""); @@ -45,10 +61,20 @@ const SceneSelector: React.FC = ({ snippetNode, onSceneChang // When the scene changes, update the description useEffect(() => { if (snippetNode.description) { - setDescription(snippetNode.description); + const normalized = normalizeSceneDescription( + snippetNode.description, + snippetNode.attachedScene?.name + ); + setDescription(normalized); + if (snippetNode.id && normalized !== snippetNode.description) { + updateNode(snippetNode.id, { description: normalized }); + } } else if (snippetNode.attachedScene) { - // Generate a description if none exists - generateDescription(); + // Seed a predefined default description for newly attached scenes. + setDescription(DEFAULT_SCENE_DESCRIPTION); + if (snippetNode.id) { + updateNode(snippetNode.id, { description: DEFAULT_SCENE_DESCRIPTION }); + } } else { setDescription(""); } @@ -149,12 +175,12 @@ const SceneSelector: React.FC = ({ snippetNode, onSceneChang } catch (error) { console.error('Error fetching scene description:', error); - setDescription(`A conversation about ${snippetNode.attachedScene.name}`); + setDescription(DEFAULT_SCENE_DESCRIPTION); // Set a fallback description if (snippetNode.id) { updateNode(snippetNode.id, { - description: `A conversation about ${snippetNode.attachedScene.name}` + description: DEFAULT_SCENE_DESCRIPTION }); // Call the callback even with fallback description diff --git a/client/src/components/inspector/SnippetInspector.tsx b/client/src/components/inspector/SnippetInspector.tsx index e23b03d..b42231b 100644 --- a/client/src/components/inspector/SnippetInspector.tsx +++ b/client/src/components/inspector/SnippetInspector.tsx @@ -12,7 +12,7 @@ import React from "react" import ConversationPrompt, { TooltipIcon } from "./ConversationPrompt" const SnippetInspector: React.FC<{ currentTopic: string }> = ({ currentTopic }) => { - const { selectedItem, updateSelectedItem, getCachedDefaultSpeakers, speakers, nodes, closeInspector, updateSnippetNode } = useEditorStore() as EditorState; + const { selectedItem, updateSelectedItem, getCachedDefaultSpeakers, speakers, nodes, closeInspector, updateSnippetNode, setSelectedItem } = useEditorStore() as EditorState; const [localNode, setLocalNode] = useState(undefined); const [isAvatarSectionExpanded, setIsAvatarSectionExpanded] = useState(false); const [isEditingTitle, setIsEditingTitle] = useState(false); @@ -600,6 +600,23 @@ const SnippetInspector: React.FC<{ currentTopic: string }> = ({ currentTopic }) )} + + {/* Edit Avatar Button */} + {speaker.party && (
    @@ -706,6 +723,19 @@ const SnippetInspector: React.FC<{ currentTopic: string }> = ({ currentTopic }) />
    +
    + +