✨ feat: add model playing

scripts/mixamo/Motion/Female/Greeting/input.json

@@ -0,0 +1,28 @@
+[
+  {
+    "id": "c9c7996a-b96c-11e4-a802-0aaa78deedf9",
+    "type": "Motion",
+    "description": "Greeting While Standing",
+    "category": "",
+    "character_type": "human",
+    "name": "Standing Greeting",
+    "thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/113350901/static.png",
+    "thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/113350901/animated.gif",
+    "motion_id": "c9c7996a-b96c-11e4-a802-0aaa78deedf9",
+    "motions": null,
+    "source": "system"
+  },
+  {
+    "id": "c9c9f343-b96c-11e4-a802-0aaa78deedf9",
+    "type": "Motion",
+    "description": "2 People Shaking Hands Part 1 -  Female",
+    "category": "",
+    "character_type": "human",
+    "name": "Shaking Hands 1",
+    "thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070901/static.png",
+    "thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070901/animated.gif",
+    "motion_id": "c9c9f343-b96c-11e4-a802-0aaa78deedf9",
+    "motions": null,
+    "source": "system"
+  }
+]

scripts/mixamo/Motion/Male/Greeting/input.json

@@ -0,0 +1,54 @@
+[
+  {
+    "id": "c9c9f403-b96c-11e4-a802-0aaa78deedf9",
+    "type": "Motion",
+    "description": "2 People Shaking Hands Part 2 -  Male",
+    "category": "",
+    "character_type": "human",
+    "name": "Shaking Hands 2",
+    "thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070902/static.png",
+    "thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070902/animated.gif",
+    "motion_id": "c9c9f403-b96c-11e4-a802-0aaa78deedf9",
+    "motions": null,
+    "source": "system"
+  },
+  {
+    "id": "c9c9f4bc-b96c-11e4-a802-0aaa78deedf9",
+    "type": "Motion",
+    "description": "Quick Formal Bow",
+    "category": "",
+    "character_type": "human",
+    "name": "Quick Formal Bow",
+    "thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110901/static.png",
+    "thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110901/animated.gif",
+    "motion_id": "c9c9f4bc-b96c-11e4-a802-0aaa78deedf9",
+    "motions": null,
+    "source": "system"
+  },
+  {
+    "id": "c9cb0ab6-b96c-11e4-a802-0aaa78deedf9",
+    "type": "Motion",
+    "description": "Formal Military Salute",
+    "category": "",
+    "character_type": "human",
+    "name": "Salute",
+    "thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/120580901/static.png",
+    "thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/120580901/animated.gif",
+    "motion_id": "c9cb0ab6-b96c-11e4-a802-0aaa78deedf9",
+    "motions": null,
+    "source": "system"
+  },
+  {
+    "id": "c9c9f57a-b96c-11e4-a802-0aaa78deedf9",
+    "type": "Motion",
+    "description": "Quick Informal Bow",
+    "category": "",
+    "character_type": "human",
+    "name": "Quick Informal Bow",
+    "thumbnail": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110902/static.png",
+    "thumbnail_animated": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110902/animated.gif",
+    "motion_id": "c9c9f57a-b96c-11e4-a802-0aaa78deedf9",
+    "motions": null,
+    "source": "system"
+  }
+]

scripts/mixamo/index.ts

@@ -1,6 +1,6 @@
 /**
  * https://www.mixamo.com/
- * Mixamo Animations 脚本处理
+ * Mixamo Animations 脚本处理, `bun index.ts`
  * @author rdmclin2
  */
 import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';

src/animations/Motion/index.json

@@ -529,6 +529,46 @@
     "url": "https://r2.vidol.chat/animations/c9c7438e-b96c-11e4-a802-0aaa78deedf9.fbx",
     "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/108840901/animated.gif"
   },
+  {
+    "id": "c9c9f403-b96c-11e4-a802-0aaa78deedf9",
+    "name": "Shaking Hands 2",
+    "type": "Motion",
+    "gender": "Male",
+    "category": "Greeting",
+    "description": "2 People Shaking Hands Part 2 -  Male",
+    "url": "https://r2.vidol.chat/animations/c9c9f403-b96c-11e4-a802-0aaa78deedf9.fbx",
+    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070902/animated.gif"
+  },
+  {
+    "id": "c9c9f4bc-b96c-11e4-a802-0aaa78deedf9",
+    "name": "Quick Formal Bow",
+    "type": "Motion",
+    "gender": "Male",
+    "category": "Greeting",
+    "description": "Quick Formal Bow",
+    "url": "https://r2.vidol.chat/animations/c9c9f4bc-b96c-11e4-a802-0aaa78deedf9.fbx",
+    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110901/animated.gif"
+  },
+  {
+    "id": "c9cb0ab6-b96c-11e4-a802-0aaa78deedf9",
+    "name": "Salute",
+    "type": "Motion",
+    "gender": "Male",
+    "category": "Greeting",
+    "description": "Formal Military Salute",
+    "url": "https://r2.vidol.chat/animations/c9cb0ab6-b96c-11e4-a802-0aaa78deedf9.fbx",
+    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/120580901/animated.gif"
+  },
+  {
+    "id": "c9c9f57a-b96c-11e4-a802-0aaa78deedf9",
+    "name": "Quick Informal Bow",
+    "type": "Motion",
+    "gender": "Male",
+    "category": "Greeting",
+    "description": "Quick Informal Bow",
+    "url": "https://r2.vidol.chat/animations/c9c9f57a-b96c-11e4-a802-0aaa78deedf9.fbx",
+    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119110902/animated.gif"
+  },
   {
     "id": "c9c609a9-b96c-11e4-a802-0aaa78deedf9",
     "name": "Walking",
@@ -1019,6 +1059,26 @@
     "url": "https://r2.vidol.chat/animations/c9c6e172-b96c-11e4-a802-0aaa78deedf9.fbx",
     "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/108660901/animated.gif"
   },
+  {
+    "id": "c9c7996a-b96c-11e4-a802-0aaa78deedf9",
+    "name": "Standing Greeting",
+    "type": "Motion",
+    "gender": "Female",
+    "category": "Greeting",
+    "description": "Greeting While Standing",
+    "url": "https://r2.vidol.chat/animations/c9c7996a-b96c-11e4-a802-0aaa78deedf9.fbx",
+    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/113350901/animated.gif"
+  },
+  {
+    "id": "c9c9f343-b96c-11e4-a802-0aaa78deedf9",
+    "name": "Shaking Hands 1",
+    "type": "Motion",
+    "gender": "Female",
+    "category": "Greeting",
+    "description": "2 People Shaking Hands Part 1 -  Female",
+    "url": "https://r2.vidol.chat/animations/c9c9f343-b96c-11e4-a802-0aaa78deedf9.fbx",
+    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/119070901/animated.gif"
+  },
   {
     "id": "c9c98a38-b96c-11e4-a802-0aaa78deedf9",
     "name": "Happy",

src/animations/Posture/index.json

@@ -639,6 +639,16 @@
     "url": "https://r2.vidol.chat/animations/27fe7228-487a-4951-af65-30cef1a9183d.fbx",
     "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140000905/animated.gif"
   },
+  {
+    "id": "8c95cac8-2068-410f-9564-d0a2a04ea0a5",
+    "name": "Female Action Pose",
+    "type": "Posture",
+    "gender": "Female",
+    "category": "Action",
+    "description": "Sliding",
+    "url": "https://r2.vidol.chat/animations/8c95cac8-2068-410f-9564-d0a2a04ea0a5.fbx",
+    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140000902/animated.gif"
+  },
   {
     "id": "eaf51ff4-0fb7-4759-aadf-edb8fd3ae6a0",
     "name": "Female Action Pose",
@@ -719,16 +729,6 @@
     "url": "https://r2.vidol.chat/animations/e3543986-d43d-4ec1-bac8-5cf7f5ec003b.fbx",
     "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140100902/animated.gif"
   },
-  {
-    "id": "8c95cac8-2068-410f-9564-d0a2a04ea0a5",
-    "name": "Female Action Pose",
-    "type": "Posture",
-    "gender": "Female",
-    "category": "Crouch",
-    "description": "Sliding",
-    "url": "https://r2.vidol.chat/animations/8c95cac8-2068-410f-9564-d0a2a04ea0a5.fbx",
-    "avatar": "https://d99n9xvb9513w.cloudfront.net/thumbnails/motions/140000902/animated.gif"
-  },
   {
     "id": "31dca3e7-dba8-4536-a70e-34d7d5976fa9",
     "name": "Female Dance Pose",

src/app/role/RoleEdit/Touch/ActionList/Actions/Play.tsx

@@ -54,7 +54,6 @@ export default memo((props: Props) => {
               ...currentAgentTTS,
               message: touchAction.text,
             },
-            motion: touchAction.motion,
           },
           viewer,
           () => {},

src/constants/touch.tsx

@@ -12,6 +12,7 @@ import {
 export const HAPPY_MOTION_ID = 'c9c98a38-b96c-11e4-a802-0aaa78deedf9';
 export const ANGRY_MOTION_ID = 'c9c98b02-b96c-11e4-a802-0aaa78deedf9';
 export const RELAX_MOTION_ID = 'c9c98361-b96c-11e4-a802-0aaa78deedf9';
+export const GREETING_MOTION_ID = 'c9c7996a-b96c-11e4-a802-0aaa78deedf9';
 
 export const DEFAULT_TOUCH_ACTION_CONFIG_FEMALE: TouchActionConfig = {
   [TouchAreaEnum.Head]: [

src/features/AgentViewer/index.tsx

@@ -1,12 +1,16 @@
+import { VRMExpressionPresetName } from '@pixiv/three-vrm';
 import { Progress } from 'antd';
 import classNames from 'classnames';
 import React, { memo, useCallback, useRef } from 'react';
 import { useTranslation } from 'react-i18next';
 
 import PageLoading from '@/components/PageLoading';
+import { DEFAULT_MOTION_ANIMATION, GREETING_MOTION_ID } from '@/constants/touch';
 import { useLoadModel } from '@/hooks/useLoadModel';
+import { speakCharacter } from '@/libs/messages/speakCharacter';
 import { useGlobalStore } from '@/store/global';
 import { Agent } from '@/types/agent';
+import { fetchWithProgress } from '@/utils/fetch';
 
 import ToolBar from './ToolBar';
 import { useStyles } from './style';
@@ -43,14 +47,38 @@ function AgentViewer(props: Props) {
             loadingScreen.append(loader);
             agentViewer.append(loadingScreen);
 
-            // load
+            // load vrm
             await viewer.loadVrm(modelUrl);
 
-            // remove loading dom
-            loadingScreen.classList.add('fade-out');
-            loadingScreen.addEventListener('transitionend', () => {
-              loadingScreen.remove();
-            });
+            // load motion
+            let motionUrl = undefined;
+            const item = DEFAULT_MOTION_ANIMATION.find((item) => item.id === GREETING_MOTION_ID);
+            if (item) {
+              const blob = await fetchWithProgress(item.url);
+              motionUrl = window.URL.createObjectURL(blob);
+            }
+
+            speakCharacter(
+              {
+                emotion: VRMExpressionPresetName.Neutral,
+                tts: {
+                  ...agent.tts,
+                  message: agent.greeting,
+                },
+                motion: motionUrl,
+              },
+              viewer,
+              () => {
+                // remove loading dom
+                loadingScreen.classList.add('fade-out');
+                loadingScreen.addEventListener('transitionend', () => {
+                  loadingScreen.remove();
+                });
+              },
+              () => {
+                viewer.model?.loadIdleAnimation();
+              },
+            );
           }
         });
 

src/features/AgentViewer/style.ts

@@ -25,6 +25,10 @@ export const useStyles = createStyles(({ css, token }) => ({
       opacity: 0;
     }
 
+    #loading-screen.fade-in {
+      opacity: 0;
+    }
+
     #loader {
       position: relative;
       top: 50%;
@@ -39,8 +43,6 @@ export const useStyles = createStyles(({ css, token }) => ({
       border: 3px solid transparent;
       border-top-color: #9370db;
       border-radius: 50%;
-
-      animation: spin 2s linear infinite;
       animation: spin 2s linear infinite;
     }
 
@@ -53,8 +55,6 @@ export const useStyles = createStyles(({ css, token }) => ({
       border: 3px solid transparent;
       border-top-color: #ba55d3;
       border-radius: 50%;
-
-      animation: spin 3s linear infinite;
       animation: spin 3s linear infinite;
     }
 
@@ -67,36 +67,26 @@ export const useStyles = createStyles(({ css, token }) => ({
       border: 3px solid transparent;
       border-top-color: #f0f;
       border-radius: 50%;
-
-      animation: spin 1.5s linear infinite;
       animation: spin 1.5s linear infinite;
     }
 
     @keyframes spin {
       0% {
         transform: rotate(0deg);
-        transform: rotate(0deg);
-        transform: rotate(0deg);
       }
 
       100% {
         transform: rotate(360deg);
-        transform: rotate(360deg);
-        transform: rotate(360deg);
       }
     }
 
     @keyframes spin {
       0% {
         transform: rotate(0deg);
-        transform: rotate(0deg);
-        transform: rotate(0deg);
       }
 
       100% {
         transform: rotate(360deg);
-        transform: rotate(360deg);
-        transform: rotate(360deg);
       }
     }
   `,

src/libs/vrmViewer/model.ts

@@ -116,7 +116,7 @@ export class Model {
     if (vrma) await this.loadAnimation(vrma);
   }
 
-  public async loadFBX(animationUrl: string) {
+  public async loadFBX(animationUrl: string, loop: boolean = true) {
     const { vrm, mixer } = this;
 
     if (vrm && mixer) {
@@ -125,6 +125,8 @@ export class Model {
       const clip = await loadMixamoAnimation(animationUrl, vrm);
       // Apply the loaded animation to mixer and play
       const action = mixer.clipAction(clip);
+      if (!loop) action.setLoop(LoopOnce, 1);
+
       action.play();
       this._action = action;
       this._clip = clip;
@@ -157,6 +159,7 @@ export class Model {
    */
   public async speak(buffer: ArrayBuffer, screenplay: Screenplay) {
     this.emoteController?.playEmotion(screenplay.emotion);
+    if (screenplay.motion) this.loadFBX(screenplay.motion);
     await new Promise((resolve) => {
       this._lipSync?.playFromArrayBuffer(buffer, () => {
         resolve(true);