Initial commit: 实时医疗对话系统

基于模块化 Handler 架构的多模态数字人交互平台，
支持 ASR、LLM、TTS、Avatar 全流程实时对话。

.dockerignore

+# Version control
+.git
+.gitignore
+.gitattributes
+
+# Documentation
+README.md
+readme_en.md
+docs/
+*.md
+
+# Development files
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.venv/
+ENV/
+env.bak/
+venv.bak/
+.pytest_cache/
+.coverage
+.tox/
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Jupyter
+.ipynb_checkpoints
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# Logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# Runtime data
+pids
+*.pid
+*.seed
+*.pid.lock
+
+# Directory for instrumented libs generated by jscoverage/JSCover
+lib-cov
+
+# Coverage directory used by tools like istanbul
+coverage
+*.lcov
+
+# nyc test coverage
+.nyc_output
+
+# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
+.grunt
+
+# Bower dependency directory (https://bower.io/)
+bower_components
+
+# node_modules
+node_modules/
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Optional npm cache directory
+.npm
+
+# Optional eslint cache
+.eslintcache
+
+# Microbundle cache
+.rpt2_cache/
+.rts2_cache_cjs/
+.rts2_cache_es/
+.rts2_cache_umd/
+
+# Optional REPL history
+.node_repl_history
+
+# Output of 'npm pack'
+*.tgz
+
+# Yarn Integrity file
+.yarn-integrity
+
+# dotenv environment variables file
+.env
+.env.test
+.env.local
+.env.production
+
+# parcel-bundler cache (https://parceljs.org/)
+.cache
+.parcel-cache
+
+# Next.js build output
+.next
+
+# Nuxt.js build / generate output
+.nuxt
+dist
+
+# Gatsby files
+.cache/
+public
+
+# Storybook build outputs
+.out
+.storybook-out
+
+# Temporary folders
+tmp/
+temp/
+
+# Editor backup files
+*~
+*.tmp
+*.bak
+
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Docker
+.dockerignore
+Dockerfile*
+docker-compose*.yml
+
+# Kubernetes
+*.yaml
+*.yml
+k8s/
+
+# Test files
+test/
+tests/
+*_test.py
+test_*.py
+
+# Models and large files (should be mounted as volumes)
+models/
+*.pth
+*.bin
+*.safetensors
+*.ckpt
+*.pt
+
+# SSL certificates (should be mounted as volumes)
+ssl_certs/
+*.pem
+*.key
+*.crt
+
+# Build artifacts that should be mounted
+build/
+output/
+results/
+
+# GitHub
+.github/workflows/
+.github/ISSUE_TEMPLATE/
+.github/PULL_REQUEST_TEMPLATE.md
+
+# IDE specific
+.vscode/
+.idea/
+*.sublime-project
+*.sublime-workspace
+
+# Temporary files
+*.tmp
+*.temp
+*.pid
+*.lock
+
+# Backup files
+*.backup
+*.old
+*.orig
\ No newline at end of file

.gitignore

+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+venv/
+env/
+.venv/
+
+# Model files & weights (too large for git)
+*.pth
+*.pt
+*.bin
+*.onnx
+*.jit
+models/
+**/weights/
+
+# Large binary / media files
+*.zip
+*.tar.gz
+*.mp4
+*.wav
+*.mp3
+*.avi
+
+# Sample data & resources
+resource/avatar/
+src/handlers/client/h5_rendering_client/lam_samples/
+src/handlers/avatar/musetalk/MuseTalk/data/
+src/handlers/avatar/musetalk/MuseTalk/assets/
+src/handlers/avatar/musetalky/MuseTalk/data/
+src/handlers/avatar/musetalky/MuseTalk/assets/
+src/handlers/avatar/lam/LAM_Audio2Expression/assets/
+src/handlers/vad/silerovad/silero_vad/examples/
+
+# Build artifacts
+wheels/
+dist/
+build/
+
+# Environment & secrets
+.env
+.env.*
+ssl_certs/
+
+# Lock files
+.lock/
+
+# Logs & debug
+*.log
+*.pcm
+src/debug_sdp.log
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo
+
+# Claude Code
+.claude/
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Misc
+*.xlsx
+*.docx
+
+# Node modules
+node_modules/

Dockerfile

+FROM nvidia/cuda:12.2.2-cudnn8-devel-ubuntu22.04
+LABEL authors="HumanAIGC-Engineering"
+
+ARG CONFIG_FILE=config/chat_with_minicpm.yaml
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Use Tsinghua University APT mirrors
+RUN sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list && \
+    sed -i 's/security.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list
+
+# Update package list and install required dependencies
+RUN apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y python3.11 python3.11-dev python3.11-venv python3.11-distutils python3-pip git libgl1 libglib2.0-0
+
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \
+    python3.11 -m ensurepip --upgrade && \
+    python3.11 -m pip install --upgrade pip
+
+ARG WORK_DIR=/root/open-avatar-chat
+WORKDIR $WORK_DIR
+
+# Install core dependencies
+COPY ./install.py $WORK_DIR/install.py
+COPY ./pyproject.toml $WORK_DIR/pyproject.toml
+COPY ./src/third_party $WORK_DIR/src/third_party
+RUN pip install uv && \
+    uv venv --python 3.11.11 && \
+    uv sync --no-install-workspace
+
+ADD ./src $WORK_DIR/src
+
+# Copy script files (must be copied before installing config dependencies)
+ADD ./scripts $WORK_DIR/scripts
+
+# Execute pre-config installation script
+RUN echo "Using config file: ${CONFIG_FILE}"
+COPY $CONFIG_FILE /tmp/build_config.yaml
+RUN chmod +x $WORK_DIR/scripts/pre_config_install.sh && \
+    $WORK_DIR/scripts/pre_config_install.sh --config /tmp/build_config.yaml
+
+# Install config dependencies
+RUN uv run install.py \
+    --config /tmp/build_config.yaml \
+    --uv \
+    --skip-core
+
+# Execute post-config installation script
+RUN chmod +x $WORK_DIR/scripts/post_config_install.sh && \
+    $WORK_DIR/scripts/post_config_install.sh --config /tmp/build_config.yaml && \
+    rm /tmp/build_config.yaml
+
+ADD ./resource $WORK_DIR/resource
+ADD ./.env* $WORK_DIR/
+
+WORKDIR $WORK_DIR
+ENTRYPOINT ["uv", "run", "src/demo.py"]

Dockerfile.cuda12.8

+FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu22.04
+
+# Re-declare ARGs after FROM to make them available in subsequent layers
+ARG WORK_DIR=/root/open-avatar-chat
+
+# Image metadata with dynamic version
+LABEL authors="HumanAIGC-Engineering"
+
+
+# Environment variables for optimized build and runtime
+ENV DEBIAN_FRONTEND=noninteractive \
+    PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONOPTIMIZE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    UV_COMPILE_BYTECODE=0 \
+    WORK_DIR=$WORK_DIR
+
+
+# =============================================================================
+# System Dependencies Installation
+# =============================================================================
+# Use Tsinghua University mirrors for faster package downloads in China
+# Install latest Python 3.11.x and essential system libraries for the application
+RUN sed -i 's/archive.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list && \
+    sed -i 's/security.ubuntu.com/mirrors.tuna.tsinghua.edu.cn/g' /etc/apt/sources.list && \
+    apt-get update && \
+    apt-get install -y software-properties-common && \
+    add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y \
+        # Python 3.11 and development tools (latest available 3.11.x version)
+        python3.11 \
+        python3.11-dev \
+        python3.11-venv \
+        python3.11-distutils \
+        python3-pip \
+        # Version control and build tools
+        git \
+        git-lfs \
+        build-essential \
+        # Graphics and image libraries (required for OpenCV/PIL)
+        libgl1 \
+        libglib2.0-0 \
+        libsm6 \
+        libxext6 \
+        libgomp1 \
+        # Audio processing libraries (required for librosa, soundfile, torchaudio)
+        libsndfile1 \
+        ffmpeg \
+        sox \
+        libsox-dev \
+        # Multimedia codec libraries (required for video processing)
+        libavcodec-dev \
+        libavformat-dev \
+        libswscale-dev \
+        # Additional utilities
+        curl \
+        ca-certificates && \
+    # Configure git lfs
+    git lfs install && \
+    # Clean up package manager cache to reduce image size
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    # Set Python 3.11 as default python3 with higher priority
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 2 && \
+    update-alternatives --set python3 /usr/bin/python3.11 && \
+    # Upgrade pip to latest version with optimizations
+    python3.11 -m ensurepip --upgrade && \
+    python3.11 -m pip install --no-cache-dir --upgrade pip setuptools wheel
+
+# Set working directory for the application
+WORKDIR $WORK_DIR
+
+# =============================================================================
+# Python Dependencies and Virtual Environment Setup
+# =============================================================================
+# Copy dependency files first for better Docker layer caching
+COPY ./install.py $WORK_DIR/install.py
+COPY ./pyproject.toml $WORK_DIR/pyproject.toml
+# Copy all workspace member pyproject.toml files while preserving directory structure
+COPY ./src/handlers/tts/edgetts/pyproject.toml $WORK_DIR/src/handlers/tts/edgetts/pyproject.toml
+COPY ./src/handlers/tts/cosyvoice/pyproject.toml $WORK_DIR/src/handlers/tts/cosyvoice/pyproject.toml
+COPY ./src/handlers/llm/minicpm/pyproject.toml $WORK_DIR/src/handlers/llm/minicpm/pyproject.toml
+COPY ./src/handlers/avatar/liteavatar/pyproject.toml $WORK_DIR/src/handlers/avatar/liteavatar/pyproject.toml
+COPY ./src/handlers/avatar/lam/pyproject.toml $WORK_DIR/src/handlers/avatar/lam/pyproject.toml
+COPY ./src/handlers/vad/silerovad/pyproject.toml $WORK_DIR/src/handlers/vad/silerovad/pyproject.toml
+COPY ./src/handlers/tts/bailian_tts/pyproject.toml $WORK_DIR/src/handlers/tts/bailian_tts/pyproject.toml
+COPY ./src/handlers/asr/sensevoice/pyproject.toml $WORK_DIR/src/handlers/asr/sensevoice/pyproject.toml
+COPY ./src/handlers/avatar/musetalk/pyproject.toml $WORK_DIR/src/handlers/avatar/musetalk/pyproject.toml
+# Copy third_party directory
+COPY ./src/third_party $WORK_DIR/src/third_party
+
+# Install uv package manager and setup Python virtual environment using system Python 3.11.x
+RUN pip install --no-cache-dir uv && \
+    uv venv --python /usr/bin/python3.11 --seed
+
+# Activate virtual environment and install dependencies
+RUN uv pip install setuptools pip wheel && \
+    uv sync --all-packages && \
+    uv pip uninstall mmcv && \
+    uv run mim install mmcv==2.2.0 --force && \
+    # Fix mmdet version compatibility
+    MMDET_INIT_FILE="$VENV_PATH/lib/python3.11/site-packages/mmdet/__init__.py" && \
+    if [ -f "$MMDET_INIT_FILE" ]; then \
+        sed -i "s/mmcv_maximum_version = '[^']*'/mmcv_maximum_version = '2.2.1'/g" "$MMDET_INIT_FILE"; \
+    fi
+
+# =============================================================================
+# Application Source Code and Configuration
+# =============================================================================
+# Copy build scripts, source code, and configuration first
+COPY ./scripts $WORK_DIR/scripts
+COPY ./src $WORK_DIR/src
+COPY ./config $WORK_DIR/config
+
+
+# =============================================================================
+# Runtime Assets
+# =============================================================================
+# Copy runtime resources and environment files
+COPY ./resource $WORK_DIR/resource
+# COPY ./.env* $WORK_DIR/
+
+RUN uv pip uninstall onnxruntime onnxruntime-gpu && \
+    uv pip install --no-cache-dir onnxruntime-gpu==1.20.2
+# =============================================================================
+# Final Cleanup and Optimization
+# =============================================================================
+# Clean up build artifacts and temporary files to reduce image size
+RUN rm -rf /root/.cache/uv/* /tmp/* /var/tmp/*
+
+ARG BUILD_VERSION=dev
+ARG BUILD_DATE=""
+ENV APP_VERSION=${BUILD_VERSION} \
+    BUILD_DATE=${BUILD_DATE}
+
+# =============================================================================
+# Container Entry Point
+# =============================================================================
+# Use uv to run the application with proper virtual environment
+ENTRYPOINT ["uv", "run", "src/demo.py"]

README.md

+# RealTimeMedicalDialogue
+
+实时医疗对话系统 —— 基于模块化架构的多模态数字人交互平台，支持语音识别、大语言模型推理、语音合成与数字人驱动的端到端实时对话。
+
+## 系统概述
+
+本系统专为医疗随访场景设计，通过 WebRTC 实现低延迟的实时音视频通信，将 ASR、LLM、TTS、数字人等 AI 能力串联为完整的对话管线。支持多种医疗角色（主治医生、责任护士、健康管理师、临床药师、心理咨询师、营养师），可在单台 GPU 服务器上完成全流程部署。
+
+## 架构
+
+```
+客户端 (Vue 3 WebUI)
+    │  WebRTC / WebSocket
+    ▼
+FastAPI 服务层 (src/demo.py)
+    │
+    ▼
+ChatEngine 编排引擎
+    │
+    ├── VAD ──── 语音活动检测 (Silero VAD)
+    ├── ASR ──── 语音识别 (SenseVoice / FunASR)
+    ├── LLM ──── 大语言模型 (DeepSeek / GPT-4o / Qwen / Ollama / Dify)
+    ├── TTS ──── 语音合成 (CosyVoice / EdgeTTS / Bailian TTS)
+    └── Avatar ─ 数字人驱动 (LiteAvatar / MuseTalk / LAM)
+```
+
+所有模块均基于 **Handler 插件体系**，可通过 YAML 配置自由组合和替换。
+
+## 核心特性
+
+- **模块化插件架构** — ASR、LLM、TTS、Avatar 各环节可独立替换，通过配置文件切换
+- **多种医疗角色** — 内置主治医生、责任护士、健康管理师、临床药师、心理咨询师、营养师 6 种专业角色模板
+- **多模型支持** — 兼容 OpenAI、DeepSeek、通义千问、Ollama、Dify 等主流 LLM 服务
+- **实时音视频** — 基于 WebRTC (aiortc) 实现低延迟双向通信
+- **数字人驱动** — 支持 LiteAvatar 轻量渲染、MuseTalk 唇形驱动、LAM 表情驱动
+- **单机全流程** — 在单台 GPU 服务器上即可运行完整系统
+- **Docker 部署** — 提供 Dockerfile 与 docker-compose，支持 TURN 服务器 (coturn)
+
+## 环境要求
+
+- **Python**: >=3.11.7, <3.12
+- **GPU**: NVIDIA CUDA 兼容显卡（推荐 >=8GB 显存）
+- **OS**: Linux / Windows
+- **Node.js**: >=18（前端开发需要）
+
+## 快速开始
+
+### 1. 安装
+
+```bash
+# 创建虚拟环境
+python -m venv venv
+source venv/bin/activate  # Windows: venv\Scripts\activate
+
+# 根据配置文件安装依赖
+python install.py --config config/chat_with_openai_compatible.yaml
+
+# 使用 uv 加速安装（可选）
+python install.py --config config/chat_with_openai_compatible.yaml --uv
+```
+
+### 2. 启动服务
+
+```bash
+# 方式一：直接启动
+python src/demo.py --config config/chat_with_openai_compatible.yaml --host 0.0.0.0 --port 8282
+
+# 方式二：一键启动所有服务
+bash start_all_services.sh
+
+# 停止所有服务
+bash stop_all_services.sh
+```
+
+### 3. 访问
+
+- 后端 API: `https://localhost:8282`
+- 前端页面: `https://localhost:5188`（开发模式）
+
+### Docker 部署
+
+```bash
+docker-compose up -d
+```
+
+## 配置说明
+
+系统使用 YAML 配置文件驱动，位于 `config/` 目录：
+
+| 配置文件 | 说明 |
+|---------|------|
+| `chat_with_openai_compatible.yaml` | OpenAI 兼容 API（DeepSeek/GPT 等）+ 本地 ASR + 本地 TTS |
+| `chat_with_openai_compatible_edge_tts.yaml` | OpenAI 兼容 API + Edge TTS |
+| `chat_with_openai_compatible_bailian_cosyvoice.yaml` | OpenAI 兼容 API + 阿里百炼 CosyVoice |
+| `chat_with_openai_compatible_bailian_cosyvoice_musetalk.yaml` | 上述配置 + MuseTalk 数字人 |
+| `chat_with_lam.yaml` | LAM 数字人 + 本地模型 |
+| `chat_with_minicpm.yaml` | 本地 MiniCPM 模型 |
+| `chat_with_qwen_omni.yaml` | 通义千问多模态模型 |
+
+配置结构示例：
+
+```yaml
+default:
+  service:
+    host: "0.0.0.0"
+    port: 8282
+  chat_engine:
+    model_root: "./models"
+    concurrent_limit: 2
+    handler_configs:
+      SenseVoice:
+        enabled: true
+      OpenAICompatible:
+        enabled: true
+        api_key: "your-api-key"
+        model: "deepseek-chat"
+      CosyVoice:
+        enabled: true
+      LiteAvatar:
+        enabled: true
+```
+
+## 项目结构
+
+```
+├── config/                  # YAML 配置文件
+├── models/                  # 模型文件目录
+├── resource/                # 数字人形象资源
+├── src/
+│   ├── demo.py              # 主入口 (FastAPI 服务)
+│   ├── core/                # 角色模板与预设
+│   ├── chat_engine/         # 编排引擎核心
+│   │   ├── chat_engine.py   # ChatEngine 主类
+│   │   ├── core/            # 会话管理、Handler 管理
+│   │   ├── common/          # Handler 基类
+│   │   └── data_models/     # 数据模型
+│   ├── handlers/            # 插件模块
+│   │   ├── asr/             # 语音识别 (SenseVoice, FunASR)
+│   │   ├── llm/             # 大语言模型 (OpenAI, MiniCPM, Qwen, Dify)
+│   │   ├── tts/             # 语音合成 (CosyVoice, EdgeTTS, Bailian)
+│   │   ├── avatar/          # 数字人 (LiteAvatar, MuseTalk, LAM)
+│   │   ├── vad/             # 语音活动检测 (Silero VAD)
+│   │   └── client/          # 客户端 (RTC WebUI)
+│   ├── service/             # HTTP/WebSocket 服务层
+│   └── third_party/         # 第三方库
+├── install.py               # 安装脚本
+├── pyproject.toml           # 项目元数据与依赖
+├── Dockerfile               # 容器构建
+├── docker-compose.yaml      # 容器编排
+├── start_all_services.sh    # 一键启动
+└── stop_all_services.sh     # 一键停止
+```
+
+## 前端开发
+
+前端位于 `src/handlers/client/rtc_client/frontend/`，基于 Vue 3 + TypeScript + Ant Design Vue。
+
+```bash
+cd src/handlers/client/rtc_client/frontend
+pnpm install
+pnpm dev    # 开发服务器 (https://localhost:5188)
+pnpm build  # 生产构建
+```
+
+## 医疗角色
+
+系统内置 6 种医疗随访角色，每种角色具有独立的对话风格和关注重点：
+
+| 角色 | 说明 |
+|------|------|
+| 主治医生 | 病情诊断、治疗方案、复查安排 |
+| 责任护士 | 日常护理、生活习惯、身体舒适度 |
+| 健康管理师 | 慢病管理、生活方式干预、健康指导 |
+| 临床药师 | 用药安全、药物相互作用、服药依从性 |
+| 心理咨询师 | 心理健康、情绪状态、心理支持 |
+| 营养师 | 饮食营养、膳食指导、营养干预 |
+
+## 技术栈
+
+| 类别 | 技术 |
+|------|------|
+| 后端框架 | FastAPI + Uvicorn |
+| 实时通信 | aiortc (WebRTC) |
+| AI/ML | PyTorch, ModelScope, OpenAI SDK |
+| 配置管理 | Dynaconf (YAML) |
+| 前端 | Vue 3, TypeScript, Vite, Ant Design Vue, Pinia |
+| 容器化 | Docker (NVIDIA CUDA 12.2 + Ubuntu 22.04) |
+
+## License
+
+本项目仅供研究与学习使用。

build_and_run.sh

+#!/usr/bin/env bash
+CONFIG_PATH=""
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        -config | --config )
+            CONFIG_PATH="$2"
+            shift 2
+            ;;
+    esac
+done
+
+echo "${CONFIG_PATH}"
+
+docker build \
+    --build-arg CONFIG_FILE=${CONFIG_PATH}  \
+    -t open-avatar-chat:0.0.1 . 
+docker run --rm --gpus all -it --name open-avatar-chat \
+    --network=host \
+    -v `pwd`/build:/root/open-avatar-chat/build \
+    -v `pwd`/models:/root/open-avatar-chat/models \
+    -v `pwd`/ssl_certs:/root/open-avatar-chat/ssl_certs \
+    -v `pwd`/config:/root/open-avatar-chat/config \
+    -v `pwd`/models/musetalk/s3fd-619a316812/:/root/.cache/torch/hub/checkpoints/ \
+    -e NVIDIA_VISIBLE_DEVICES=all \
+    -e NVIDIA_DRIVER_CAPABILITIES=compute,video,utility \
+    -p 8282:8282 \
+    open-avatar-chat:0.0.1 \
+    --config ${CONFIG_PATH}

build_cuda128.sh

+#!/usr/bin/env bash
+
+# Build script for CUDA 12.8 Dockerfile with dynamic versioning
+# Usage: ./build_cuda128.sh [--tag TAG] [--no-cache] [--push REGISTRY]
+
+set -e
+
+# Default values
+IMAGE_TAG=""
+NO_CACHE=""
+PUSH_REGISTRY=""
+CONFIG_PATH="config/chat_with_openai_compatible_bailian_cosyvoice.yaml"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Logging functions
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# Function to get version from git
+get_version() {
+    if [ -d .git ]; then
+        # # Try to get version from git tag first
+        # local git_tag=$(git describe --tags --exact-match 2>/dev/null)
+        # if [ $? -eq 0 ] && [ -n "$git_tag" ]; then
+        #     echo "$git_tag"
+        #     return
+        # fi
+        
+        # # Try to get version from nearest tag + commits
+        # local desc_tag=$(git describe --tags --always 2>/dev/null)
+        # if [ $? -eq 0 ] && [ -n "$desc_tag" ] && [[ "$desc_tag" == *"-"* ]]; then
+        #     # Replace slash with underscore for Docker compatibility
+        #     echo "$desc_tag" | sed 's/\//_/g'
+        #     return
+        # fi
+        
+        # Fallback: Generate version from branch and short commit
+        local branch=$(git rev-parse --abbrev-ref HEAD 2>/dev/null)
+        local commit=$(git rev-parse --short HEAD 2>/dev/null)
+        
+        if [ -n "$branch" ] && [ -n "$commit" ]; then
+            # Replace slash with underscore for Docker compatibility  
+            local clean_branch=$(echo "$branch" | sed 's/\//_/g')
+            echo "${clean_branch}-${commit}"
+        else
+            echo "unknown-$(date +%Y%m%d)"
+        fi
+    else
+        echo "dev-$(date +%Y%m%d)"
+    fi
+}
+
+# Parse arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --tag )
+            IMAGE_TAG="$2"
+            shift 2
+            ;;
+        --no-cache )
+            NO_CACHE="--no-cache"
+            shift
+            ;;
+        --push )
+            PUSH_REGISTRY="$2"
+            shift 2
+            ;;
+        --config )
+            CONFIG_PATH="$2"
+            shift 2
+            ;;
+        -h | --help )
+            echo "Usage: $0 [OPTIONS]"
+            echo "Options:"
+            echo "  --tag TAG         Docker镜像标签 (默认: open-avatar-chat:VERSION)"
+            echo "  --no-cache        不使用Docker构建缓存"
+            echo "  --push REGISTRY   构建后推送到指定注册表"
+            echo "  --config PATH     运行时使用的配置文件路径"
+            echo "  -h, --help        显示帮助信息"
+            echo ""
+            echo "Examples:"
+            echo "  $0                                          # 基本构建"
+            echo "  $0 --tag myapp:latest                       # 自定义标签"
+            echo "  $0 --push ghcr.io/user/repo                # 构建并推送"
+            echo "  $0 --no-cache --push ghcr.io/user/repo     # 强制重建并推送"
+            exit 0
+            ;;
+        * )
+            log_error "未知参数: $1"
+            exit 1
+            ;;
+    esac
+done
+
+# Validate Docker is available
+if ! command -v docker &> /dev/null; then
+    log_error "Docker 未找到，请先安装Docker"
+    exit 1
+fi
+
+# Check if Dockerfile.cuda12.8 exists
+if [ ! -f "Dockerfile.cuda12.8" ]; then
+    log_error "Dockerfile.cuda12.8 未找到"
+    exit 1
+fi
+
+# Set default image tag if not provided
+if [ -z "$IMAGE_TAG" ]; then
+    IMAGE_TAG="open-avatar-chat:latest"
+fi
+
+# Get build metadata
+BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+BUILD_VERSION=$(get_version)
+
+log_info "=== 构建信息 ==="
+log_info "镜像标签: ${IMAGE_TAG}"
+log_info "构建版本: ${BUILD_VERSION} (嵌入到镜像标签中)"
+log_info "构建时间: ${BUILD_DATE} (嵌入到镜像标签中)"
+log_info "Dockerfile: Dockerfile.cuda12.8"
+
+#Build command
+BUILD_CMD="docker build ${NO_CACHE} \
+    --build-arg BUILD_VERSION=\"${BUILD_VERSION}\" \
+    --build-arg BUILD_DATE=\"${BUILD_DATE}\" \
+    -f Dockerfile.cuda12.8 \
+    -t \"${IMAGE_TAG}\" ."
+
+log_info "=== 开始构建 ==="
+log_info "执行命令: ${BUILD_CMD}"
+
+# Execute build
+if eval $BUILD_CMD; then
+    log_success "镜像构建成功: ${IMAGE_TAG}"
+    
+    # Show image info
+    log_info "=== 镜像信息 ==="
+    docker images "${IMAGE_TAG}" --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedAt}}"
+    
+    # Push if requested
+    if [ -n "$PUSH_REGISTRY" ]; then
+        PUSH_TAG="${PUSH_REGISTRY}:${BUILD_VERSION}"
+        log_info "=== 推送镜像 ==="
+        log_info "标记镜像: ${PUSH_TAG}"
+        
+        if docker tag "${IMAGE_TAG}" "${PUSH_TAG}"; then
+            log_info "推送镜像: ${PUSH_TAG}"
+            if docker push "${PUSH_TAG}"; then
+                log_success "镜像推送成功: ${PUSH_TAG}"
+            else
+                log_error "镜像推送失败"
+                exit 1
+            fi
+        else
+            log_error "镜像标记失败"
+            exit 1
+        fi
+    fi
+    
+    # Provide usage instructions
+    log_info "=== 使用说明 ==="
+    echo ""
+    echo "查看镜像中的commit信息:"
+    echo "docker inspect \"${IMAGE_TAG}\" | grep -A 10 'Labels'"
+    echo ""
+    echo "启动容器:"
+    echo "docker run --rm --gpus all -it \\"
+    echo "  --network=host \\"
+    echo "  -v \$(pwd)/build:/root/open-avatar-chat/build \\"
+    echo "  -v \$(pwd)/models:/root/open-avatar-chat/models \\"
+    echo "  -v \$(pwd)/ssl_certs:/root/open-avatar-chat/ssl_certs \\"
+    echo "  -v \$(pwd)/config:/root/open-avatar-chat/config \\"
+    echo "  -v \$(pwd)/models/musetalk/s3fd-619a316812/:/root/.cache/torch/hub/checkpoints/ \\"
+    echo "  -p 8282:8282 \\"
+    echo "  \"${IMAGE_TAG}\" \\"
+    echo "  --config \"${CONFIG_PATH}\""
+    echo ""
+    echo "在容器内查看版本信息:"
+    echo "docker run --rm \"${IMAGE_TAG}\" printenv | grep -E '^(APP_VERSION|BUILD_DATE|BUILD_COMMIT)='"
+    echo ""
+    
+else
+    log_error "镜像构建失败"
+    exit 1
+fi

config/chat_with_lam.yaml

+default:
+  logger:
+    log_level: "INFO"
+  service:
+    host: "0.0.0.0"
+    port: 8282
+    cert_file: "ssl_certs/localhost.crt"
+    cert_key: "ssl_certs/localhost.key"
+  chat_engine:
+    model_root: "models"
+    concurrent_limit: 5
+    handler_search_path:
+      - "src/handlers"
+    handler_configs:
+      LamClient:
+        module: client/h5_rendering_client/client_handler_lam
+        asset_path: "lam_samples/barbara.zip"
+        # max time a session will last for
+        connection_ttl: 900
+      SileroVad:
+        module: vad/silerovad/vad_handler_silero
+        speaking_threshold: 0.5
+        start_delay: 2048
+        end_delay: 5000
+        buffer_look_back: 5000
+        speech_padding: 512
+      SenseVoice:
+        enabled: True
+        module: asr/sensevoice/asr_handler_sensevoice
+        model_name: "iic/SenseVoiceSmall"
+      CosyVoice:
+        enabled: True
+        module: tts/bailian_tts/tts_handler_cosyvoice_bailian
+        voice: "longxiaocheng"
+        model_name: "cosyvoice-v1"
+        # api_key: "" # default=os.getenv("DASHSCOPE_API_KEY")
+      LLMOpenAICompatible:
+        enabled: True
+        module: llm/openai_compatible/llm_handler_openai_compatible
+        model_name: "qwen-plus"
+        enable_video_input: False # ensure your llm support video input
+        history_length: 20
+        # model_name: "gemini-2.0-flash"
+        system_prompt: "请你扮演一个 AI 助手，用简短的两三句对话来回答用户的问题，并在对话内容中加入合适的标点符号，不需要讨论标点符号相关的内容"
+        api_url: "https://dashscope.aliyuncs.com/compatible-mode/v1"
+        # api_url: 'http://127.0.0.1:11434/v1' # ollama
+        # api_url: 'https://generativelanguage.googleapis.com/v1beta/openai/'
+        # api_key: "" # default=os.getenv("DASHSCOPE_API_KEY")
+      # Dify:
+      #   enabled: True
+      #   module: llm/dify/llm_handler_dify
+      #   api_key: app-J3LiQRqh4MrOOvS6vpyrgIOW
+      #   api_url: 'http://localhost/v1'
+      LAM_Driver:
+        module: avatar/lam/avatar_handler_lam_audio2expression

config/chat_with_minicpm.yaml

+default:
+  logger:
+    log_level: "INFO"
+  service:
+    host: "0.0.0.0"
+    port: 8282
+    cert_file: "ssl_certs/localhost.crt"
+    cert_key: "ssl_certs/localhost.key"
+  chat_engine:
+    model_root: "models"
+    handler_search_path:
+      - "src/handlers"
+    handler_configs:
+      RtcClient:
+        module: client/rtc_client/client_handler_rtc
+        # max time a session will last for
+        connection_ttl: 900
+      SileroVad:
+        module: vad/silerovad/vad_handler_silero
+        speaking_threshold: 0.5
+        start_delay: 2048
+        end_delay: 5000
+        buffer_look_back: 5000
+        speech_padding: 512
+      MiniCPM-o:
+        enabled: True
+        module: llm/minicpm/llm_handler_minicpm
+        model_name: "MiniCPM-o-2_6"
+        # model_name: "MiniCPM-o-2_6-int4"
+        voice_prompt: "你是一个AI助手。你能接受视频，音频和文本输入并输出语音和文本。模仿输入音频中的声音特征。"
+        assistant_prompt: "作为助手，你将使用这种声音风格说话。"
+        enable_video_input: True
+        skip_video_frame: 2
+      LiteAvatar:
+        module: avatar/liteavatar/avatar_handler_liteavatar
+        avatar_name: 20250408/sample_data
+        fps: 25
+        debug: false
+        enable_fast_mode: false
+        use_gpu: false

config/chat_with_openai_compatible.yaml

+default:
+  logger:
+    log_level: "INFO"
+  service:
+    host: "0.0.0.0"
+    port: 8282
+    cert_file: "ssl_certs/localhost.crt"
+    cert_key: "ssl_certs/localhost.key"
+  chat_engine:
+    model_root: "models"
+    handler_search_path:
+      - "src/handlers"
+    handler_configs:
+      RtcClient:
+        module: client/rtc_client/client_handler_rtc
+        # max time a session will last for
+        connection_ttl: 900
+      SileroVad:
+        module: vad/silerovad/vad_handler_silero
+        speaking_threshold: 0.5
+        start_delay: 2048
+        end_delay: 5000
+        buffer_look_back: 5000
+        speech_padding: 512
+      SenseVoice:
+        enabled: True
+        module: asr/sensevoice/asr_handler_sensevoice
+        model_name: "iic/SenseVoiceSmall"
+
+      CosyVoice:
+        enabled: True
+        # switch to local cosyvoice handler
+        module: tts/cosyvoice/tts_handler_cosyvoice 
+        # model_name: "pretrained_models/CosyVoice-300M-SFT" 
+        api_url: "http://127.0.0.1:50000/inference_zero_shot" # 改为你实际的API地址
+        # spk_id: "中文女" # 确保API端支持该音色ID
+      LLMOpenAICompatible:
+        enabled: True
+        module: llm/openai_compatible/llm_handler_openai_compatible
+        model_name: "qwen-plus"
+        enable_video_input: False # ensure your llm support video input
+        history_length: 20
+        # model_name: "gemini-2.0-flash"
+        system_prompt: "请你扮演一个 AI 助手，用简短的两三句对话来回答用户的问题，并在对话内容中加入合适的标点符号，不需要讨论标点符号相关的内容"
+        api_url: "https://dashscope.aliyuncs.com/compatible-mode/v1"
+        # api_url: 'http://127.0.0.1:11434/v1' # ollama
+        # api_url: 'https://generativelanguage.googleapis.com/v1beta/openai/'
+        # api_key: '' # default=os.getenv("DASHSCOPE_API_KEY")
+      LiteAvatar:
+        module: avatar/liteavatar/avatar_handler_liteavatar
+        avatar_name: 20250408/sample_data
+        fps: 25
+        debug: false
+        enable_fast_mode: false
+        use_gpu: true

config/chat_with_openai_compatible_bailian_cosyvoice.yaml

+default:
+  logger:
+    log_level: "INFO"
+  service:
+    host: "0.0.0.0"
+    port: 8282
+    cert_file: "ssl_certs/localhost.crt"
+    cert_key: "ssl_certs/localhost.key"
+  chat_engine:
+    model_root: "models"
+    concurrent_limit: '2'
+    handler_search_path:
+      - "src/handlers"
+    handler_configs:
+      RtcClient:
+        module: client/rtc_client/client_handler_rtc
+        # max time a session will last for
+        connection_ttl: 900
+        turn_config:
+          turn_provider: "turn_server"
+          urls: ["turn:10.10.0.102:8282"]
+          username: "admin"
+          credential: "admin"
+      SileroVad:
+        module: vad/silerovad/vad_handler_silero
+        speaking_threshold: 0.5
+        start_delay: 2048
+        end_delay: 5000
+        buffer_look_back: 5000
+        speech_padding: 512
+      # SenseVoice:
+      #   enabled: True
+      #   module: asr/sensevoice/asr_handler_sensevoice
+      #   model_name: "iic/SenseVoiceSmall"
+      # 使用本地 FunASR API
+      FunASRAPI:
+        enabled: True
+        module: asr/funasr_api/asr_handler_funasr_api
+        api_url: "http://localhost:8000/transcribe"
+        language: "中文"
+        use_vad: False
+        itn: True
+        # hotwords: "关键词1,关键词2"
+        timeout: 30.0
+      # CosyVoice:
+      #   enabled: True
+      #   module: tts/bailian_tts/tts_handler_cosyvoice_bailian
+      #   voice: "longxiaochun"
+      #   model_name: "cosyvoice-v1"
+      #   api_key: "" # default=os.getenv("DASHSCOPE_API_KEY")
+      CosyVoice:
+        enabled: True
+        # switch to local cosyvoice handler
+        module: tts/cosyvoice/tts_handler_cosyvoice 
+        # model_name: "pretrained_models/CosyVoice-300M-SFT" 
+        api_url: "http://127.0.0.1:50000/inference_zero_shot" # 改为你实际的API地址
+        # spk_id: "中文女" # 确保API端支持该音色ID
+        
+        process_num: 1
+      LLMOpenAICompatible:
+        enabled: True
+        module: llm/openai_compatible/llm_handler_openai_compatible
+        model_name: "deepseek-chat"
+        enable_video_input: False # ensure your llm support video input
+        history_length: 20
+        # model_name: "gemini-2.0-flash"
+        # system_prompt: "请你扮演一个 AI 助手，用简短的两三句对话来回答用户的问题，并在对话内容中加入合适的标点符号，不需要讨论标点符号相关的内容"
+        # api_url: "https://dashscope.aliyuncs.com/compatible-mode/v1"
+        # api_url: 'http://127.0.0.1:11434/v1' # ollama
+        # api_url: 'https://generativelanguage.googleapis.com/v1beta/openai/'
+        # api_key: "" # default=os.getenv("DASHSCOPE_API_KEY")
+        api_url: "https://api.deepseek.com/v1"
+        api_key: ""
+      LiteAvatar:
+        module: avatar/liteavatar/avatar_handler_liteavatar
+        avatar_name: 20250408/sample_data
+        fps: 25
+        debug: false
+        enable_fast_mode: false
+        use_gpu: true

config/chat_with_openai_compatible_bailian_cosyvoice_musetalk.yaml

+default:
+  logger:
+    log_level: "INFO"
+  service:
+    host: "0.0.0.0"
+    port: 8282
+    cert_file: "ssl_certs/localhost.crt"
+    cert_key: "ssl_certs/localhost.key"
+  chat_engine:
+    model_root: "models"
+    concurrent_limit: 10
+    handler_search_path:
+      - "src/handlers"
+    handler_configs:
+      RtcClient:
+        module: client/rtc_client/client_handler_rtc
+        # max time a session will last for
+        connection_ttl: 900
+        turn_config:
+          turn_provider: "turn_server"
+          urls: ["turn:10.10.0.102:8282"]
+          username: "admin"
+          credential: "admin"
+      SileroVad:
+        module: vad/silerovad/vad_handler_silero
+        speaking_threshold: 0.5
+        start_delay: 2048
+        end_delay: 5000
+        buffer_look_back: 5000
+        speech_padding: 512
+      # 使用本地 FunASR API
+      FunASRAPI:
+        enabled: True
+        module: asr/funasr_api/asr_handler_funasr_api
+        api_url: "http://localhost:8000/transcribe"
+        language: "中文"
+        use_vad: False
+        itn: True
+        # hotwords: "关键词1,关键词2"
+        timeout: 30.0
+      CosyVoice:
+        enabled: True
+        # switch to local cosyvoice handler
+        module: tts/cosyvoice/tts_handler_cosyvoice 
+        # # model_name: "pretrained_models/CosyVoice-300M-SFT" 
+        # api_url: "http://127.0.0.1:50000/inference_zero_shot" # 改为你实际的API地址
+        api_url: "http://127.0.0.1:50001/tts" # 改为你实际的API地址
+        # spk_id: "中文女" # 确保API端支持该音色ID
+        process_num: 10
+      LLMOpenAICompatible:
+        enabled: True
+        module: llm/openai_compatible/llm_handler_openai_compatible
+        model_name: "deepseek-chat"
+        enable_video_input: False # ensure your llm support video input
+        history_length: 20
+        api_url: "https://api.deepseek.com/v1"
+        api_key: ""
+      AvatarMusetalk:
+        module: avatar/musetalk_api_gai/avatar_handler_musetalk_api
+        ws_url: "ws://localhost:8777"  # WebSocket 地址
+        avatar_id: "psychologist25f"  # Avatar ID（需要和 WebSocket 服务中的一致）
+        output_audio_sample_rate: 16000  # 输出音频采样率
+        fps: 25  # 视频帧率（用于数据定义，应该和 WebSocket 服务一致）
+        debug: false  # 是否开启调试模式
+        concurrent_limit: 10

config/chat_with_openai_compatible_edge_tts.yaml

+default:
+  logger:
+    log_level: "INFO"
+  service:
+    host: "0.0.0.0"
+    port: 8282
+    cert_file: "ssl_certs/localhost.crt"
+    cert_key: "ssl_certs/localhost.key"
+  chat_engine:
+    model_root: "models"
+    concurrent_limit: 1
+    handler_search_path:
+      - "src/handlers"
+    handler_configs:
+      RtcClient:
+        module: client/rtc_client/client_handler_rtc
+        # max time a session will last for
+        connection_ttl: 900
+      SileroVad:
+        module: vad/silerovad/vad_handler_silero
+        speaking_threshold: 0.5
+        start_delay: 2048
+        end_delay: 5000
+        buffer_look_back: 5000
+        speech_padding: 512
+      SenseVoice:
+        enabled: True
+        module: asr/sensevoice/asr_handler_sensevoice
+        model_name: "iic/SenseVoiceSmall"
+      Edge_TTS:
+        enabled: True
+        module: tts/edgetts/tts_handler_edgetts
+        voice: "zh-CN-XiaoxiaoNeural"
+      LLMOpenAICompatible:
+        enabled: True
+        module: llm/openai_compatible/llm_handler_openai_compatible
+        model_name: "qwen-plus"
+        enable_video_input: False # ensure your llm support video input
+        history_length: 20
+        # model_name: "gemini-2.0-flash"
+        system_prompt: "请你扮演一个 AI 助手，用简短的两三句对话来回答用户的问题，并在对话内容中加入合适的标点符号，不需要讨论标点符号相关的内容"
+        api_url: "https://dashscope.aliyuncs.com/compatible-mode/v1"
+        # api_url: 'http://127.0.0.1:11434/v1' # ollama
+        # api_url: 'https://generativelanguage.googleapis.com/v1beta/openai/'
+        # api_key: "" # default=os.getenv("DASHSCOPE_API_KEY")
+      LiteAvatar:
+        module: avatar/liteavatar/avatar_handler_liteavatar
+        avatar_name: 20250408/sample_data
+        fps: 25
+        debug: false
+        enable_fast_mode: false
+        use_gpu: true

config/chat_with_qwen_omni.yaml

+default:
+  logger:
+    log_level: "INFO"
+  service:
+    host: "0.0.0.0"
+    port: 8282
+    cert_file: "ssl_certs/localhost.crt"
+    cert_key: "ssl_certs/localhost.key"
+  chat_engine:
+    model_root: "models"
+    concurrent_limit: 1
+    handler_search_path:
+      - "src/handlers"
+    handler_configs:
+      RtcClient:
+        module: client/rtc_client/client_handler_rtc
+        connection_ttl: 36000
+        # max time a session will last for
+        connection_ttl: 900
+      SileroVad:
+        module: vad/silerovad/vad_handler_silero
+        speaking_threshold: 0.5
+        start_delay: 2048
+        end_delay: 5000
+        buffer_look_back: 5000
+        speech_padding: 512
+      SenseVoice:
+        enabled: True
+        module: asr/sensevoice/asr_handler_sensevoice
+        model_name: "iic/SenseVoiceSmall"
+      QwenOmni:
+        enabled: true
+        module: llm/qwen_omni/llm_handler_qwen_omni
+        model_name: "qwen-omni-turbo-realtime"
+        # api_key: "" # default=os.getenv("DASHSCOPE_API_KEY") api_key可以通过此处设置，或者通过环境变量，没设必挂
+        voice: "Serena" #"Chelsie", "Serena", "Ethan", "Cherry"
+        enable_video_input: true
+        video_frame_interval_ms: 1000 # 每1000ms发送一帧图片，防止拥塞，可按需调整，根据omni的文档，不要低于500ms
+        enable_text_output: true
+        input_audio_format: "PCM_16000HZ_MONO_16BIT"
+        output_audio_format: "PCM_24000HZ_MONO_16BIT"
+        enable_turn_detection: false # 目前暂未支持服务端vad模式，此处默认为false，不要修改
+        enable_input_transcription: false # 手动模式下，omni的输入的语音识别效果非常差，几乎不可用，默认关闭，直接使用正常asr的结果
+        transcription_model: "gummy-realtime-v1"
+      # Musetalk和LiteAvatar可以通过配置自由选，默认使用LiteAvatar了
+      # AvatarMusetalk:
+      #   module: avatar/musetalk/avatar_handler_musetalk
+      #   fps: 20  # Video frame rate
+      #   batch_size: 2  # Batch processing frame count, must be greater than 2
+      #   avatar_video_path: "src/handlers/avatar/musetalk/MuseTalk/data/video/yongen.mp4"  # Initialization video path
+      #   avatar_model_dir: "models/musetalk/avatar_model"  # Default avatar model directory
+      #   force_create_avatar: false  # Whether to force regenerate digital human data
+      #   debug: false  # Whether to enable debug mode
+      #   multi_thread_inference: true  # true: use 2 threads to inference (unet, vae), false: use single-thread to inference
+      LiteAvatar:
+        module: avatar/liteavatar/avatar_handler_liteavatar
+        avatar_name: 20250408/sample_data
+        fps: 25
+        debug: false
+        enable_fast_mode: false
+        use_gpu: true

coturn-data/turnserver.conf

+listening-port=3478
+listening-ip=0.0.0.0
+tls-listening-port=5349
+realm=turn.open-avatar-chat.turnserver
+user=admin:admin
+lt-cred-mech
+min-port:49152
+max-port:65535
+cert=/etc/turn_cert.pem
+key=/etc/turn_key.pem
+allowed-peer-ip=0.0.0.0

docker-compose.yml

+services:
+  coturn:
+    image: coturn/coturn
+    container_name: coturn-server
+    command: >
+      --log-file=stdout
+    environment:
+      - DETECT_EXTERNAL_IP=yes
+      - DETECT_RELAY_IP=yes
+    volumes:
+      - ./coturn-data/turnserver.conf:/etc/coturn/turnserver.conf
+      - ./ssl_certs/localhost.crt:/etc/turn_cert.pem
+      - ./ssl_certs/localhost.crt:/etc/turn_key.pem
+    network_mode: host
+    restart: unless-stopped
+
+  open-avatar-chat:
+    image: open-avatar-chat:latest
+    container_name: open-avatar-chat
+    environment:
+      - NVIDIA_VISIBLE_DEVICES=all
+      - NVIDIA_DRIVER_CAPABILITIES=compute,utility,video
+      # - UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/
+    command: >
+      --config=/root/open-avatar-chat/config/chat_with_openai_compatible_bailian_cosyvoice.yaml
+    volumes:
+      - ./build:/root/open-avatar-chat/build \
+      - ./models:/root/open-avatar-chat/models \
+      - ./ssl_certs:/root/open-avatar-chat/ssl_certs \
+      - ./config:/root/open-avatar-chat/config \
+      - ./.env:/root/open-avatar-chat/.env \
+      - ./models/musetalk/s3fd-619a316812/:/root/.cache/torch/hub/checkpoints/ \
+      - ./exp:/root/open-avatar-chat/exp \
+      - ./resource:/root/open-avatar-chat/resource \
+    ports:
+      - "8282:8282"
+    networks:
+      - nat_network
+    shm_size: "2gb"
+    stdin_open: true
+    tty: true
+    restart: unless-stopped
+    depends_on:
+      - coturn
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [ gpu ]
+
+networks:
+  nat_network:
+    driver: bridge
+    

docs/FAQ.md

+# 🚨FAQs | 常见问题🚨
+
+> [!NOTE]
+> Please avoid creating issues regarding the following questions, as they might be closed without a response.
+> 
+> 请避免创建与下述问题有关的 issues，这些 issues 可能不会被回复。
+
+
+## Deployment Related Issues / 部署相关问题
+
+### Environment Configuration / 环境配置
+
+**Q: What operating systems are supported by the project? / 项目支持哪些操作系统**  
+
+Currently supports Linux and Windows.  
+目前支持Linux和Windows。
+
+```
+The LAM can be run on a mac, just remove cuda related dependencies like onnxruntime-gpu to run on a cpu!
+LAM 部分可以使用mac运行，只需移除cuda 相关的依赖，比如onnxruntime-gpu，就可以在cpu 上运行
+```
+
+### Dependency Installation / 依赖安装
+
+**Q: How to resolve onnxruntime-gpu installation failure? / 安装 onnxruntime-gpu 失败怎么办？**  
+
+1. Verify CUDA version compatibility  
+2. Check Python version compatibility  
+3. Try installing via conda environment  
+4. Pay attention to platform compatibility (manylinux_2_27_x86_64, manylinux_2_28_x86_64, win_amd64)  
+<!-- new list start -->
+
+1. 确认 CUDA 版本兼容性  
+2. 检查 Python 版本是否匹配  
+3. 尝试使用 conda 环境安装  
+4. 注意平台兼容性（manylinux_2_27_x86_64, manylinux_2_28_x86_64, win_amd64）
+
+---
+
+**Q: Is RTX 50 supported / 50系显卡是否支持**
+Currently, 50 series need to use cuda12.8 or above, the corresponding pytorch-related packages need to be installed as version 12.8.
+目前50系显卡需要使用cuda12.8以上，对应pytorch相关的包需要安装成12.8的版本
+```
+#https://pytorch.org/get-started/locally/
+pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128
+```
+---
+
+**Q: Is CPU(or mac) supported / 纯CPU或者mac机器是否能部署**
+It can only run smoothly with config/chat_with_lam.yaml, but the lite-avatar self-test won't run. It seems like have to manually change all device to mps to make it work.
+只能顺畅的运行 config/chat_with_lam.yaml, lite-avatar 自测跑不动，估计要全部手动改成 mps 才可能。
+```
+#运行chat_with_lam.yaml 所需额外步骤
+#第一步：移除torchvision 依赖
+#第二步：替换onnxruntime-gpu 为 onnxruntime 依赖
+#修改 src/handlers/avatar/lam/LAM_Audio2Expression/engines/infer.py ,删除所有.cuda()方法的调用
+#按照readme 按照依赖，并运行即可
+
+```
+---
+
+**Q: How to resolve pynini installation issues? / pynini 安装出现问题怎么办？**  
+
+Refer to the cosyvoice module installation section in README.  
+查看readme中关于cosyvoice模块安装的部分。
+
+---
+
+**Q: Error: fastrtc-0.0.19.dev0-py3-none-any.whl not found during installation / 安装报错文件fastrtc-0.0.19.dev0-py3-none-any.whl不存在**  
+
+This error indicates incomplete submodule retrieval. Re-pull submodules from the project root directory.  
+这个报错说明子模块没有全部拉下来。在项目根目录重新拉取子模块。
+
+---
+
+**Q: Error related to 'gbk' encoding on Windows / windows下出现'gbk'编码相关的错误**
+
+Manually set environment variable PYTHONUTF8=1.  
+可以手动设置环境变量PYTHONUTF8=1。
+
+---
+
+**Q: Error when running deployment after pip install requirements.txt / 使用pip install安装对应的requirements.txt，部署运行时报错**  
+
+Project dependencies are modularized. The root requirements.txt contains only public dependencies. Use uv for installation, or manually install required module dependencies based on .toml files.  
+项目依赖以模块化的方式存放，根目录下的requirements.txt只包含公共依赖。请使用uv进行依赖安装，或者根据需要用到的模块下的.toml文件，手动安装所需模块的对应依赖。
+
+### Deployment Issues / 部署问题
+
+**Q: AutoDL deployment of TURN Server fails to enable remote access? / AutoDL部署了TURN Server之后还是不能远程打开？**  
+
+AutoDL does not allow personal users to open custom ports, remote login unavailable.  
+AutoDL不支持个人用户开启自定义端口，无法远程登录。
+
+## Runtime Related Issues / 运行相关问题
+
+### Performance Issues / 性能问题
+
+**Q: Intermittent lag when using minicpm as LLM on 4090 / 4090下使用minicpm作为llm，对话过程中时不时存在卡顿**
+
+Non-quantized minicpm shows lag on 4090. Recommend switching LLM to API call or using non-multimodal local models.  
+minicpm非量化版本在4090上测试下来是会存在卡顿，建议llm改用api调用，或者本地的非多模态模型。
+
+### Common Unexpected Behaviours / 常见意外情况
+
+**Q: Session stopped unexpectedly without any error log after a while / 运行过程中session意外的停止，但是日志中并没有明显异常**
+
+There is max duration limit of session in the rtc handler. It's called 'connection_ttl' in config of client handler, the default is 900 seconds. It can be altered by user.
+rtc handler中有一个session的时长限制，这个参数可以被配置在client handler下，参数名是connection_ttl，默认是900秒。
+
+## Audio Related Issues / 语音相关问题
+
+### TTS Model / TTS 模型
+
+**Q: How to resolve audio model lag? / 语音模型卡顿怎么解决？** 
+
+For local cosyvoice, check GPU memory usage and adjust batch size in config. Switch to API call if needed. For API lag, check network issues and API response latency.  
+如果调用的是本地的cosyvoice，请检查GPU显存使用情况，可以在配置文件中调整批处理大小。或者改为API调用。如果是API调用卡顿，请排查网络问题和API本身的调用返回延迟。
+
+### Audio Interaction / 语音交互
+
+**Q: How to improve speech recognition accuracy? / 语音识别不准确怎么办？**  
+
+1. Check microphone settings  
+2. Ensure low ambient noise  
+3. Adjust speech recognition parameters  
+<!-- new list start -->
+
+1. 检查麦克风设置  
+2. 确保环境噪音较小  
+3. 调整语音识别参数
+
+## Feature Usage / 功能使用
+
+### Digital Human / 数字人
+
+**Q: How to customize digital human appearance? / 如何自定义数字人外观？**
+
+LiteAvatar does not support customization but provides [official character library](https://modelscope.cn/models/HumanAIGC-Engineering/LiteAvatarGallery). LAM supports customization via [Git project](https://github.com/aigc3d/LAM).  
+LiteAvatar暂不支持自定义，但可以使用[官方形象库](https://modelscope.cn/models/HumanAIGC-Engineering/LiteAvatarGallery)。LAM 数字人支持自定义，参考对应的[git项目地址](https://github.com/aigc3d/LAM)。
+
+---
+
+**Q: How to change digital human models? / 如何更换数字人模型？**  
+
+Locate the target character and modify the appearance parameters in config file.  
+找到对应想修改的角色，然后更换config文件中对应的形象参数。
+
+---
+
+**Q: How to enable model vision capabilities and how are they implemented? / 怎么开启模型的视觉功能。模型的视觉具体是如何实现的** 
+
+Select model IDs with vision capabilities like qwen_vl via API, or use local multimodal models. Implementation combines LLM with the last captured video frame during user interaction.  
+在api调用时选择具有视觉功能的model id如qwen_vl，或使用本地的多模态模型。具体实现是将用户对话时摄像头捕获到的最后一帧画面一起提交给llm。
+
+---
+
+**Q: Does the project support multi-channel concurrency? / 项目目前支持多路并发吗？**
+
+LiteAvatar does not support concurrency while LAM supports it via configuration file changes.  
+目前LiteAvatar数字人不支持多路并发，LAM数字人支持多路并发，可以在对应配置文件中修改。
+
+---
+**Q: Where is the front-end code? / 前端代码在哪里？**
+gradio_webrtc in the git submodule, which contains wrappers for webrtc functionality and UI-related code
+git submodule 中的 gradio_webrtc，这个组件包含了webrtc 功能的封装和 UI 相关的代码
+
+Path：OpenAvatarChat\src\third_party\gradio_webrtc_videochat
+Link： https://github.com/HumanAIGC-Engineering/gradio-webrtc.git
+
+### Integrated Features / 集成功能
+
+**Q: How to configure Turn Server? / 如何配置 Turn Server？**  
+
+Refer to Turn Server configuration README.  
+参考Turn Server配置的readme。
+
+
+## Best Practices / 最佳实践
+
+1. Use officially recommended configuration environment  
+2. Pull latest code  
+3. Maintain environment isolation using uv for dependency management  
+<!-- new list start -->
+
+1. 使用官方推荐的配置环境  
+2. 拉取最新代码  
+3. 做好环境隔离，使用uv进行依赖管理和配置
+
+> [!Tip]
+> If the problems still exist with the latest code, please create an issue.
+> 若使用最新的代码仍然无法解决问题，请创建一个 issue。

install.py

+import argparse
+import os
+import subprocess
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+import yaml
+
+from src.engine_utils.directory_info import DirectoryInfo
+
+
+def is_venv_active():
+    """Check if running inside a virtual environment (supports venv, virtualenv, and conda)"""
+    # Standard venv or virtualenv
+    if hasattr(sys, 'real_prefix') or (
+        hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix):
+        return True
+    # Conda environment
+    if os.getenv('CONDA_DEFAULT_ENV') is not None:
+        return True
+    # Virtualenv via VIRTUAL_ENV
+    if os.getenv('VIRTUAL_ENV') is not None:
+        return True
+    return False
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", type=str, default="/home/VoiceModel/shuziren/RealTimeMedicalDialogue/config/chat_with_openai_compatible_bailian_cosyvoice_musetalk.yaml",
+                        help="Path to config file")
+    parser.add_argument("--uv", action="store_true",
+                        help="Use uv pip compiler instead of standard pip")
+    parser.add_argument("--skip-core", action="store_true",
+                        help="Skip installation of core dependencies")
+    return parser.parse_args()
+
+
+def load_configs(in_args):
+    base_dir = DirectoryInfo.get_project_dir()
+    config_path = Path(in_args.config) if os.path.isabs(in_args.config) \
+        else Path(base_dir) / in_args.config
+
+    print(f"Loading config from {config_path}")
+    with open(config_path, "r", encoding="utf-8") as f:
+        return yaml.safe_load(f)
+
+
+def get_module_files(config, use_uv=False):
+    """Collect dependency files for enabled modules"""
+    base_dir = Path(DirectoryInfo.get_project_dir())
+    handler_configs = config.get("default", {}).get("chat_engine", {}).get("handler_configs", {})
+
+    module_files = {}
+    for handler_name, cfg in handler_configs.items():
+        if not cfg.get("enabled", True):
+            continue
+
+        module_path = Path(cfg.get("module", "")).parent
+        handler_dir = base_dir / "src/handlers" / module_path
+
+        # Prefer pyproject.toml when using uv
+        if use_uv:
+            toml_file = handler_dir / "pyproject.toml"
+            if toml_file.exists():
+                module_files[handler_name] = toml_file
+                continue
+
+        # Fallback to requirements.txt
+        req_file = handler_dir / "requirements.txt"
+        if req_file.exists():
+            module_files[handler_name] = req_file
+
+    return module_files
+
+
+def collect_root_file(use_uv=False):
+    """Get root dependency file based on tool preference"""
+    base_dir = Path(DirectoryInfo.get_project_dir())
+
+    if use_uv:
+        root_toml = base_dir / "pyproject.toml"
+        if root_toml.exists():
+            return root_toml
+
+    root_req = base_dir / "requirements.txt"
+    return root_req if root_req.exists() else None
+
+
+def install_files(file_paths, use_uv=False):
+    """Install dependencies from collected files"""
+    try:
+        for dep_file in file_paths:
+            print(f"Installing from {dep_file}")
+
+            if use_uv:
+                cmd = ["uv", "pip", "install", "-r", str(dep_file)]
+            else:
+                cmd = [sys.executable, "-m", "pip", "install", "-r", str(dep_file)]
+
+            subprocess.run(cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Installation failed: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    # Check virtual environment first (now supports conda!)
+    if not is_venv_active():
+        print("Error: Not running in a virtual environment.")
+        print("Create and activate a venv or conda environment first.")
+        sys.exit(1)
+
+    args = parse_args()
+    config = load_configs(args)
+
+    # Collect dependency files
+    root_file = collect_root_file(args.uv)
+    module_files = get_module_files(config, args.uv)
+
+    # Prepare installation list
+    install_paths = []
+    if root_file and not args.skip_core:
+        install_paths.append(root_file)
+    install_paths.extend(module_files.values())
+
+    if not install_paths:
+        print("No dependency files found!")
+        sys.exit(1)
+
+    # Perform installation
+    install_files(install_paths, args.uv)
+    print("Dependencies installed successfully")
\ No newline at end of file

pyproject.toml

+[project]
+name = "open-video-chat"
+version = "0.1.0"
+description = "A modular interactive digital human conversation implementation that runs full-featured on a single PC."
+readme = "README.md"
+requires-python = ">=3.11.7, <3.12"
+dependencies = [
+    "aiohttp~=3.11.12",
+    "aiortc~=1.12.0",
+    "dynaconf~=3.2.7",
+    "fastapi[standard]~=0.115.12",
+    "fastrtc",
+    "gradio~=5.9.1",
+    "librosa~=0.10.2",
+    "loguru~=0.7.3",
+    "modelscope>=1.25.0",
+    "numpy<=1.26.4",
+    "openai>=1.72.0",
+    "opencv-python-headless~=4.11.0",
+    "pip>=25.0.1",
+    "pyaml>=25.1.0",
+    "pydantic~=2.10.6",
+    "pyyaml~=6.0.2",
+    "scipy~=1.15.1",
+    "setuptools>=78.1.0",
+    "soundfile~=0.13.1",
+    "starlette~=0.41.3",
+    "tqdm~=4.67.1",
+    "typing-extensions~=4.12.2",
+    "uvicorn~=0.34.0",
+    "torch==2.8.0",
+    "torchvision",
+    "torchaudio",
+    "python-dotenv>=1.1.0",
+]
+
+[tool.uv.workspace]
+members = [
+    "src/handlers/tts/edgetts",
+    "src/handlers/tts/cosyvoice",
+    "src/handlers/llm/minicpm",
+    "src/handlers/avatar/liteavatar",
+    "src/handlers/avatar/lam",
+    "src/handlers/vad/silerovad",
+    "src/handlers/tts/bailian_tts",
+    "src/handlers/asr/sensevoice",
+    "src/handlers/avatar/musetalk",
+]
+
+[tool.uv.sources]
+fastrtc = { path = "src/third_party/gradio_webrtc_videochat/dist/fastrtc-0.0.28.dev0-py3-none-any.whl" }
+torch = [
+    { index = "pytorch-cu128" },
+]
+torchvision = [
+    { index = "pytorch-cu128" },
+]
+torchaudio = [
+    { index = "pytorch-cu128" },
+]
+
+[[tool.uv.index]]
+name = "pytorch-cu128"
+url = "https://download.pytorch.org/whl/cu128"
+explicit = true
+
+[tool.uv]
+no-build-isolation-package = ["chumpy"]
+
+[tool.uv.extra-build-dependencies]
+chumpy = ["wheel"]
\ No newline at end of file

reference/funasr_api.py

+import os
+# os.environ["CUDA_VISIBLE_DEVICES"] = "1"
+
+# 禁用 FunASR 更新检查和 HuggingFace/ModelScope 网络请求
+os.environ["HF_HUB_OFFLINE"] = "1"
+os.environ["TRANSFORMERS_OFFLINE"] = "1"
+
+import setproctitle
+setproctitle.setproctitle("asr_funasr")
+
+import tempfile
+import logging
+import torch
+from fastapi import FastAPI, UploadFile, File, Form, HTTPException
+from fastapi.responses import JSONResponse
+from funasr import AutoModel
+from typing import Optional
+import uvicorn
+
+# 配置日志格式
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger = logging.getLogger(__name__)
+
+app = FastAPI(title="FunASR API", description="语音识别接口")
+
+# 全局模型实例
+model = None
+model_with_vad = None
+def get_device():
+    """获取可用设备"""
+    if torch.cuda.is_available():
+        return "cuda:1"
+    elif torch.backends.mps.is_available():
+        return "mps"
+    return "cpu"
+
+
+def load_models():
+    """加载模型"""
+    global model, model_with_vad
+    model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+    device = get_device()
+    
+    logger.info(f"加载模型: {model_dir}, 设备: {device}")
+    
+    # 基础模型
+    model = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        remote_code="./model.py",
+        device=device,
+        disable_update=True,  # 禁用更新检查
+    )
+    
+    # 带 VAD 的模型
+    model_with_vad = AutoModel(
+        model=model_dir,
+        trust_remote_code=True,
+        vad_model="fsmn-vad",
+        vad_kwargs={"max_single_segment_time": 30000},
+        remote_code="./model.py",
+        device=device,
+        disable_update=True,  # 禁用更新检查
+    )
+
+
+@app.on_event("startup")
+async def startup_event():
+    """启动时加载模型"""
+    logger.info("服务启动中...")
+    load_models()
+    logger.info("模型加载完成，服务已就绪")
+
+
+@app.post("/transcribe")
+async def transcribe(
+    file: UploadFile = File(..., description="音频文件"),
+    language: Optional[str] = Form("中文", description="语言：中文、英文、日文等"),
+    hotwords: Optional[str] = Form(None, description="热词，逗号分隔"),
+    itn: Optional[bool] = Form(True, description="是否进行文本规整"),
+    use_vad: Optional[bool] = Form(False, description="是否使用VAD分段"),
+):
+    """
+    语音转文字接口
+    
+    - file: 音频文件 (mp3, wav 等)
+    - language: 目标语言
+    - hotwords: 热词列表，逗号分隔
+    - itn: 是否进行文本规整
+    - use_vad: 是否使用VAD进行长音频分段
+    """
+    if model is None:
+        raise HTTPException(status_code=503, detail="模型未加载")
+    
+    # 保存上传的文件到临时目录
+    suffix = os.path.splitext(file.filename)[1] or ".wav"
+    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
+        content = await file.read()
+        tmp.write(content)
+        tmp_path = tmp.name
+    
+    try:
+        # 解析热词
+        hotword_list = []
+        if hotwords:
+            hotword_list = [w.strip() for w in hotwords.split(",") if w.strip()]
+        
+        # 选择模型
+        current_model = model_with_vad if use_vad else model
+        
+        # 执行识别
+        res = current_model.generate(
+            input=[tmp_path],
+            cache={},
+            batch_size=1,
+            hotwords=hotword_list,
+            language=language,
+            itn=itn,
+        )
+        
+        text = res[0]["text"] if res else ""
+        
+        logger.info(f"识别完成 - 文件: {file.filename}, 语言: {language}, 结果长度: {len(text)}")
+        
+        return JSONResponse(content={
+            "success": True,
+            "text": text,
+            "language": language,
+        })
+    
+    except Exception as e:
+        logger.error(f"识别失败 - 文件: {file.filename}, 错误: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
+    
+    finally:
+        # 清理临时文件
+        if os.path.exists(tmp_path):
+            os.remove(tmp_path)
+
+
+@app.get("/health")
+async def health():
+    """健康检查"""
+    return {"status": "ok", "model_loaded": model is not None}
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

reference/kokoro_api.py

+"""
+Kokoro TTS HTTP 服务（完全离线，严格按官方推理流程，修复电流声）
+修改点：
+1. voice参数由客户端指定，不再限制固定值
+2. 移除voice默认值，客户端必须传递该参数
+3. 动态加载音色文件，无需硬编码音色名称
+"""
+
+import os
+import sys
+
+import setproctitle
+setproctitle.setproctitle("tts_kokoro")
+
+# ========== 强制离线 + Patch config ==========
+os.environ["HF_HUB_OFFLINE"] = "1"
+os.environ["CUDA_VISIBLE_DEVICES"] = "1"
+
+def offline_download(repo_id, filename, **kw):
+    if filename == "config.json":
+        return "/home/VoiceModel/tts/kokoro-tts-zh/config.json"
+    elif filename == "kokoro-v1_1-zh.pth":
+        return "/home/VoiceModel/tts/kokoro-tts-zh/models/kokoro-v1_1-zh.pth"
+    raise FileNotFoundError(filename)
+
+import huggingface_hub
+huggingface_hub.hf_hub_download = offline_download
+if 'huggingface_hub.file_download' in sys.modules:
+    sys.modules['huggingface_hub.file_download'].hf_hub_download = offline_download
+# ============================================
+
+import torch
+import numpy as np
+import time
+import io
+from fastapi import FastAPI, Form, HTTPException
+from fastapi.responses import StreamingResponse
+import uvicorn
+
+# 路径配置
+repo_id = 'hexgrad/Kokoro-82M-v1.1-zh'
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+model_path = '/home/VoiceModel/tts/kokoro-tts-zh/ckpts/kokoro-v1.1/kokoro-v1_1-zh.pth'
+config_path = '/home/VoiceModel/tts/kokoro-tts-zh/ckpts/kokoro-v1.1/config.json'
+voice_dir = '/home/VoiceModel/tts/kokoro-tts-zh/models/voices'
+
+# ========== 关键修改1：移除硬编码的音色加载 ==========
+# 不再提前加载固定音色，改为客户端请求时动态加载
+
+# 加载模型（严格按官方流程）
+from kokoro import KPipeline, KModel
+model = KModel(model=model_path, config=config_path, repo_id=repo_id).to(device).eval()
+zh_pipeline = KPipeline(lang_code='zh', repo_id=repo_id, model=model)
+
+# 预热 jieba 分词（解决首次请求延迟问题）
+print("🔥 Warming up jieba tokenizer...")
+try:
+    import jieba
+    jieba.initialize()
+    # 模拟一次简单的分词调用，确保词典完全加载
+    list(jieba.cut("预热分词"))
+    print("✅ Jieba warmup complete")
+except Exception as e:
+    print(f"⚠️ Jieba warmup failed: {e}")
+
+def speed_callable(len_ps):
+    speed = 1.1
+    # if len_ps <= 83:
+    #     speed = 1
+    # elif len_ps < 183:
+    #     speed = 1 - (len_ps - 83) / 500
+    return speed
+
+app = FastAPI(title="Kokoro TTS Service")
+
+def pcm_to_wav(pcm_data: bytes, sample_rate: int = 24000, channels: int = 1) -> bytes:
+    import wave
+    wav_buffer = io.BytesIO()
+    with wave.open(wav_buffer, 'wb') as wf:
+        wf.setnchannels(channels)
+        wf.setsampwidth(2)  # 16-bit
+        wf.setframerate(sample_rate)
+        wf.writeframes(pcm_data)
+    return wav_buffer.getvalue()
+
+@app.get("/")
+async def root():
+    return {
+        "service": "Kokoro TTS",
+        "status": "running",
+        "endpoints": ["/tts (POST)", "/health (GET)"],
+        "tips": f"voice参数需传递音色文件名（不含.pt），支持的音色文件位于：{voice_dir}"
+    }
+
+@app.get("/health")
+async def health():
+    return {"status": "healthy"}
+
+@app.post("/tts")
+async def text_to_speech(
+    # ========== 关键修改2：移除voice默认值，客户端必须传递 ==========
+    text: str = Form(...),
+    voice: str = Form(...)  # 移除默认值，改为必填项
+):
+    # 校验文本非空
+    if not text or not text.strip():
+        raise HTTPException(status_code=400, detail="文本不能为空")
+    # 校验voice非空
+    if not voice or not voice.strip():
+        raise HTTPException(status_code=400, detail="音色参数voice不能为空")
+    
+    try:
+        # ========== 关键修改3：动态加载客户端指定的音色文件 ==========
+        voice_file = os.path.join(voice_dir, f"{voice.strip()}.pt")
+        # 检查音色文件是否存在
+        if not os.path.exists(voice_file):
+            # 列出当前支持的音色，方便客户端排查
+            supported_voices = []
+            for f in os.listdir(voice_dir):
+                if f.endswith(".pt"):
+                    supported_voices.append(f[:-3])  # 去掉.pt后缀
+            raise HTTPException(
+                status_code=400,
+                detail=f"指定的音色文件不存在！当前支持的音色：{supported_voices}，请检查voice参数是否正确"
+            )
+        
+        # 动态加载音色张量
+        voice_tensor = torch.load(voice_file, weights_only=True)
+        
+        print(f"📝 TTS Request: text_len={len(text)}, voice={voice}")
+        print(f"大模型的回答是{text}")
+        start_time = time.time()
+        generator = zh_pipeline(text, voice=voice_tensor, speed=speed_callable)
+        result = next(generator)
+        wav = result.audio  # float32, -1~1
+
+        # 关键修复：float32 → int16，防止电流声
+        if hasattr(wav, "numpy"):
+            wav = wav.numpy()
+        wav = np.clip(wav, -1, 1)
+        wav_int16 = (wav * 32767).astype(np.int16)
+
+        wav_bytes = pcm_to_wav(wav_int16.tobytes(), sample_rate=24000, channels=1)
+        speech_len = len(wav) / 24000
+        rtf = (time.time() - start_time) / speech_len if speech_len > 0 else 0.0
+        return StreamingResponse(
+            io.BytesIO(wav_bytes),
+            media_type="audio/wav",
+            headers={
+                "Content-Disposition": f"inline; filename={voice}_output.wav",
+                "X-Speech-Length": str(speech_len),
+                "X-RTF": str(rtf)
+            }
+        )
+    except HTTPException:
+        # 重新抛出参数错误类的异常，不额外处理
+        raise
+    except Exception as e:
+        print(f"❌ TTS Error: {e}")
+        import traceback
+        traceback.print_exc()
+        raise HTTPException(status_code=500, detail=f"TTS 生成失败: {str(e)}")
+
+if __name__ == "__main__":
+    print("🚀 Starting Kokoro TTS Server...")
+    print(f"📍 支持的音色文件目录：{voice_dir}")
+    print("📍 API Docs: http://0.0.0.0:50001/docs")
+    uvicorn.run(app, host="0.0.0.0", port=50001)
\ No newline at end of file

run_docker_cuda128.sh

+CONFIG_PATH=""
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        -config | --config )
+            CONFIG_PATH="$2"
+            shift 2
+            ;;
+    esac
+done
+
+# Determine config file path
+CONFIG_FILE="${CONFIG_PATH:-config/chat_with_openai_compatible_bailian_cosyvoice.yaml}"
+
+# Check for AvatarMusetalk module and set environment variables（if Musetalk is used, set PYTORCH_JIT=0）
+if grep -q "AvatarMusetalk:" "$CONFIG_FILE" 2>/dev/null && grep -q "module: avatar/musetalk/avatar_handler_musetalk" "$CONFIG_FILE" 2>/dev/null; then
+    echo "Detected AvatarMusetalk module, adding PYTORCH_JIT=0 environment variable"
+    ENV_VARS="-e PYTORCH_JIT=0"
+else
+    echo "No AvatarMusetalk module detected in config, skipping PYTORCH_JIT environment variable"
+fi
+
+docker run --rm --gpus all -it --name open-avatar-chat \
+    --network=host \
+    $ENV_VARS \
+    -v `pwd`/build:/root/open-avatar-chat/build \
+    -v `pwd`/models:/root/open-avatar-chat/models \
+    -v `pwd`/ssl_certs:/root/open-avatar-chat/ssl_certs \
+    -v `pwd`/config:/root/open-avatar-chat/config \
+    -v `pwd`/.env:/root/open-avatar-chat/.env \
+    -v `pwd`/models/musetalk/s3fd-619a316812/:/root/.cache/torch/hub/checkpoints/ \
+    -v `pwd`/exp:/root/open-avatar-chat/exp \
+    -v `pwd`/resource:/root/open-avatar-chat/resource \
+    -e NVIDIA_VISIBLE_DEVICES=all \
+    -e NVIDIA_DRIVER_CAPABILITIES=compute,video,utility \
+    -p 8282:8282 \
+    open-avatar-chat:latest \
+    --config ${CONFIG_PATH:-config/chat_with_openai_compatible_bailian_cosyvoice.yaml}
\ No newline at end of file

scripts/compile_requirements.sh

+#!/bin/sh
+
+# Get the directory where the script is located and navigate to the project root directory
+SCRIPT_DIR=$(dirname "$0")
+ROOT_DIR=$(cd "$SCRIPT_DIR/.." && pwd)
+
+# Set the upgrade flag
+UPGRADE_FLAG=""
+if [ "$1" = "--upgrade" ]; then
+    UPGRADE_FLAG="--upgrade"
+    echo "[INFO] Upgrade mode enabled"
+fi
+
+# Process a directory
+process_directory() {
+    DIR=$1
+    if [ -f "$DIR/pyproject.toml" ]; then
+        echo "[INFO] Processing directory: ${DIR#$ROOT_DIR/}"
+        (cd "$DIR" && uv pip compile pyproject.toml -o requirements.txt $UPGRADE_FLAG)
+        echo "[INFO] Done processing directory: ${DIR#$ROOT_DIR/}"
+    else
+        echo "[WARN] pyproject.toml not found in ${DIR#$ROOT_DIR/}"
+    fi
+}
+
+# Process the root directory
+process_directory "$ROOT_DIR"
+
+# Process secondary subdirectories under src/handlers
+find "$ROOT_DIR/src/handlers" -maxdepth 2 -mindepth 2 -type d | while read -r DIR; do
+    process_directory "$DIR"
+done
+
+echo "[INFO] Dependency compilation completed"

scripts/create_ssl_certs.sh

+#!/usr/bin/env bash
+
+openssl req -newkey rsa:2048 -nodes -keyout ./ssl_certs/localhost.key -x509 -days 365 -out ./ssl_certs/localhost.crt
\ No newline at end of file

scripts/download_MiniCPM-o_2.6-int4.sh

+#!/usr/bin/env bash
+
+if [ ! -d "./models/MiniCPM-o-2_6-int4" ]; then
+  echo "Downloading MiniCPM-o 2.6 int4"
+  git lfs install
+  git clone https://www.modelscope.cn/OpenBMB/MiniCPM-o-2_6-int4.git ./models/MiniCPM-o-2_6-int4
+  echo "Download complete"
+else
+  echo "MiniCPM-o-2_6-int4 already exists"
+fi
\ No newline at end of file

scripts/download_MiniCPM-o_2.6.sh

+#!/usr/bin/env bash
+
+if [ ! -d "./models/MiniCPM-o-2_6" ]; then
+  echo "Downloading MiniCPM-o 2.6"
+  git lfs install
+  git clone https://www.modelscope.cn/OpenBMB/MiniCPM-o-2_6.git ./models/MiniCPM-o-2_6
+  echo "Download complete"
+else
+  echo "MiniCPM-o-2_6 already exists"
+fi
\ No newline at end of file

scripts/download_liteavatar_weights.sh

+LITE_AVATAR_DIR="src/handlers/avatar/liteavatar/algo/liteavatar"
+
+pushd $LITE_AVATAR_DIR
+
+bash download_model.sh
+
+popd

scripts/download_musetalk_weights.sh

+#!/bin/bash
+
+# Set root directory
+MUSETALK_ROOT_DIR="models/musetalk"
+MODEL_DIR="models"
+
+# Create all necessary subdirectories
+mkdir -p "$MODEL_DIR"
+mkdir -p "$MUSETALK_ROOT_DIR"
+mkdir -p "$MUSETALK_ROOT_DIR/musetalkV15"
+mkdir -p "$MUSETALK_ROOT_DIR/syncnet"
+mkdir -p "$MUSETALK_ROOT_DIR/dwpose"
+mkdir -p "$MUSETALK_ROOT_DIR/whisper"
+mkdir -p "$MODEL_DIR/sd-vae"
+mkdir -p "$MODEL_DIR/face-parse-bisent"
+
+
+# Install required packages
+pip install -U "huggingface_hub[cli]"
+
+# Set HuggingFace mirror (for use in mainland China)
+export HF_ENDPOINT=https://hf-mirror.com
+
+# Download MuseTalk weights (TMElyralab/MuseTalk) - download to root directory, may contain subdirectories
+echo "Downloading MuseTalk main weights to $MUSETALK_ROOT_DIR..."
+hf download TMElyralab/MuseTalk --local-dir "$MUSETALK_ROOT_DIR"
+
+
+echo "Downloading SD VAE weights to $MODEL_DIR/sd-vae..."
+hf download stabilityai/sd-vae-ft-mse --local-dir "$MODEL_DIR/sd-vae"
+
+
+echo "Downloading Whisper weights to $MUSETALK_ROOT_DIR/whisper..."
+hf download openai/whisper-tiny --local-dir "$MUSETALK_ROOT_DIR/whisper" --include "config.json" "pytorch_model.bin" "preprocessor_config.json"
+
+echo "Downloading DWPose weights to $MUSETALK_ROOT_DIR/dwpose..."
+hf download yzd-v/DWPose --local-dir "$MUSETALK_ROOT_DIR/dwpose" --include "dw-ll_ucoco_384.pth"
+
+# Download SyncNet weights to syncnet subdirectory
+echo "Downloading SyncNet weights to $MUSETALK_ROOT_DIR/syncnet..."
+hf download ByteDance/LatentSync --local-dir "$MUSETALK_ROOT_DIR/syncnet" --include "latentsync_syncnet.pt"
+
+
+echo "Downloading Face Parse Bisent weights to $MODEL_DIR/face-parse-bisent..."
+hf download ManyOtherFunctions/face-parse-bisent --local-dir "$MODEL_DIR/face-parse-bisent" --include "79999_iter.pth" "resnet18-5c106cde.pth"
+
+echo "Downloading s3fd-619a316812 weights to $MUSETALK_ROOT_DIR/s3fd-619a316812..."
+git clone https://www.modelscope.cn/HaveAnApplePie/s3fd-619a316812.git $MUSETALK_ROOT_DIR/s3fd-619a316812
+
+echo "All download commands have been executed, but the model files may not be downloaded. Please check the following directories and files exist:"
+echo "- models/musetalk/ (MuseTalk main weights)"
+echo "- models/musetalk/whisper/config.json"
+echo "- models/musetalk/whisper/pytorch_model.bin"
+echo "- models/musetalk/whisper/preprocessor_config.json"
+echo "- models/musetalk/dwpose/dw-ll_ucoco_384.pth"
+echo "- models/musetalk/syncnet/latentsync_syncnet.pt"
+echo "- models/sd-vae/ (SD VAE weights)"
+echo "- models/face-parse-bisent/79999_iter.pth"
+echo "- models/face-parse-bisent/resnet18-5c106cde.pth"
+echo "If any file is missing, please check the download logs above."
+
+echo "If files are missing, run the script again."
+

scripts/post_config_install.sh

+#!/usr/bin/env bash
+
+# Initialize variables
+CONFIG_FILE=""
+
+# Detect workspace directory based on script location
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORKSPACE_DIR="$(dirname "$SCRIPT_DIR")"
+VENV_PATH="$WORKSPACE_DIR/.venv"
+
+# Parse arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --config )
+            CONFIG_FILE="$2"
+            shift 2 # Skip current and next argument
+            ;;
+        * )
+            shift # Skip unknown arguments
+            ;;
+    esac
+done
+
+if [[ -z "$CONFIG_FILE" ]]; then
+    echo "Error: No config file specified. Please use --config parameter to specify the config file path."
+    exit 1
+fi
+
+if [[ ! -f "$CONFIG_FILE" ]]; then
+    echo "Error: Config file $CONFIG_FILE does not exist."
+    exit 1
+fi
+
+echo "Parsing config file: $CONFIG_FILE"
+
+# Check if config file contains AvatarMusetalk configuration
+if grep -q "AvatarMusetalk:" "$CONFIG_FILE"; then
+    echo "AvatarMusetalk configuration detected, starting additional configuration..."
+    
+    # 1. Modify mmcv_maximum_version in mmdet's __init__.py file
+    MMDET_INIT_FILE="$VENV_PATH/lib/python3.11/site-packages/mmdet/__init__.py"
+    
+    if [[ -f "$MMDET_INIT_FILE" ]]; then
+        echo "Modifying mmcv_maximum_version in mmdet/__init__.py..."
+        sed -i "s/mmcv_maximum_version = '[^']*'/mmcv_maximum_version = '2.2.1'/g" "$MMDET_INIT_FILE"
+        echo "mmcv_maximum_version has been updated to 2.2.1"
+    else
+        echo "Warning: $MMDET_INIT_FILE file not found"
+    fi
+    
+    echo "AvatarMusetalk additional configuration completed."
+
+    # 2. Install mmcv==2.2.0
+    echo "Installing mmcv==2.2.0..."
+    uv pip uninstall mmcv
+    uv run mim install mmcv==2.2.0 --force
+else
+    echo "No AvatarMusetalk configuration found in config file, skipping additional configuration."
+fi 

scripts/pre_config_install.sh

+#!/usr/bin/env bash
+
+# Initialize variables
+CONFIG_FILE=""
+
+# Parse arguments
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        --config )
+            CONFIG_FILE="$2"
+            shift 2 # Skip current and next argument
+            ;;
+        * )
+            shift # Skip unknown arguments
+            ;;
+    esac
+done
+
+if [[ -z "$CONFIG_FILE" ]]; then
+    echo "Error: No config file specified. Please use --config parameter to specify the config file path."
+    exit 1
+fi
+
+if [[ ! -f "$CONFIG_FILE" ]]; then
+    echo "Error: Config file $CONFIG_FILE does not exist."
+    exit 1
+fi
+
+echo "Parsing config file: $CONFIG_FILE"
+
+# Check if config file contains AvatarMusetalk configuration
+if grep -q "AvatarMusetalk:" "$CONFIG_FILE"; then
+    echo "AvatarMusetalk configuration detected, starting dependency installation..."
+    
+    # Install dependencies
+    echo "Installing setuptools and pip..."
+    uv pip install setuptools pip
+    
+    echo "Installing chumpy==0.70..."
+    uv pip install chumpy==0.70 --no-build-isolation
+    
+    echo "AvatarMusetalk dependencies installation completed."
+else
+    echo "No AvatarMusetalk configuration found in config file, skipping dependency installation."
+fi 

scripts/setup_coturn.sh

+#!/bin/bash
+
+# 检查是否以root权限运行
+if [ "$EUID" -ne 0 ]; then 
+    echo "请使用sudo运行此脚本"
+    exit 1
+fi
+
+# 安装coturn
+echo "正在安装coturn..."
+if command -v apt-get &> /dev/null; then
+    apt-get update
+    apt-get install -y coturn
+elif command -v yum &> /dev/null; then
+    yum install -y coturn
+elif command -v brew &> /dev/null; then
+    brew install coturn
+else
+    echo "未找到包管理器，请手动安装coturn"
+    exit 1
+fi
+
+# 获取内网IP地址
+PRIVATE_IP=$(ip addr | grep 'inet ' | grep -v '127.0.0.1' | grep -v 'docker' | awk '{print $2}' | cut -d/ -f1 | head -n 1)
+if [ -z "$PRIVATE_IP" ]; then
+    PRIVATE_IP=$(ifconfig | grep 'inet ' | grep -v '127.0.0.1' | grep -v 'docker' | awk '{print $2}' | head -n 1)
+fi
+
+# 获取公网IP地址
+PUBLIC_IP=$(curl -s ifconfig.me)
+if [ -z "$PUBLIC_IP" ]; then
+    PUBLIC_IP=$(curl -s ipinfo.io/ip)
+fi
+
+# 生成一个基于公网IP的realm标识符
+REALM="turn.${PUBLIC_IP//./-}.turnserver"
+
+echo "检测到内网IP地址: $PRIVATE_IP"
+echo "检测到公网IP地址: $PUBLIC_IP"
+echo "生成的realm标识符: $REALM"
+
+# 配置coturn
+echo "正在配置coturn..."
+CONFIG_FILE="/etc/turnserver.conf"
+MIN_PORT=49152
+MAX_PORT=65535
+
+# 备份原始配置文件
+if [ -f "$CONFIG_FILE" ]; then
+    cp "$CONFIG_FILE" "${CONFIG_FILE}.bak"
+fi
+
+# 写入新的配置
+cat > "$CONFIG_FILE" << EOF
+listening-port=3478
+tls-listening-port=5349
+listening-ip=0.0.0.0
+relay-ip=$PRIVATE_IP
+external-ip=$PUBLIC_IP
+min-port=$MIN_PORT
+max-port=$MAX_PORT
+verbose
+fingerprint
+lt-cred-mech
+user=username:password
+realm=$REALM
+EOF
+
+# 启用coturn服务
+echo "正在启用coturn服务..."
+if command -v systemctl &> /dev/null; then
+    systemctl enable coturn
+    systemctl restart coturn
+elif command -v service &> /dev/null; then
+    service coturn restart
+else
+    echo "无法自动重启coturn服务，请手动重启"
+fi
+
+echo "coturn安装和配置完成！"
+echo "请确保以下端口已开放："
+echo "- UDP 3478"
+echo "- TCP 3478"
+echo "- UDP 5349"
+echo "- TCP 5349"
+echo "- UDP $MIN_PORT:$MAX_PORT"

setup.cfg

+[flake8]
+max-line-length = 120
+exclude = src/avatar/algo/tts2face_cpu/*
\ No newline at end of file

src/chat_engine/chat_engine.py

+import os
+import uuid
+from typing import Optional, Dict
+
+from loguru import logger
+
+from chat_engine.common.client_handler_base import ClientHandlerBase
+from chat_engine.contexts.session_context import SessionContext
+from chat_engine.core.chat_session import ChatSession
+from chat_engine.core.handler_manager import HandlerManager
+from chat_engine.data_models.chat_engine_config_data import ChatEngineConfigModel, EngineChannelType
+from chat_engine.data_models.session_info_data import SessionInfoData, IOQueueType
+from engine_utils.directory_info import DirectoryInfo
+from dotenv import load_dotenv
+
+
+class ChatEngine(object):
+    def __init__(self):
+        self.inited = False
+        self.engine_config: Optional[ChatEngineConfigModel] = None
+        self.handler_manager: HandlerManager = HandlerManager(self)
+
+        self.sessions: Dict[str, ChatSession] = {}
+
+    def initialize(self, engine_config: ChatEngineConfigModel, app=None, ui=None, parent_block=None):
+        if self.inited:
+            return
+
+        load_dotenv()
+
+        self.engine_config = engine_config
+        if not os.path.isabs(engine_config.model_root):
+            engine_config.model_root = os.path.join(DirectoryInfo.get_project_dir(), engine_config.model_root)
+        self.handler_manager.initialize(engine_config)
+        self.handler_manager.load_handlers(engine_config, app, ui, parent_block)
+        self.inited = True
+    def _create_session(self, session_info: SessionInfoData,
+                        input_queues: Dict[EngineChannelType, IOQueueType],
+                        output_queues: Dict[EngineChannelType, IOQueueType]):
+        if not session_info.session_id:
+            session_info.session_id = str(uuid.uuid4())
+        if session_info.session_id in self.sessions:
+            raise RuntimeError(f"session {session_info.session_id} already exists")
+        session_context = SessionContext(session_info=session_info,
+                                         input_queues=input_queues,
+                                         output_queues=output_queues)
+        session = ChatSession(session_context, self.engine_config)
+        handlers = self.handler_manager.get_enabled_handler_registries()
+        for registry in handlers:
+            if isinstance(registry.handler, ClientHandlerBase):
+                # client create_context and data_sink creation of handler is not called here,
+                # they are created by its internal logic after every other handlers are ready.
+                continue
+            session.prepare_handler(registry.handler, registry.base_info, registry.handler_config)
+        self.sessions[session_info.session_id] = session
+        return session
+    def create_client_session(self, session_info: SessionInfoData, client_handler: ClientHandlerBase):
+        # TODO currently multi client in one session is not allowed.
+        if session_info.session_id in self.sessions:
+            msg = f"Session {session_info.session_id} already exists."
+            raise RuntimeError(msg)
+
+        session = self._create_session(session_info, {}, {})
+
+        registry = self.handler_manager.find_client_handler(client_handler)
+        if registry is None:
+            raise RuntimeError(f"client handler {client_handler} not found")
+
+        handler_env = session.prepare_handler(client_handler, registry.base_info, registry.handler_config)
+        return session, handler_env
+
+    def stop_session(self, session_id: str):
+        session = self.sessions.pop(session_id)
+        if session is None:
+            logger.error(f"Session {session_id} is not found.")
+            return
+        session.stop()
+    
+    def shutdown(self):
+        self.handler_manager.destroy()

src/chat_engine/common/client_handler_base.py

+import weakref
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Union, Tuple, Optional
+
+import gradio
+import numpy as np
+from fastapi import FastAPI
+from loguru import logger
+
+from chat_engine.common.engine_channel_type import EngineChannelType
+from chat_engine.common.handler_base import HandlerBase
+from chat_engine.contexts.handler_context import HandlerContext
+from chat_engine.contexts.session_context import SessionContext
+from chat_engine.data_models.chat_data.chat_data_model import ChatData
+from chat_engine.data_models.chat_signal import ChatSignal
+from chat_engine.data_models.session_info_data import SessionInfoData
+
+
+class ClientSessionDelegate(ABC):
+    @abstractmethod
+    async def get_data(self, modality: EngineChannelType, timeout: Optional[float] = 0.1) -> Optional[ChatData]:
+        pass
+
+    @abstractmethod
+    def put_data(self, modality: EngineChannelType, data: Union[np.ndarray, str],
+                 timestamp: Optional[Tuple[int, int]] = None,
+                 samplerate: Optional[int] = None, loopback: bool = False):
+        pass
+
+    @abstractmethod
+    def get_timestamp(self) -> Tuple[int, int]:
+        pass
+
+    @abstractmethod
+    def emit_signal(self, signal: ChatSignal):
+        pass
+
+    @abstractmethod
+    def clear_data(self):
+        pass
+
+
+class ClientHandlerDelegate:
+    def __init__(self, engine_ref, client_handler):
+        self.engine_ref = engine_ref
+        self.client_handler_ref = weakref.ref(client_handler)
+
+        self.session_delegates = {}
+
+    def start_session(self, session_id: str, **kwargs) -> ClientSessionDelegate:
+        logger.info(f"Starting session {session_id}")
+        engine = self.engine_ref()
+        handler = self.client_handler_ref()
+        assert engine is not None
+        assert handler is not None
+
+        kwargs["session_id"] = session_id
+        session_info = SessionInfoData.model_validate(kwargs)
+
+        session, handler_env = engine.create_client_session(session_info, handler)
+        session.start()
+        if handler_env.handler_info.client_session_delegate_class is None:
+            msg = f"Client handler {handler_env.handler_info.handler_name} does not provide a session delegate."
+            raise RuntimeError(msg)
+        session_delegate = handler_env.handler_info.client_session_delegate_class()
+        handler_env.handler.on_setup_session_delegate(session.session_context, handler_env.context, session_delegate)
+        self.session_delegates[session_id] = session_delegate
+        return session_delegate
+
+    def stop_session(self, session_id: str):
+        engine = self.engine_ref()
+        assert engine is not None
+        engine.stop_session(session_id)
+        self.session_delegates.pop(session_id)
+
+    def find_session_delegate(self, session_id: str):
+        return self.session_delegates.get(session_id)
+
+
+@dataclass
+class ClientHandlerInfo:
+    session_delegate_class: type[ClientSessionDelegate]
+
+
+class ClientHandlerBase(HandlerBase, ABC):
+    def __init__(self):
+        super().__init__()
+        self.handler_delegate = ClientHandlerDelegate(self.engine, self)
+
+    def on_before_register(self):
+        self.handler_delegate.engine_ref = self.engine
+
+    @abstractmethod
+    def on_setup_app(self, app: FastAPI, ui: gradio.blocks.Block, parent_block: Optional[gradio.blocks.Block]=None):
+        pass
+
+    @abstractmethod
+    def on_setup_session_delegate(self, session_context: SessionContext, handler_context: HandlerContext,
+                                  session_delegate: ClientSessionDelegate):
+        pass

src/chat_engine/common/engine_channel_type.py

+from enum import Enum
+
+
+class EngineChannelType(str, Enum):
+    NONE = "none"
+    TEXT = "text"
+    AUDIO = "audio"
+    VIDEO = "video"
+    EVENT = "event"
+    MOTION_DATA = "motion_data"

src/chat_engine/common/handler_base.py

+import weakref
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Optional, Dict
+
+from chat_engine.data_models.chat_data_type import ChatDataType
+from chat_engine.contexts.handler_context import HandlerContext
+from chat_engine.contexts.session_context import SessionContext
+from chat_engine.data_models.chat_data.chat_data_model import ChatData
+from chat_engine.data_models.chat_engine_config_data import ChatEngineConfigModel, HandlerBaseConfigModel
+from chat_engine.data_models.runtime_data.data_bundle import DataBundleDefinition
+
+
+class ChatDataConsumeMode(Enum):
+    ONCE = -1
+    DEFAULT = 0
+
+
+@dataclass
+class HandlerBaseInfo:
+    name: Optional[str] = None
+    config_model: Optional[type[HandlerBaseConfigModel]] = None
+    client_session_delegate_class: Optional[type] = None
+    # Handler load priority, the smaller, the higher
+    load_priority: int = 0
+
+
+@dataclass
+class HandlerDataInfo:
+    type: ChatDataType = ChatDataType.NONE
+    definition: Optional[DataBundleDefinition] = None
+    input_priority: int = 0
+    input_consume_mode: ChatDataConsumeMode = ChatDataConsumeMode.DEFAULT
+
+    def __lt__(self, other):
+        if self.input_priority == other.input_priority:
+            return self.type.value < other.type.value
+        return self.input_priority < other.input_priority
+
+
+@dataclass
+class HandlerDetail:
+    inputs: Dict[ChatDataType, HandlerDataInfo] = field(default_factory=dict)
+    outputs: Dict[ChatDataType, HandlerDataInfo] = field(default_factory=dict)
+
+
+
+class HandlerBase(ABC):
+    def __init__(self):
+        self.engine: Optional[weakref.ReferenceType] = None
+        self.handler_root: Optional[str] = None
+
+    def on_before_register(self):
+        pass
+
+    @abstractmethod
+    def get_handler_info(self) -> HandlerBaseInfo:
+        pass
+
+    @abstractmethod
+    def load(self, engine_config: ChatEngineConfigModel, handler_config: Optional[HandlerBaseConfigModel] = None):
+        pass
+
+    @abstractmethod
+    def create_context(self, session_context: SessionContext,
+                       handler_config: Optional[HandlerBaseConfigModel] = None) -> HandlerContext:
+        pass
+
+    @abstractmethod
+    def start_context(self, session_context: SessionContext, handler_context: HandlerContext):
+        pass
+
+    @abstractmethod
+    def get_handler_detail(self, session_context: SessionContext,
+                           context: HandlerContext) -> HandlerDetail:
+        pass
+
+    @abstractmethod
+    def handle(self, context: HandlerContext, inputs: ChatData,
+                     output_definitions: Dict[ChatDataType, HandlerDataInfo]):
+        pass
+
+    @abstractmethod
+    def destroy_context(self, context: HandlerContext):
+        pass
+
+    def destroy(self):
+        pass

src/chat_engine/contexts/handler_context.py

+from typing import Tuple, Union
+from loguru import logger
+import numpy as np
+
+from chat_engine.data_models.chat_data_type import ChatDataType
+from chat_engine.data_models.chat_data.chat_data_model import ChatData
+from chat_engine.data_models.runtime_data.data_bundle import DataBundle
+
+HandlerResultType = Union[
+    ChatData,
+    DataBundle,
+    Tuple[ChatDataType, np.ndarray]
+]
+
+
+class HandlerContext(object):
+    def __init__(self, session_id: str):
+        self.session_id = session_id
+        self.owner = None
+        self.data_submitter = None
+
+    def submit_data(self, data: HandlerResultType):
+        if self.data_submitter is None:
+            logger.error(f"Session is not started, data submitter not ready. owner={self.owner}, session_id={self.session_id}")
+            return
+        
+        self.data_submitter.submit(data)

src/chat_engine/contexts/session_context.py

+import time
+from dataclasses import dataclass
+from typing import Dict, List, Tuple
+
+from loguru import logger
+
+from chat_engine.data_models.chat_engine_config_data import EngineChannelType
+from chat_engine.data_models.runtime_data.data_bundle import DataBundleDefinition, DataBundleEntry
+from chat_engine.data_models.session_info_data import SessionInfoData, IOQueueType
+
+
+@dataclass
+class SharedStates:
+    active: bool = False
+    enable_vad: bool = True
+    handler_config: dict = None  # 前端传来的handler配置
+
+
+class SessionContext(object):
+    def __init__(self, session_info: SessionInfoData,
+                 input_queues: Dict[EngineChannelType, IOQueueType],
+                 output_queues: Dict[EngineChannelType, IOQueueType]):
+        self.session_info = session_info
+        self.input_queues = input_queues
+        self.output_queues = output_queues
+        self.shared_states = SharedStates()
+        self.input_definitions: Dict[EngineChannelType, DataBundleDefinition] = {}
+        self.input_start_time: float = -1.0
+
+    def get_input_audio_definition(self, sample_rate: int, channel_num: int = 1, entry_name: str = "mic_audio"):
+        definition = self.input_definitions.get(EngineChannelType.AUDIO, None)
+        if definition is None:
+            definition = DataBundleDefinition()
+            definition.add_entry(DataBundleEntry.create_audio_entry(entry_name, channel_num, sample_rate))
+            definition.lockdown()
+            self.input_definitions[EngineChannelType.AUDIO] = definition
+        return definition
+
+    def get_input_video_definition(self, frame_shape: List[int], frame_rate: int, entry_name: str = "camera_video",
+                                   allow_shape_change: bool = False):
+        data_shape = [1] + frame_shape
+        definition = self.input_definitions.get(EngineChannelType.VIDEO, None)
+        if allow_shape_change and definition is not None:
+            defined_shape = list(definition.get_main_entry().shape)
+            if defined_shape != data_shape:
+                logger.warning(f"Video definition updated, orign shape={defined_shape}, new shape={data_shape}")
+                definition = None
+        if definition is None:
+            definition = DataBundleDefinition()
+            definition.add_entry(DataBundleEntry.create_framed_entry(entry_name, data_shape, 0, frame_rate))
+            definition.lockdown()
+            self.input_definitions[EngineChannelType.VIDEO] = definition
+        return definition
+    
+    def get_input_text_definition(self, entry_name: str = "human_text"):
+        definition = self.input_definitions.get(EngineChannelType.TEXT, None)
+        if definition is None:
+            definition = DataBundleDefinition()
+            definition.add_entry(DataBundleEntry.create_text_entry(entry_name))
+            definition.lockdown()
+            self.input_definitions[EngineChannelType.TEXT] = definition
+        return definition
+
+    def cleanup(self):
+        for data_queue in self.input_queues.values():
+            while not data_queue.empty():
+                data_queue.get_nowait()
+        for data_queue in self.output_queues.values():
+            while not data_queue.empty():
+                data_queue.get_nowait()
+
+    def set_input_start(self):
+        if self.input_start_time < 0:
+            now = time.monotonic()
+            self.input_start_time = now
+
+    def get_timestamp(self) -> Tuple[int, int]:
+        now = time.monotonic()
+        if self.input_start_time < 0:
+            return -1, 0
+        else:
+            return (
+                round((now - self.input_start_time) * self.session_info.timestamp_base),
+                self.session_info.timestamp_base
+            )

src/chat_engine/data_models/chat_data/chat_data_model.py

+from dataclasses import dataclass
+from typing import Tuple, Optional
+
+from chat_engine.data_models.chat_data_type import ChatDataType
+from chat_engine.data_models.runtime_data.data_bundle import DataBundle
+
+
+@dataclass
+class ChatData:
+    source: Optional[str] = None
+    type: ChatDataType = ChatDataType.NONE
+    timestamp: Tuple[int, int] = (0, 0)
+    data: Optional[DataBundle] = None
+
+    def is_timestamp_valid(self) -> bool:
+        return self.timestamp[0] >= 0 and self.timestamp[1] > 0

src/chat_engine/data_models/chat_data_type.py

+from enum import Enum
+
+from chat_engine.common.engine_channel_type import EngineChannelType
+
+
+class ChatDataType(Enum):
+
+    def __init__(self, value: str, channel_type: EngineChannelType):
+        self._value_ = value
+        self.channel_type = channel_type
+
+    NONE = ("none", EngineChannelType.NONE)
+    HUMAN_TEXT = ("human_text", EngineChannelType.TEXT)
+    AVATAR_TEXT = ("avatar_text", EngineChannelType.TEXT)
+    HUMAN_VOICE_ACTIVITY = ("human_vad", EngineChannelType.EVENT)
+    MIC_AUDIO = ("mic_audio", EngineChannelType.AUDIO)
+    HUMAN_AUDIO = ("human_audio", EngineChannelType.AUDIO)
+    AVATAR_AUDIO = ("avatar_audio", EngineChannelType.AUDIO)
+    CAMERA_VIDEO = ("camera_video", EngineChannelType.VIDEO)
+    AVATAR_VIDEO = ("avatar_video", EngineChannelType.VIDEO)
+    AVATAR_MOTION_DATA = ("avatar_motion_data", EngineChannelType.MOTION_DATA)

src/chat_engine/data_models/chat_engine_config_data.py

+from typing import Dict, Optional, List, Union
+
+from pydantic import BaseModel, Field
+
+from chat_engine.data_models.chat_data_type import ChatDataType
+from chat_engine.common.engine_channel_type import EngineChannelType
+
+
+class HandlerBaseConfigModel(BaseModel):
+    enabled: bool = Field(default=True)
+    module: Optional[str] = Field(default=None)
+    concurrent_limit: int = Field(default=1)
+
+
+class ChatEngineOutputSource(BaseModel):
+    handler: Optional[Union[str, List[str]]]
+    type: ChatDataType
+
+
+class ChatEngineConfigModel(BaseModel):
+    model_root: str = ""
+    concurrent_limit: int = Field(default=1)
+    handler_search_path: List[str] = Field(default_factory=list)
+    handler_configs: Optional[Dict[str, Dict]] = None
+    outputs: Dict[EngineChannelType, ChatEngineOutputSource] = Field(default_factory=dict)
+    turn_config: Optional[Dict] = Field(default=None)

src/chat_engine/data_models/chat_signal.py

+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from chat_engine.data_models.chat_data_type import ChatDataType
+from chat_engine.data_models.chat_signal_type import ChatSignalType, ChatSignalSourceType
+
+
+class ChatSignal(BaseModel):
+    type: Optional[ChatSignalType] = Field(default=None)
+    stream_type: Optional[ChatDataType] = Field(default=None)
+    source_type: Optional[ChatSignalSourceType] = Field(default=None)
+    source_name: str = Field(default="")

src/chat_engine/data_models/chat_signal_type.py

+from enum import Enum
+
+
+class ChatSignalType(str, Enum):
+    # START = "start"
+    BEGIN = "begin"
+    END = "end"
+    # CANCEL = "cancel"
+    INTERRUPT = "interrupt"
+    # RESET = "reset"
+    # ERROR = "error"
+    # STOP = "stop"
+
+
+class ChatSignalSourceType(str, Enum):
+    CLIENT = "client"
+    # LOGIC = "logic"
+    # HANDLER = "handler"
+    # ENGINE = "engine"

src/config/__init__.py

+"""配置模块"""
+from .agent_config import AgentConfig, validate_config, get_default_config
+
+__all__ = ['AgentConfig', 'validate_config', 'get_default_config']