更新

2026-08-03 10:00:05 +08:00
parent 0fb03d0bca
commit 01729b1e0b
109 changed files with 7577 additions and 1153 deletions
@@ -0,0 +1,86 @@
+# CosyVoice 客服语音部署
+
+当前聊天应用已经内置 CosyVoice FastAPI 适配器。语音对话使用 SSE 持续转发 PCM16，浏览器收到第一段音频后立即播放；兼容接口仍可把 PCM16 封装为 WAV。CosyVoice 异常时会熔断 20 秒，并自动回落到 OpenAI/浏览器语音。
+
+## 当前部署（2026-07-23）
+
+- 服务地址：`http://192.168.110.111:50000`（仅绑定局域网地址）
+- 容器：`cosyvoice3`，重启策略 `unless-stopped`
+- 镜像：`local/cosyvoice3:cu128-stream4-cancel`
+- 模型：`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`
+- GPU：第 4 张 NVIDIA RTX 6000D（Docker 设备编号 `3`）
+- 服务目录：`/home/ps/services/cosyvoice3`
+- 输出格式：24 kHz、单声道、PCM16；语音对话由 PHP 以 SSE 原样转发并通过 Web Audio 边收边播
+- 打断：每次合成带唯一 `request_id`；浏览器插话后会停止已排期音频、终止文本请求，并调用 GPU 服务的 `/cancel/{request_id}` 停止后续生成
+- 回滚容器：`cosyvoice3-stream2-rollback-20260723`（上一版流式服务）和 `cosyvoice3-rollback-20260723`（最初稳定版），均保持停止状态
+
+本机端到端实测：SSE 响应头约 0.09 秒返回，短句第一段可播放音频约 2.1–2.6 秒到达。长文本收到第一段音频后执行取消，GPU 在约 0.8 秒内关闭生成流，未继续生成后续段落。数据库已启用会员端语音开关，并配置为 `zero_shot` 模式。当前使用官方仓库参考音频作为临时演示音色，正式上线前应在管理后台替换为已获得授权的真人客服 WAV 及其完全一致的逐字稿。
+
+## 1. GPU 主机部署官方服务
+
+建议使用 NVIDIA GPU 和 Linux/Docker。按照官方仓库构建：
+
+```bash
+git clone --recursive https://github.com/FunAudioLLM/CosyVoice.git
+cd CosyVoice/runtime/python
+docker build -t cosyvoice:v1.0 .
+```
+
+快速使用内置中文女声（SFT）：
+
+```bash
+docker run -d --gpus all --restart unless-stopped \
+  -p 50000:50000 cosyvoice:v1.0 \
+  /bin/bash -lc "cd /opt/CosyVoice/CosyVoice/runtime/python/fastapi && python3 server.py --port 50000 --model_dir iic/CosyVoice-300M-SFT"
+```
+
+生产环境请只允许聊天后端访问 50000 端口，或在反向代理中设置 Bearer Token；不要把官方无鉴权 FastAPI 直接暴露到公网。
+
+## 2. 在管理后台配置 AI 客服人物
+
+登录管理后台，进入「系统设置 → AI 客服人物」，填写 CosyVoice 服务地址，并选择人物模板、合成模式、说话人和采样率。点击「保存并试听音色」可以立即验证服务。
+
+后台保存的设置优先级高于环境变量；环境变量用于首次启动时提供默认值：
+
+在 `backend/.env` 中加入：
+
+```dotenv
+COSYVOICE_ENABLED=true
+COSYVOICE_BASE_URL=http://GPU服务器内网地址:50000
+COSYVOICE_MODE=sft
+COSYVOICE_SPEAKER=中文女
+COSYVOICE_SAMPLE_RATE=22050
+COSYVOICE_CONNECT_TIMEOUT_MS=800
+COSYVOICE_TIMEOUT_SECONDS=8
+```
+
+只使用管理后台修改时无需重启 PHP 服务。
+
+## 3. 最佳质量：CosyVoice 3 零样本客服音色
+
+使用一段已获得说话人明确授权、干净无背景音乐的客服录音，并准备完全一致的逐字稿。不要克隆未授权的真人声音。
+
+GPU 服务改用 `FunAudioLLM/Fun-CosyVoice3-0.5B-2512`。在管理后台将模式切换为 `Zero-shot 克隆音色`，上传 WAV 并填写与录音完全一致的逐字稿；音色文件会安全保存到 `backend/storage/cosyvoice/`。
+
+也可以在首次启动前通过环境变量提供默认值：
+
+```dotenv
+COSYVOICE_MODE=zero_shot
+COSYVOICE_PROMPT_WAV=D:/web/chat/backend/storage/cosyvoice/customer-service.wav
+COSYVOICE_PROMPT_TEXT=You are a helpful assistant.<|endofprompt|>这里填写参考音频的完整逐字稿。
+COSYVOICE_SAMPLE_RATE=24000
+```
+
+音色样本由聊天后端通过 multipart 请求发送给 CosyVoice，因此 GPU 服务和 PHP 后端可以位于不同主机。
+
+## 4. 情绪与语速控制
+
+使用 Instruct 模型时：
+
+```dotenv
+COSYVOICE_MODE=instruct
+COSYVOICE_SPEAKER=中文女
+COSYVOICE_INSTRUCT=请用温暖、自然、耐心的中文客服语气表达，语速适中，停顿真实，避免播音腔和夸张情绪。
+```
+
+如果服务返回错误或超时，页面会显示设备语音兜底状态，不会中断文字客服对话。
@@ -0,0 +1,2 @@
+FROM local/cosyvoice3:cu128-api4
+COPY server_cosyvoice3.py /opt/CosyVoice/CosyVoice/runtime/python/fastapi/server_cosyvoice3.py
@@ -0,0 +1,238 @@
+import argparse
+import hashlib
+import logging
+import os
+import shutil
+import sys
+import tempfile
+import threading
+
+import numpy as np
+import uvicorn
+from fastapi import FastAPI, File, Form, UploadFile
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import StreamingResponse
+
+logging.getLogger("matplotlib").setLevel(logging.WARNING)
+ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.join(ROOT_DIR, "../../.."))
+sys.path.append(os.path.join(ROOT_DIR, "../../../third_party/Matcha-TTS"))
+
+from cosyvoice.cli.cosyvoice import AutoModel
+
+app = FastAPI(title="CosyVoice 3 streaming API")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+cosyvoice = None
+model_name = ""
+fp16_enabled = False
+speaker_cache_lock = threading.Lock()
+cancel_events = {}
+cancel_events_lock = threading.Lock()
+
+
+def persist_upload(upload: UploadFile) -> str:
+    suffix = os.path.splitext(upload.filename or "")[1] or ".wav"
+    with tempfile.NamedTemporaryFile(prefix="cosyvoice-prompt-", suffix=suffix, delete=False) as target:
+        upload.file.seek(0)
+        shutil.copyfileobj(upload.file, target)
+        return target.name
+
+
+def speaker_cache_id(prompt_text: str, prompt_wav: str) -> str:
+    digest = hashlib.sha256(prompt_text.encode("utf-8"))
+    with open(prompt_wav, "rb") as source:
+        for chunk in iter(lambda: source.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return "voice-" + digest.hexdigest()[:24]
+
+
+def ensure_cached_speaker(prompt_text: str, prompt_wav: str) -> str:
+    cache_id = speaker_cache_id(prompt_text, prompt_wav)
+    with speaker_cache_lock:
+        if cache_id not in cosyvoice.frontend.spk2info:
+            cosyvoice.add_zero_shot_spk(prompt_text, prompt_wav, cache_id)
+            logging.info("cached zero-shot speaker %s", cache_id)
+    return cache_id
+
+
+def register_cancel_event(request_id: str):
+    request_id = request_id.strip()
+    if not request_id:
+        return None
+
+    event = threading.Event()
+    with cancel_events_lock:
+        previous = cancel_events.get(request_id)
+        if previous is not None:
+            previous.set()
+        cancel_events[request_id] = event
+    return event
+
+
+def release_cancel_event(request_id: str, event):
+    if not request_id or event is None:
+        return
+    with cancel_events_lock:
+        if cancel_events.get(request_id) is event:
+            cancel_events.pop(request_id, None)
+
+
+def pcm_stream(model_output, cleanup_path: str = "", request_id: str = "", cancel_event=None):
+    iterator = iter(model_output)
+    try:
+        while True:
+            if cancel_event is not None and cancel_event.is_set():
+                break
+            try:
+                item = next(iterator)
+            except StopIteration:
+                break
+            if cancel_event is not None and cancel_event.is_set():
+                break
+            audio = item["tts_speech"].detach().cpu().numpy()
+            yield (audio * (2**15)).astype(np.int16).tobytes()
+    finally:
+        close = getattr(iterator, "close", None)
+        if callable(close):
+            close()
+        release_cancel_event(request_id, cancel_event)
+        if cleanup_path:
+            try:
+                os.remove(cleanup_path)
+            except FileNotFoundError:
+                pass
+
+
+def response(model_output, cleanup_path: str = "", request_id: str = ""):
+    request_id = request_id.strip()[:128]
+    cancel_event = register_cancel_event(request_id)
+    return StreamingResponse(
+        pcm_stream(model_output, cleanup_path, request_id, cancel_event),
+        media_type="application/octet-stream",
+        headers={
+            "X-Sample-Rate": str(cosyvoice.sample_rate),
+            "X-Audio-Format": "pcm_s16le",
+            "X-Accel-Buffering": "no",
+            "Cache-Control": "no-store, no-transform",
+        },
+    )
+
+
+@app.get("/health")
+def health():
+    return {
+        "status": "ok",
+        "model": model_name,
+        "sample_rate": cosyvoice.sample_rate,
+        "streaming": True,
+        "fp16": fp16_enabled,
+        "cached_speakers": len(cosyvoice.frontend.spk2info),
+        "active_streams": len(cancel_events),
+    }
+
+
+@app.post("/cancel/{request_id}")
+def cancel(request_id: str):
+    with cancel_events_lock:
+        event = cancel_events.get(request_id)
+        if event is not None:
+            event.set()
+    return {"cancelled": event is not None, "request_id": request_id}
+
+
+@app.get("/inference_sft")
+@app.post("/inference_sft")
+def inference_sft(tts_text: str = Form(), spk_id: str = Form(), request_id: str = Form("")):
+    return response(cosyvoice.inference_sft(tts_text, spk_id, stream=True), request_id=request_id)
+
+
+@app.get("/inference_zero_shot")
+@app.post("/inference_zero_shot")
+def inference_zero_shot(
+    tts_text: str = Form(),
+    prompt_text: str = Form(),
+    prompt_wav: UploadFile = File(),
+    request_id: str = Form(""),
+):
+    prompt_path = persist_upload(prompt_wav)
+    cache_id = ensure_cached_speaker(prompt_text, prompt_path)
+    return response(
+        cosyvoice.inference_zero_shot(
+            tts_text,
+            prompt_text,
+            prompt_path,
+            zero_shot_spk_id=cache_id,
+            stream=True,
+        ),
+        prompt_path,
+        request_id,
+    )
+
+
+@app.get("/inference_cross_lingual")
+@app.post("/inference_cross_lingual")
+def inference_cross_lingual(
+    tts_text: str = Form(),
+    prompt_wav: UploadFile = File(),
+    request_id: str = Form(""),
+):
+    prompt_path = persist_upload(prompt_wav)
+    return response(
+        cosyvoice.inference_cross_lingual(tts_text, prompt_path, stream=True),
+        prompt_path,
+        request_id,
+    )
+
+
+@app.get("/inference_instruct")
+@app.post("/inference_instruct")
+def inference_instruct(
+    tts_text: str = Form(),
+    spk_id: str = Form(),
+    instruct_text: str = Form(),
+    request_id: str = Form(""),
+):
+    return response(
+        cosyvoice.inference_instruct(tts_text, spk_id, instruct_text, stream=True),
+        request_id=request_id,
+    )
+
+
+@app.get("/inference_instruct2")
+@app.post("/inference_instruct2")
+def inference_instruct2(
+    tts_text: str = Form(),
+    instruct_text: str = Form(),
+    prompt_wav: UploadFile = File(),
+    request_id: str = Form(""),
+):
+    prompt_path = persist_upload(prompt_wav)
+    return response(
+        cosyvoice.inference_instruct2(tts_text, instruct_text, prompt_path, stream=True),
+        prompt_path,
+        request_id,
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--port", type=int, default=50000)
+    parser.add_argument("--model_dir", type=str, default="FunAudioLLM/Fun-CosyVoice3-0.5B-2512")
+    parser.add_argument("--fp16", action="store_true", help="Run the PyTorch model in FP16 on CUDA")
+    args = parser.parse_args()
+    model_name = args.model_dir
+    fp16_enabled = bool(args.fp16)
+    cosyvoice = AutoModel(model_dir=args.model_dir, fp16=fp16_enabled)
+
+    default_prompt_wav = os.path.join(ROOT_DIR, "../../../asset/zero_shot_prompt.wav")
+    default_prompt_text = "You are a helpful assistant.<|endofprompt|>希望你以后能够做的比我还好呦。"
+    if os.path.isfile(default_prompt_wav):
+        ensure_cached_speaker(default_prompt_text, default_prompt_wav)
+
+    uvicorn.run(app, host="0.0.0.0", port=args.port)