Files
zyt/training-videos/scripts/coach_vision.py
T
long 3cd3553af2 feat: 新增 training-videos 素材流水线
AI 跟练的素材生产工具,产物写入 TUICallKit-Vue3/training:
- 教练动作帧:AI 设定稿 → 切帧对齐、锁脸、插帧 → 导出小程序帧与 COS 高清帧
- 示范视频:Remotion 渲染四个动作的示范视频
- 背景音乐:程序合成的原创循环音乐(npm run bgm)
2026-10-31 10:33:25 +08:00

103 lines
3.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
教练帧处理用到的两个模型
- 关键点:torchvision Keypoint R-CNN(COCO 17 点),用来对齐、统一比例、测量每帧的动作幅度
- 超分:Real-ESRGAN x4plus(经 spandrel 加载),把设定稿里 300~450px 高的人物放大后再缩到主帧尺寸,
边缘和五官比直接插值放大清晰得多
模型权重首次使用时自动下载:Keypoint R-CNN 由 torchvision 缓存到 TORCH_HOME,
Real-ESRGAN 下载到 training-videos/work/models。
"""
import math
import urllib.request
from pathlib import Path
import cv2
import numpy as np
import torch
import torchvision
ROOT = Path(__file__).resolve().parents[1]
MODEL_DIR = ROOT / "work" / "models"
SR_URL = "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth"
KP_NAMES = ["nose", "leye", "reye", "lear", "rear", "lsh", "rsh", "lel", "rel", "lwr", "rwr", "lhip", "rhip", "lkn", "rkn", "lan", "ran"]
BG = (240, 244, 248)
_kp_model = None
_sr_model = None
def _device():
return torch.device("mps" if torch.backends.mps.is_available() else "cpu")
def on_background(rgba: np.ndarray) -> np.ndarray:
a = rgba[..., 3:4].astype(np.float32) / 255
return (rgba[..., :3] * a + np.array(BG, np.float32) * (1 - a)).round().astype(np.uint8)
def keypoints(rgba: np.ndarray) -> dict:
"""人物关键点 {名称: (x, y, 置信度)};被拍者的左侧(l*)在画面右侧"""
global _kp_model
if _kp_model is None:
_kp_model = torchvision.models.detection.keypointrcnn_resnet50_fpn(weights="DEFAULT").eval()
x = torch.from_numpy(on_background(rgba)).permute(2, 0, 1).float() / 255
with torch.no_grad():
out = _kp_model([x])[0]
if len(out["scores"]) == 0:
raise ValueError("没有检测到人物")
i = int(out["scores"].argmax())
k, s = out["keypoints"][i].numpy(), out["keypoints_scores"][i].numpy()
return {n: (float(k[j, 0]), float(k[j, 1]), float(s[j])) for j, n in enumerate(KP_NAMES)}
def arm_angle(kp: dict, side: str) -> float:
"""手臂外展角:0 = 竖直向下,90 = 水平,180 = 竖直向上"""
sh, wr = kp[f"{side}sh"], kp[f"{side}wr"]
return math.degrees(math.atan2(abs(wr[0] - sh[0]), wr[1] - sh[1]))
def shoulder_center(kp: dict):
return (kp["lsh"][0] + kp["rsh"][0]) / 2, (kp["lsh"][1] + kp["rsh"][1]) / 2
def _sr():
global _sr_model
if _sr_model is None:
from spandrel import ModelLoader
path = MODEL_DIR / Path(SR_URL).name
if not path.exists():
MODEL_DIR.mkdir(parents=True, exist_ok=True)
print(f"下载 {path.name} …")
urllib.request.urlretrieve(SR_URL, path)
_sr_model = ModelLoader().load_from_file(str(path)).model.eval().to(_device())
return _sr_model
def _run_sr(rgb: np.ndarray) -> np.ndarray:
x = torch.from_numpy(rgb.astype(np.float32) / 255).permute(2, 0, 1)[None].to(_device())
with torch.no_grad():
y = _sr()(x)
return (y[0].permute(1, 2, 0).clamp(0, 1).cpu().numpy() * 255).round().astype(np.uint8)
def _bleed(rgba: np.ndarray) -> np.ndarray:
"""把前景颜色向透明区域外扩,放大时边缘不会混进背景色(黑边 / 灰边)"""
hole = (rgba[..., 3] < 16).astype(np.uint8)
rgb = rgba[..., :3].copy()
small = cv2.resize(rgb, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
small_hole = cv2.resize(hole, (small.shape[1], small.shape[0]), interpolation=cv2.INTER_NEAREST)
filled = cv2.resize(cv2.inpaint(small, small_hole, 3, cv2.INPAINT_TELEA), (rgb.shape[1], rgb.shape[0]))
rgb[hole > 0] = filled[hole > 0]
return rgb
def upscale4(rgba: np.ndarray) -> np.ndarray:
"""透明图 4 倍超分:颜色与 alpha 分别放大"""
rgb = _run_sr(_bleed(rgba))
a = rgba[..., 3]
alpha = _run_sr(np.dstack([a, a, a]))[..., 0]
return np.dstack([rgb, alpha])