AI 跟练的素材生产工具,产物写入 TUICallKit-Vue3/training: - 教练动作帧:AI 设定稿 → 切帧对齐、锁脸、插帧 → 导出小程序帧与 COS 高清帧 - 示范视频:Remotion 渲染四个动作的示范视频 - 背景音乐:程序合成的原创循环音乐(npm run bgm)
103 lines
3.8 KiB
Python
103 lines
3.8 KiB
Python
"""
|
||
教练帧处理用到的两个模型
|
||
|
||
- 关键点:torchvision Keypoint R-CNN(COCO 17 点),用来对齐、统一比例、测量每帧的动作幅度
|
||
- 超分:Real-ESRGAN x4plus(经 spandrel 加载),把设定稿里 300~450px 高的人物放大后再缩到主帧尺寸,
|
||
边缘和五官比直接插值放大清晰得多
|
||
|
||
模型权重首次使用时自动下载:Keypoint R-CNN 由 torchvision 缓存到 TORCH_HOME,
|
||
Real-ESRGAN 下载到 training-videos/work/models。
|
||
"""
|
||
import math
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
import cv2
|
||
import numpy as np
|
||
import torch
|
||
import torchvision
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
MODEL_DIR = ROOT / "work" / "models"
|
||
SR_URL = "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth"
|
||
|
||
KP_NAMES = ["nose", "leye", "reye", "lear", "rear", "lsh", "rsh", "lel", "rel", "lwr", "rwr", "lhip", "rhip", "lkn", "rkn", "lan", "ran"]
|
||
BG = (240, 244, 248)
|
||
|
||
_kp_model = None
|
||
_sr_model = None
|
||
|
||
|
||
def _device():
|
||
return torch.device("mps" if torch.backends.mps.is_available() else "cpu")
|
||
|
||
|
||
def on_background(rgba: np.ndarray) -> np.ndarray:
|
||
a = rgba[..., 3:4].astype(np.float32) / 255
|
||
return (rgba[..., :3] * a + np.array(BG, np.float32) * (1 - a)).round().astype(np.uint8)
|
||
|
||
|
||
def keypoints(rgba: np.ndarray) -> dict:
|
||
"""人物关键点 {名称: (x, y, 置信度)};被拍者的左侧(l*)在画面右侧"""
|
||
global _kp_model
|
||
if _kp_model is None:
|
||
_kp_model = torchvision.models.detection.keypointrcnn_resnet50_fpn(weights="DEFAULT").eval()
|
||
x = torch.from_numpy(on_background(rgba)).permute(2, 0, 1).float() / 255
|
||
with torch.no_grad():
|
||
out = _kp_model([x])[0]
|
||
if len(out["scores"]) == 0:
|
||
raise ValueError("没有检测到人物")
|
||
i = int(out["scores"].argmax())
|
||
k, s = out["keypoints"][i].numpy(), out["keypoints_scores"][i].numpy()
|
||
return {n: (float(k[j, 0]), float(k[j, 1]), float(s[j])) for j, n in enumerate(KP_NAMES)}
|
||
|
||
|
||
def arm_angle(kp: dict, side: str) -> float:
|
||
"""手臂外展角:0 = 竖直向下,90 = 水平,180 = 竖直向上"""
|
||
sh, wr = kp[f"{side}sh"], kp[f"{side}wr"]
|
||
return math.degrees(math.atan2(abs(wr[0] - sh[0]), wr[1] - sh[1]))
|
||
|
||
|
||
def shoulder_center(kp: dict):
|
||
return (kp["lsh"][0] + kp["rsh"][0]) / 2, (kp["lsh"][1] + kp["rsh"][1]) / 2
|
||
|
||
|
||
def _sr():
|
||
global _sr_model
|
||
if _sr_model is None:
|
||
from spandrel import ModelLoader
|
||
|
||
path = MODEL_DIR / Path(SR_URL).name
|
||
if not path.exists():
|
||
MODEL_DIR.mkdir(parents=True, exist_ok=True)
|
||
print(f"下载 {path.name} …")
|
||
urllib.request.urlretrieve(SR_URL, path)
|
||
_sr_model = ModelLoader().load_from_file(str(path)).model.eval().to(_device())
|
||
return _sr_model
|
||
|
||
|
||
def _run_sr(rgb: np.ndarray) -> np.ndarray:
|
||
x = torch.from_numpy(rgb.astype(np.float32) / 255).permute(2, 0, 1)[None].to(_device())
|
||
with torch.no_grad():
|
||
y = _sr()(x)
|
||
return (y[0].permute(1, 2, 0).clamp(0, 1).cpu().numpy() * 255).round().astype(np.uint8)
|
||
|
||
|
||
def _bleed(rgba: np.ndarray) -> np.ndarray:
|
||
"""把前景颜色向透明区域外扩,放大时边缘不会混进背景色(黑边 / 灰边)"""
|
||
hole = (rgba[..., 3] < 16).astype(np.uint8)
|
||
rgb = rgba[..., :3].copy()
|
||
small = cv2.resize(rgb, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA)
|
||
small_hole = cv2.resize(hole, (small.shape[1], small.shape[0]), interpolation=cv2.INTER_NEAREST)
|
||
filled = cv2.resize(cv2.inpaint(small, small_hole, 3, cv2.INPAINT_TELEA), (rgb.shape[1], rgb.shape[0]))
|
||
rgb[hole > 0] = filled[hole > 0]
|
||
return rgb
|
||
|
||
|
||
def upscale4(rgba: np.ndarray) -> np.ndarray:
|
||
"""透明图 4 倍超分:颜色与 alpha 分别放大"""
|
||
rgb = _run_sr(_bleed(rgba))
|
||
a = rgba[..., 3]
|
||
alpha = _run_sr(np.dstack([a, a, a]))[..., 0]
|
||
return np.dstack([rgb, alpha])
|