""" 教练帧处理用到的两个模型 - 关键点:torchvision Keypoint R-CNN(COCO 17 点),用来对齐、统一比例、测量每帧的动作幅度 - 超分:Real-ESRGAN x4plus(经 spandrel 加载),把设定稿里 300~450px 高的人物放大后再缩到主帧尺寸, 边缘和五官比直接插值放大清晰得多 模型权重首次使用时自动下载:Keypoint R-CNN 由 torchvision 缓存到 TORCH_HOME, Real-ESRGAN 下载到 training-videos/work/models。 """ import math import urllib.request from pathlib import Path import cv2 import numpy as np import torch import torchvision ROOT = Path(__file__).resolve().parents[1] MODEL_DIR = ROOT / "work" / "models" SR_URL = "https://github.com/xinntao/Real-ESRGAN/releases/download/v0.1.0/RealESRGAN_x4plus.pth" KP_NAMES = ["nose", "leye", "reye", "lear", "rear", "lsh", "rsh", "lel", "rel", "lwr", "rwr", "lhip", "rhip", "lkn", "rkn", "lan", "ran"] BG = (240, 244, 248) _kp_model = None _sr_model = None def _device(): return torch.device("mps" if torch.backends.mps.is_available() else "cpu") def on_background(rgba: np.ndarray) -> np.ndarray: a = rgba[..., 3:4].astype(np.float32) / 255 return (rgba[..., :3] * a + np.array(BG, np.float32) * (1 - a)).round().astype(np.uint8) def keypoints(rgba: np.ndarray) -> dict: """人物关键点 {名称: (x, y, 置信度)};被拍者的左侧(l*)在画面右侧""" global _kp_model if _kp_model is None: _kp_model = torchvision.models.detection.keypointrcnn_resnet50_fpn(weights="DEFAULT").eval() x = torch.from_numpy(on_background(rgba)).permute(2, 0, 1).float() / 255 with torch.no_grad(): out = _kp_model([x])[0] if len(out["scores"]) == 0: raise ValueError("没有检测到人物") i = int(out["scores"].argmax()) k, s = out["keypoints"][i].numpy(), out["keypoints_scores"][i].numpy() return {n: (float(k[j, 0]), float(k[j, 1]), float(s[j])) for j, n in enumerate(KP_NAMES)} def arm_angle(kp: dict, side: str) -> float: """手臂外展角:0 = 竖直向下,90 = 水平,180 = 竖直向上""" sh, wr = kp[f"{side}sh"], kp[f"{side}wr"] return math.degrees(math.atan2(abs(wr[0] - sh[0]), wr[1] - sh[1])) def shoulder_center(kp: dict): return (kp["lsh"][0] + kp["rsh"][0]) / 2, (kp["lsh"][1] + kp["rsh"][1]) / 2 def _sr(): global _sr_model if _sr_model is None: from spandrel import ModelLoader path = MODEL_DIR / Path(SR_URL).name if not path.exists(): MODEL_DIR.mkdir(parents=True, exist_ok=True) print(f"下载 {path.name} …") urllib.request.urlretrieve(SR_URL, path) _sr_model = ModelLoader().load_from_file(str(path)).model.eval().to(_device()) return _sr_model def _run_sr(rgb: np.ndarray) -> np.ndarray: x = torch.from_numpy(rgb.astype(np.float32) / 255).permute(2, 0, 1)[None].to(_device()) with torch.no_grad(): y = _sr()(x) return (y[0].permute(1, 2, 0).clamp(0, 1).cpu().numpy() * 255).round().astype(np.uint8) def _bleed(rgba: np.ndarray) -> np.ndarray: """把前景颜色向透明区域外扩,放大时边缘不会混进背景色(黑边 / 灰边)""" hole = (rgba[..., 3] < 16).astype(np.uint8) rgb = rgba[..., :3].copy() small = cv2.resize(rgb, None, fx=0.5, fy=0.5, interpolation=cv2.INTER_AREA) small_hole = cv2.resize(hole, (small.shape[1], small.shape[0]), interpolation=cv2.INTER_NEAREST) filled = cv2.resize(cv2.inpaint(small, small_hole, 3, cv2.INPAINT_TELEA), (rgb.shape[1], rgb.shape[0])) rgb[hole > 0] = filled[hole > 0] return rgb def upscale4(rgba: np.ndarray) -> np.ndarray: """透明图 4 倍超分:颜色与 alpha 分别放大""" rgb = _run_sr(_bleed(rgba)) a = rgba[..., 3] alpha = _run_sr(np.dstack([a, a, a]))[..., 0] return np.dstack([rgb, alpha])