Files
long 3cd3553af2 feat: 新增 training-videos 素材流水线
AI 跟练的素材生产工具,产物写入 TUICallKit-Vue3/training:
- 教练动作帧:AI 设定稿 → 切帧对齐、锁脸、插帧 → 导出小程序帧与 COS 高清帧
- 示范视频:Remotion 渲染四个动作的示范视频
- 背景音乐:程序合成的原创循环音乐(npm run bgm)
2026-10-31 10:33:25 +08:00

104 lines
4.3 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
插帧与挑帧
插帧:在相邻两帧之间用 RIFE(rife-ncnn-vulkan,v4.6 模型)生成中间帧。
帧已经很密、相邻两帧动作很小,插出来的中间帧和原帧难以区分;
但手从下垂变成握拳这类“外形突变”的地方会插出半透明残影——用中间帧里半透明像素的多少判断,
比两侧原帧多出 10% 以上就不用。
RIFE 不处理透明通道:颜色贴在浅灰底上插(边缘就算有一点偏差也是浅灰,在亮色背景上看不出),
透明度单独插一次,再反解出前景色。
挑帧:按动作幅度均匀地挑出 N 帧(包体有上限,帧数和清晰度要平衡)。
RIFE 首次使用时自动下载到 training-videos/work/models(macOS 版)。
"""
import subprocess
import tempfile
import urllib.request
import zipfile
from pathlib import Path
import numpy as np
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
MODEL_DIR = ROOT / "work" / "models"
RIFE_URL = "https://github.com/nihui/rife-ncnn-vulkan/releases/download/20221029/rife-ncnn-vulkan-20221029-macos.zip"
RIFE_DIR = MODEL_DIR / "rife-ncnn-vulkan-20221029-macos"
MODEL = "rife-v4.6"
BG = np.array([237, 241, 239], np.float32)
GHOST_RATIO = 1.10
def _rife() -> Path:
exe = RIFE_DIR / "rife-ncnn-vulkan"
if not exe.exists():
MODEL_DIR.mkdir(parents=True, exist_ok=True)
zip_path = MODEL_DIR / "rife.zip"
print("下载 RIFE …")
urllib.request.urlretrieve(RIFE_URL, zip_path)
with zipfile.ZipFile(zip_path) as z:
z.extractall(MODEL_DIR)
exe.chmod(0o755)
return exe
def _run(a: Path, b: Path, out: Path):
subprocess.run([str(_rife()), "-0", str(a), "-1", str(b), "-o", str(out), "-m", str(RIFE_DIR / MODEL), "-s", "0.5"],
check=True, capture_output=True)
def interpolate(a: np.ndarray, b: np.ndarray) -> np.ndarray:
"""两张 RGBA 主帧 → 正中间的一帧"""
with tempfile.TemporaryDirectory() as tmp:
t = Path(tmp)
for name, f in (("a", a), ("b", b)):
x = f.astype(np.float32)
al = x[..., 3:4] / 255
Image.fromarray((x[..., :3] * al + BG * (1 - al)).round().astype(np.uint8)).save(t / f"{name}_rgb.png")
Image.fromarray(np.dstack([f[..., 3]] * 3)).save(t / f"{name}_a.png")
_run(t / "a_rgb.png", t / "b_rgb.png", t / "mid_rgb.png")
_run(t / "a_a.png", t / "b_a.png", t / "mid_a.png")
rgb = np.array(Image.open(t / "mid_rgb.png").convert("RGB")).astype(np.float32)
al = np.array(Image.open(t / "mid_a.png").convert("L")).astype(np.float32) / 255
al = np.where(al < 0.04, 0, al)
fg = np.clip((rgb - (1 - al[..., None]) * BG) / np.maximum(al, 1e-3)[..., None], 0, 255)
return np.dstack([fg, al * 255]).round().astype(np.uint8)
def _semi(f: np.ndarray) -> int:
al = f[..., 3] / 255.0
return int(((al > 0.25) & (al < 0.75)).sum())
def densify(frames: list, kps: list, amounts: list, max_gap: float):
"""幅度差大于 max_gap 的相邻两帧之间插一帧(有残影的跳过);返回新的帧、关键点、幅度与插入数"""
out_f, out_k, out_a = [frames[0]], [kps[0]], [amounts[0]]
added = 0
for i in range(len(frames) - 1):
a, b = frames[i], frames[i + 1]
if amounts[i + 1] - amounts[i] > max_gap:
mid = interpolate(a, b)
if _semi(mid) <= GHOST_RATIO * (_semi(a) + _semi(b)) / 2:
out_f.append(mid)
out_k.append({k: tuple((np.array(kps[i][k]) + np.array(kps[i + 1][k])) / 2) for k in kps[i]})
out_a.append(round((amounts[i] + amounts[i + 1]) / 2, 3))
added += 1
out_f.append(b)
out_k.append(kps[i + 1])
out_a.append(amounts[i + 1])
return out_f, out_k, out_a, added
def thin(amounts: list, n: int) -> list:
"""按动作幅度均匀挑 n 帧(保留首尾),返回下标"""
if len(amounts) <= n:
return list(range(len(amounts)))
picked = [0]
for k in range(1, n - 1):
target = k / (n - 1)
# 在剩余可选范围内找幅度最接近目标的帧,保证下标递增、给后面留够位置
lo, hi = picked[-1] + 1, len(amounts) - (n - 1 - k)
picked.append(min(range(lo, hi), key=lambda i: abs(amounts[i] - target)))
picked.append(len(amounts) - 1)
return picked