AI 跟练的素材生产工具,产物写入 TUICallKit-Vue3/training: - 教练动作帧:AI 设定稿 → 切帧对齐、锁脸、插帧 → 导出小程序帧与 COS 高清帧 - 示范视频:Remotion 渲染四个动作的示范视频 - 背景音乐:程序合成的原创循环音乐(npm run bgm)
104 lines
4.3 KiB
Python
104 lines
4.3 KiB
Python
"""
|
||
插帧与挑帧
|
||
|
||
插帧:在相邻两帧之间用 RIFE(rife-ncnn-vulkan,v4.6 模型)生成中间帧。
|
||
帧已经很密、相邻两帧动作很小,插出来的中间帧和原帧难以区分;
|
||
但手从下垂变成握拳这类“外形突变”的地方会插出半透明残影——用中间帧里半透明像素的多少判断,
|
||
比两侧原帧多出 10% 以上就不用。
|
||
RIFE 不处理透明通道:颜色贴在浅灰底上插(边缘就算有一点偏差也是浅灰,在亮色背景上看不出),
|
||
透明度单独插一次,再反解出前景色。
|
||
挑帧:按动作幅度均匀地挑出 N 帧(包体有上限,帧数和清晰度要平衡)。
|
||
|
||
RIFE 首次使用时自动下载到 training-videos/work/models(macOS 版)。
|
||
"""
|
||
import subprocess
|
||
import tempfile
|
||
import urllib.request
|
||
import zipfile
|
||
from pathlib import Path
|
||
|
||
import numpy as np
|
||
from PIL import Image
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
MODEL_DIR = ROOT / "work" / "models"
|
||
RIFE_URL = "https://github.com/nihui/rife-ncnn-vulkan/releases/download/20221029/rife-ncnn-vulkan-20221029-macos.zip"
|
||
RIFE_DIR = MODEL_DIR / "rife-ncnn-vulkan-20221029-macos"
|
||
MODEL = "rife-v4.6"
|
||
BG = np.array([237, 241, 239], np.float32)
|
||
GHOST_RATIO = 1.10
|
||
|
||
|
||
def _rife() -> Path:
|
||
exe = RIFE_DIR / "rife-ncnn-vulkan"
|
||
if not exe.exists():
|
||
MODEL_DIR.mkdir(parents=True, exist_ok=True)
|
||
zip_path = MODEL_DIR / "rife.zip"
|
||
print("下载 RIFE …")
|
||
urllib.request.urlretrieve(RIFE_URL, zip_path)
|
||
with zipfile.ZipFile(zip_path) as z:
|
||
z.extractall(MODEL_DIR)
|
||
exe.chmod(0o755)
|
||
return exe
|
||
|
||
|
||
def _run(a: Path, b: Path, out: Path):
|
||
subprocess.run([str(_rife()), "-0", str(a), "-1", str(b), "-o", str(out), "-m", str(RIFE_DIR / MODEL), "-s", "0.5"],
|
||
check=True, capture_output=True)
|
||
|
||
|
||
def interpolate(a: np.ndarray, b: np.ndarray) -> np.ndarray:
|
||
"""两张 RGBA 主帧 → 正中间的一帧"""
|
||
with tempfile.TemporaryDirectory() as tmp:
|
||
t = Path(tmp)
|
||
for name, f in (("a", a), ("b", b)):
|
||
x = f.astype(np.float32)
|
||
al = x[..., 3:4] / 255
|
||
Image.fromarray((x[..., :3] * al + BG * (1 - al)).round().astype(np.uint8)).save(t / f"{name}_rgb.png")
|
||
Image.fromarray(np.dstack([f[..., 3]] * 3)).save(t / f"{name}_a.png")
|
||
_run(t / "a_rgb.png", t / "b_rgb.png", t / "mid_rgb.png")
|
||
_run(t / "a_a.png", t / "b_a.png", t / "mid_a.png")
|
||
rgb = np.array(Image.open(t / "mid_rgb.png").convert("RGB")).astype(np.float32)
|
||
al = np.array(Image.open(t / "mid_a.png").convert("L")).astype(np.float32) / 255
|
||
al = np.where(al < 0.04, 0, al)
|
||
fg = np.clip((rgb - (1 - al[..., None]) * BG) / np.maximum(al, 1e-3)[..., None], 0, 255)
|
||
return np.dstack([fg, al * 255]).round().astype(np.uint8)
|
||
|
||
|
||
def _semi(f: np.ndarray) -> int:
|
||
al = f[..., 3] / 255.0
|
||
return int(((al > 0.25) & (al < 0.75)).sum())
|
||
|
||
|
||
def densify(frames: list, kps: list, amounts: list, max_gap: float):
|
||
"""幅度差大于 max_gap 的相邻两帧之间插一帧(有残影的跳过);返回新的帧、关键点、幅度与插入数"""
|
||
out_f, out_k, out_a = [frames[0]], [kps[0]], [amounts[0]]
|
||
added = 0
|
||
for i in range(len(frames) - 1):
|
||
a, b = frames[i], frames[i + 1]
|
||
if amounts[i + 1] - amounts[i] > max_gap:
|
||
mid = interpolate(a, b)
|
||
if _semi(mid) <= GHOST_RATIO * (_semi(a) + _semi(b)) / 2:
|
||
out_f.append(mid)
|
||
out_k.append({k: tuple((np.array(kps[i][k]) + np.array(kps[i + 1][k])) / 2) for k in kps[i]})
|
||
out_a.append(round((amounts[i] + amounts[i + 1]) / 2, 3))
|
||
added += 1
|
||
out_f.append(b)
|
||
out_k.append(kps[i + 1])
|
||
out_a.append(amounts[i + 1])
|
||
return out_f, out_k, out_a, added
|
||
|
||
|
||
def thin(amounts: list, n: int) -> list:
|
||
"""按动作幅度均匀挑 n 帧(保留首尾),返回下标"""
|
||
if len(amounts) <= n:
|
||
return list(range(len(amounts)))
|
||
picked = [0]
|
||
for k in range(1, n - 1):
|
||
target = k / (n - 1)
|
||
# 在剩余可选范围内找幅度最接近目标的帧,保证下标递增、给后面留够位置
|
||
lo, hi = picked[-1] + 1, len(amounts) - (n - 1 - k)
|
||
picked.append(min(range(lo, hi), key=lambda i: abs(amounts[i] - target)))
|
||
picked.append(len(amounts) - 1)
|
||
return picked
|