Files
long 3cd3553af2 feat: 新增 training-videos 素材流水线
AI 跟练的素材生产工具,产物写入 TUICallKit-Vue3/training:
- 教练动作帧:AI 设定稿 → 切帧对齐、锁脸、插帧 → 导出小程序帧与 COS 高清帧
- 示范视频:Remotion 渲染四个动作的示范视频
- 背景音乐:程序合成的原创循环音乐(npm run bgm)
2026-10-31 10:33:25 +08:00

329 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
教练动作帧流水线:连续动作设定稿 → 对齐的高清主帧 + 帧清单
输入:assets/sheets/<动作>.webp —— Codex 生成的透明背景设定稿,一张图里按阅读顺序画一个动作的全部帧
(同一次渲染,前后帧的脸和衣服完全一致,播放时不会“闪”)
处理:
1. 连通域切出每个人物,丢弃远处的彩色噪点
2. Real-ESRGAN 4 倍超分后再缩到主帧尺寸(设定稿里人物只有 300~450px 高)
3. Keypoint R-CNN 测关键点:肩部中点对齐画布中线,脚底对齐地平线
4. 按实测动作幅度挑帧:去掉倒退、重复的帧,幅度归一化到 0~1(播放时按幅度查帧)
5. 锁脸:所有帧都用第一帧的脸,并按头部对齐,播放时脸不会“一闪一闪”、头不会左右晃
6. 插帧:相邻两帧之间用 RIFE 补中间帧(有残影的跳过),再按幅度均匀挑出目标帧数(包体有上限)
7. 左右交替的动作只画左侧,右侧由镜像换边生成(头部保留原图方向,马尾不会左右跳)
输出:work/frames/master/*.png(1000px 主帧)、work/frames/sequences.json、work/preview/contact_*.jpg
用法:python3 coach_frames.py [动作 ...] (默认处理全部动作)
"""
import json
import sys
from pathlib import Path
import cv2
import numpy as np
from PIL import Image
from coach_facelock import lock_faces
from coach_interp import densify, thin
from coach_mirror import mirror_frame
from coach_vision import arm_angle, keypoints, shoulder_center, upscale4
ROOT = Path(__file__).resolve().parents[1]
SHEETS = ROOT / "assets" / "sheets"
WORK = ROOT / "work"
MASTER = WORK / "frames" / "master"
PREVIEW = WORK / "preview"
CANVAS = 1000 # 主帧画布边长
STAND_H = 720 # 标准站姿身高
GROUND_Y = 950 # 地平线
ALPHA_ON = 40
MIN_GAP = 0.015 # 幅度差小于该值的帧视为重复
# 每个动作:设定稿版式(行、列)、取第几个人物(阅读顺序)、幅度指标、帧名前缀、比例基准、镜像生成的另一侧
# scale = "first":整张设定稿按第一个人物(站姿)统一缩放(下蹲、开合跳的身高本来就会变)
# scale = "each":每个人物各自缩放到标准身高(全程站直的动作,顺带消除模型画的大小误差)
# densify:幅度差大于该值的相邻两帧之间插一帧;frames:最终按幅度均匀挑出的帧数
# 帧数按“动作过程约每秒 20 帧以上”定:深蹲、侧平举动作慢(约 1.1 秒)要多;开合跳、高抬腿快(约 0.4 秒)少一些
MOVES = {
"squat": dict(grid=(2, 9), take=range(18), metric="hip_drop", prefix="sq", seq="squat", scale="first", densify=0.03, frames=24),
"raise": dict(grid=(3, 6), take=range(18), metric="arms", prefix="ra", seq="raise", scale="each", densify=0.03, frames=24),
"jack": dict(grid=(3, 6), take=range(18), metric="arms", prefix="jl", seq="jack_l", scale="first", frames=10, mirror=("jack_r", "jr")),
"march": dict(grid=(2, 9), take=range(9), metric="knee_l", prefix="ml", seq="march_l", scale="each", densify=0.03, frames=13, mirror=("march_r", "mr")),
}
def bbox(mask: np.ndarray):
ys, xs = np.where(mask)
return xs.min(), ys.min(), xs.max() + 1, ys.max() + 1
def _centers_inside(lo, hi, n, size):
return [(k + 0.5) * size / n for k in range(n) if lo < (k + 0.5) * size / n < hi]
def _cut_positions(profile, centers):
"""相邻两个格子中心之间,像素最少的位置"""
cuts = []
for a, b in zip(centers, centers[1:]):
lo, hi = int(a + (b - a) * 0.25), int(b - (b - a) * 0.25)
cuts.append(lo + int(np.argmin(profile[lo:hi])))
return cuts
def row_grounds(stats, mains, rows: int, height: int):
"""每一行的地平线 = 该行没有粘连的人物脚底的中位数"""
row_h = height / rows
bottoms = {}
for i in mains:
y0, y1 = stats[i, 1], stats[i, 1] + stats[i, 3]
if y1 - y0 < row_h * 1.2:
bottoms.setdefault(int((y0 + y1) / 2 // row_h), []).append(y1)
return {r: int(np.median(b)) for r, b in bottoms.items()}
def _split_rows(mask: np.ndarray, rows: int, grounds: dict):
"""上下粘连(举过头顶的手碰到上一行的脚):沿上一行的地平线切开,鞋底以下一定属于下一行的人物"""
_, y0, _, y1 = bbox(mask)
height = mask.shape[0]
centers = _centers_inside(y0, y1, rows, height)
if len(centers) <= 1:
return [mask]
first = int(centers[0] // (height / rows))
fallback = _cut_positions(mask.sum(1), centers)
cuts = [grounds.get(first + k, fallback[k]) for k in range(len(centers) - 1)]
parts = []
for a, b in zip([y0] + cuts, cuts + [y1]):
part = np.zeros_like(mask)
part[a:b] = mask[a:b]
if part.any():
parts.append(part)
return parts
def _split_cols(mask: np.ndarray, cols: int):
"""
左右粘连(平举的指尖碰到隔壁):腐蚀断开细小的接触点得到各人物主体,每个像素归给最近的主体;
接触面太大腐蚀不开时,按格子中心之间像素最少的位置直线切开
"""
x0, _, x1, _ = bbox(mask)
centers = _centers_inside(x0, x1, cols, mask.shape[1])
expected = len(centers)
if expected <= 1:
return [mask]
m8 = mask.astype(np.uint8)
for k in (3, 5, 7, 9, 11):
n, lab, stats, _ = cv2.connectedComponentsWithStats(cv2.erode(m8, np.ones((k, k), np.uint8)), connectivity=8)
if n <= 1:
break
areas = stats[1:, cv2.CC_STAT_AREA]
big = [i + 1 for i in np.argsort(-areas)[:expected] if areas[i] > areas.max() * 0.2]
if len(big) == expected:
dist = np.stack([cv2.distanceTransform((lab != i).astype(np.uint8), cv2.DIST_L2, 3) for i in big])
nearest = np.argmin(dist, axis=0)
return [mask & (nearest == j) for j in range(expected)]
cuts = _cut_positions(mask.sum(0), centers)
parts = []
for a, b in zip([x0] + cuts, cuts + [x1]):
part = np.zeros_like(mask)
part[:, a:b] = mask[:, a:b]
parts.append(part)
return parts
def _reassign(parts):
"""切开后,脱离主体的碎块(被切断的指尖)归还给与它接触最多的那个人物"""
mains, frags = [], []
for p in parts:
n, lab, stats, _ = cv2.connectedComponentsWithStats(p.astype(np.uint8), connectivity=8)
k = 1 + int(np.argmax(stats[1:, cv2.CC_STAT_AREA]))
mains.append(lab == k)
frags += [lab == j for j in range(1, n) if j != k]
grown = [cv2.dilate(m.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0 for m in mains]
for f in frags:
contact = [int((g & f).sum()) for g in grown]
if max(contact) > 0:
best = int(np.argmax(contact))
mains[best] = mains[best] | f
return mains
def find_figures(rgba: np.ndarray, grid):
"""连通域切出每个人物(按阅读顺序);紧挨人物的碎块(发梢、指尖)并入,远处的噪点丢弃"""
on = (rgba[..., 3] > ALPHA_ON).astype(np.uint8)
n, lab, stats, _ = cv2.connectedComponentsWithStats(on, connectivity=8)
comps = sorted(range(1, n), key=lambda i: -stats[i, cv2.CC_STAT_AREA])
big = stats[comps[0], cv2.CC_STAT_AREA]
mains = [i for i in comps if stats[i, cv2.CC_STAT_AREA] >= big * 0.18]
rows_n, cols_n = grid
grounds = row_grounds(stats, mains, rows_n, rgba.shape[0])
figs = []
for i in mains:
parts = [p for part in _split_rows(lab == i, rows_n, grounds) for p in _split_cols(part, cols_n)]
figs += _reassign(parts) if len(parts) > 1 else parts
union = np.logical_or.reduce(figs)
box = [bbox(m) for m in figs]
cy = [(b[1] + b[3]) / 2 for b in box]
rows = []
for i in sorted(range(len(figs)), key=lambda i: cy[i]):
if rows and cy[i] - np.mean([cy[j] for j in rows[-1]]) < (box[rows[-1][0]][3] - box[rows[-1][0]][1]) / 2:
rows[-1].append(i)
else:
rows.append([i])
ordered = [i for row in rows for i in sorted(row, key=lambda i: box[i][0])]
dist = [cv2.distanceTransform((~m).astype(np.uint8), cv2.DIST_L2, 3) for m in figs]
extra = [np.zeros_like(m) for m in figs]
for c in comps:
if c in mains or stats[c, cv2.CC_STAT_AREA] < 12:
continue
pts = lab == c
near = min(range(len(figs)), key=lambda k: dist[k][pts].min())
if dist[near][pts].min() <= 6:
extra[near] |= pts
result = []
for i in ordered:
mask = cv2.dilate((figs[i] | extra[i]).astype(np.uint8), np.ones((5, 5), np.uint8)) > 0
mask &= ~(union & ~figs[i]) # 外扩时不要把相邻人物的像素带进来
ys, xs = np.where(mask)
x0, y0, x1, y1 = xs.min(), ys.min(), xs.max() + 1, ys.max() + 1
fig = rgba[y0:y1, x0:x1].copy()
fig[..., 3] = (fig[..., 3] * mask[y0:y1, x0:x1]).astype(np.uint8)
ys, xs = np.where(fig[..., 3] > 8)
result.append(fig[ys.min() : ys.max() + 1, xs.min() : xs.max() + 1])
return result
def resize_rgba(arr: np.ndarray, size) -> np.ndarray:
"""在预乘 Alpha 空间缩放:直接缩放 RGBA 会把透明像素的黑色混进边缘"""
im = Image.fromarray(arr, "RGBA").convert("RGBa").resize(size, Image.Resampling.LANCZOS)
return np.array(im.convert("RGBA"))
def metric(kind: str, kp: dict) -> float:
if kind == "hip_drop":
return (kp["lhip"][1] + kp["rhip"][1]) / 2
if kind == "arms":
return (arm_angle(kp, "l") + arm_angle(kp, "r")) / 2
if kind == "knee_l":
return -kp["lkn"][1]
raise ValueError(kind)
def select(values):
"""按幅度排序,去掉倒退、重复的帧;第一帧(站姿)= 0,幅度最大的帧 = 1"""
v0, v1 = values[0], max(values)
norm = [(v - v0) / (v1 - v0) for v in values]
peak = max(range(len(values)), key=lambda i: norm[i])
chosen = [0]
for i in sorted(range(1, len(values)), key=lambda i: (norm[i], i)):
if norm[i] - norm[chosen[-1]] >= MIN_GAP:
chosen.append(i)
if chosen[-1] != peak:
chosen[-1] = peak
return chosen, [round(norm[i], 3) for i in chosen]
def place(up: np.ndarray, kp_up: dict, scale: float):
"""把 4 倍超分后的人物放到主帧画布:肩部中点对齐中线,脚底对齐地平线;返回画布与画布坐标下的关键点"""
f = scale / 4
w, h = max(1, round(up.shape[1] * f)), max(1, round(up.shape[0] * f))
small = resize_rgba(up, (w, h))
sx, _ = shoulder_center(kp_up)
left = round(CANVAS / 2 - sx * f)
top = GROUND_Y - h
canvas = np.zeros((CANVAS, CANVAS, 4), np.uint8)
dx0, dy0 = max(0, left), max(0, top)
sx0, sy0 = dx0 - left, dy0 - top
cw, ch = min(w - sx0, CANVAS - dx0), min(h - sy0, CANVAS - dy0)
if cw < w or ch < h:
print(f" ! 人物超出画布(left={left}, top={top}, {w}×{h})", file=sys.stderr)
canvas[dy0 : dy0 + ch, dx0 : dx0 + cw] = small[sy0 : sy0 + ch, sx0 : sx0 + cw]
kp = {k: (x * f + left, y * f + top, s) for k, (x, y, s) in kp_up.items()}
return canvas, kp
def strip_foreign_skin(canvas: np.ndarray, kp: dict):
"""脚踝以下只有鞋子,不会有皮肤:出现的肤色像素是下一行人物举起的指尖(设定稿里碰到了鞋),去掉"""
y = int(max(kp["lan"][1], kp["ran"][1]) + 6)
rgb = canvas[y:, :, :3].astype(np.int32)
skin = (rgb[..., 0] > 150) & (rgb[..., 0] - rgb[..., 2] > 50) & (rgb[..., 0] - rgb[..., 1] > 20) & (canvas[y:, :, 3] > 0)
skin = cv2.dilate(skin.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0
canvas[y:][skin] = 0
def process(move: str, cfg: dict, sequences: dict):
rgba = np.array(Image.open(SHEETS / f"{move}.webp").convert("RGBA"))
rgba[rgba[..., 3] < 24] = 0 # 透明背景里的低 alpha 彩色雾点
figures = find_figures(rgba, cfg["grid"])
take = list(cfg["take"])
print(f"{move}: 设定稿 {len(figures)} 个人物,取 {len(take)} 个")
ups, kps = [], []
for i in take:
up = upscale4(figures[i])
ups.append(up)
kps.append(keypoints(up))
heights = [figures[i].shape[0] for i in take]
placed = []
for up, kp_up, h in zip(ups, kps, heights):
scale = STAND_H / (heights[0] if cfg["scale"] == "first" else h)
canvas, kp = place(up, kp_up, scale)
strip_foreign_skin(canvas, kp)
placed.append((canvas, kp))
values = [metric(cfg["metric"], kp) for _, kp in placed]
chosen, amounts = select(values)
dropped = [take[i] for i in range(len(take)) if i not in chosen]
print(f" 幅度 {[round(v, 1) for v in values]}")
print(f" 保留 {len(chosen)} 帧,去掉第 {dropped} 个人物(倒退或重复)")
for prefix in [cfg["prefix"]] + ([cfg["mirror"][1]] if "mirror" in cfg else []):
for old in MASTER.glob(f"{prefix}_*.png"):
old.unlink() # 上一次生成的帧数可能更多
frames, frame_kps = lock_faces([placed[i][0] for i in chosen], [placed[i][1] for i in chosen])
if cfg.get("densify"):
frames, frame_kps, amounts, added = densify(frames, frame_kps, amounts, cfg["densify"])
print(f" 插帧 {added} 帧")
if cfg.get("frames"):
keep = thin(amounts, cfg["frames"])
frames, frame_kps, amounts = [frames[i] for i in keep], [frame_kps[i] for i in keep], [amounts[i] for i in keep]
print(f" 最终 {len(frames)} 帧,幅度 {amounts}")
names = []
for n, frame in enumerate(frames):
name = f"{cfg['prefix']}_{n:02d}"
Image.fromarray(frame, "RGBA").save(MASTER / f"{name}.png")
names.append(name)
sequences[cfg["seq"]] = {"frames": names, "amounts": amounts}
if "mirror" in cfg:
seq, prefix = cfg["mirror"]
mirrored = [names[0]] # 共用起始站姿,换边时不会跳
for n in range(1, len(frames)):
name = f"{prefix}_{n:02d}"
Image.fromarray(mirror_frame(frames[n], frame_kps[n]), "RGBA").save(MASTER / f"{name}.png")
mirrored.append(name)
sequences[seq] = {"frames": mirrored, "amounts": amounts}
contact([sequences[cfg["seq"]]["frames"]] + ([sequences[cfg["mirror"][0]]["frames"]] if "mirror" in cfg else []), PREVIEW / f"contact_{move}.jpg")
def contact(rows, out: Path, cell=160):
cols = max(len(r) for r in rows)
sheet = Image.new("RGB", (cols * cell, len(rows) * cell), (236, 241, 246))
for r, names in enumerate(rows):
for c, n in enumerate(names):
im = Image.open(MASTER / f"{n}.png").crop((100, 120, 900, 970)).resize((cell * 800 // 850, cell), Image.Resampling.LANCZOS)
sheet.paste(im, (c * cell + (cell - im.width) // 2, r * cell), im)
sheet.save(out, quality=88)
def main():
moves = sys.argv[1:] or list(MOVES)
MASTER.mkdir(parents=True, exist_ok=True)
PREVIEW.mkdir(parents=True, exist_ok=True)
out = WORK / "frames" / "sequences.json"
sequences = json.loads(out.read_text()) if out.exists() else {}
for move in moves:
process(move, MOVES[move], sequences)
out.write_text(json.dumps(sequences, ensure_ascii=False, indent=2))
print(f"已写入 {out.relative_to(ROOT)}")
if __name__ == "__main__":
main()