""" 教练动作帧流水线:连续动作设定稿 → 对齐的高清主帧 + 帧清单 输入:assets/sheets/<动作>.webp —— Codex 生成的透明背景设定稿,一张图里按阅读顺序画一个动作的全部帧 (同一次渲染,前后帧的脸和衣服完全一致,播放时不会“闪”) 处理: 1. 连通域切出每个人物,丢弃远处的彩色噪点 2. Real-ESRGAN 4 倍超分后再缩到主帧尺寸(设定稿里人物只有 300~450px 高) 3. Keypoint R-CNN 测关键点:肩部中点对齐画布中线,脚底对齐地平线 4. 按实测动作幅度挑帧:去掉倒退、重复的帧,幅度归一化到 0~1(播放时按幅度查帧) 5. 锁脸:所有帧都用第一帧的脸,并按头部对齐,播放时脸不会“一闪一闪”、头不会左右晃 6. 插帧:相邻两帧之间用 RIFE 补中间帧(有残影的跳过),再按幅度均匀挑出目标帧数(包体有上限) 7. 左右交替的动作只画左侧,右侧由镜像换边生成(头部保留原图方向,马尾不会左右跳) 输出:work/frames/master/*.png(1000px 主帧)、work/frames/sequences.json、work/preview/contact_*.jpg 用法:python3 coach_frames.py [动作 ...] (默认处理全部动作) """ import json import sys from pathlib import Path import cv2 import numpy as np from PIL import Image from coach_facelock import lock_faces from coach_interp import densify, thin from coach_mirror import mirror_frame from coach_vision import arm_angle, keypoints, shoulder_center, upscale4 ROOT = Path(__file__).resolve().parents[1] SHEETS = ROOT / "assets" / "sheets" WORK = ROOT / "work" MASTER = WORK / "frames" / "master" PREVIEW = WORK / "preview" CANVAS = 1000 # 主帧画布边长 STAND_H = 720 # 标准站姿身高 GROUND_Y = 950 # 地平线 ALPHA_ON = 40 MIN_GAP = 0.015 # 幅度差小于该值的帧视为重复 # 每个动作:设定稿版式(行、列)、取第几个人物(阅读顺序)、幅度指标、帧名前缀、比例基准、镜像生成的另一侧 # scale = "first":整张设定稿按第一个人物(站姿)统一缩放(下蹲、开合跳的身高本来就会变) # scale = "each":每个人物各自缩放到标准身高(全程站直的动作,顺带消除模型画的大小误差) # densify:幅度差大于该值的相邻两帧之间插一帧;frames:最终按幅度均匀挑出的帧数 # 帧数按“动作过程约每秒 20 帧以上”定:深蹲、侧平举动作慢(约 1.1 秒)要多;开合跳、高抬腿快(约 0.4 秒)少一些 MOVES = { "squat": dict(grid=(2, 9), take=range(18), metric="hip_drop", prefix="sq", seq="squat", scale="first", densify=0.03, frames=24), "raise": dict(grid=(3, 6), take=range(18), metric="arms", prefix="ra", seq="raise", scale="each", densify=0.03, frames=24), "jack": dict(grid=(3, 6), take=range(18), metric="arms", prefix="jl", seq="jack_l", scale="first", frames=10, mirror=("jack_r", "jr")), "march": dict(grid=(2, 9), take=range(9), metric="knee_l", prefix="ml", seq="march_l", scale="each", densify=0.03, frames=13, mirror=("march_r", "mr")), } def bbox(mask: np.ndarray): ys, xs = np.where(mask) return xs.min(), ys.min(), xs.max() + 1, ys.max() + 1 def _centers_inside(lo, hi, n, size): return [(k + 0.5) * size / n for k in range(n) if lo < (k + 0.5) * size / n < hi] def _cut_positions(profile, centers): """相邻两个格子中心之间,像素最少的位置""" cuts = [] for a, b in zip(centers, centers[1:]): lo, hi = int(a + (b - a) * 0.25), int(b - (b - a) * 0.25) cuts.append(lo + int(np.argmin(profile[lo:hi]))) return cuts def row_grounds(stats, mains, rows: int, height: int): """每一行的地平线 = 该行没有粘连的人物脚底的中位数""" row_h = height / rows bottoms = {} for i in mains: y0, y1 = stats[i, 1], stats[i, 1] + stats[i, 3] if y1 - y0 < row_h * 1.2: bottoms.setdefault(int((y0 + y1) / 2 // row_h), []).append(y1) return {r: int(np.median(b)) for r, b in bottoms.items()} def _split_rows(mask: np.ndarray, rows: int, grounds: dict): """上下粘连(举过头顶的手碰到上一行的脚):沿上一行的地平线切开,鞋底以下一定属于下一行的人物""" _, y0, _, y1 = bbox(mask) height = mask.shape[0] centers = _centers_inside(y0, y1, rows, height) if len(centers) <= 1: return [mask] first = int(centers[0] // (height / rows)) fallback = _cut_positions(mask.sum(1), centers) cuts = [grounds.get(first + k, fallback[k]) for k in range(len(centers) - 1)] parts = [] for a, b in zip([y0] + cuts, cuts + [y1]): part = np.zeros_like(mask) part[a:b] = mask[a:b] if part.any(): parts.append(part) return parts def _split_cols(mask: np.ndarray, cols: int): """ 左右粘连(平举的指尖碰到隔壁):腐蚀断开细小的接触点得到各人物主体,每个像素归给最近的主体; 接触面太大腐蚀不开时,按格子中心之间像素最少的位置直线切开 """ x0, _, x1, _ = bbox(mask) centers = _centers_inside(x0, x1, cols, mask.shape[1]) expected = len(centers) if expected <= 1: return [mask] m8 = mask.astype(np.uint8) for k in (3, 5, 7, 9, 11): n, lab, stats, _ = cv2.connectedComponentsWithStats(cv2.erode(m8, np.ones((k, k), np.uint8)), connectivity=8) if n <= 1: break areas = stats[1:, cv2.CC_STAT_AREA] big = [i + 1 for i in np.argsort(-areas)[:expected] if areas[i] > areas.max() * 0.2] if len(big) == expected: dist = np.stack([cv2.distanceTransform((lab != i).astype(np.uint8), cv2.DIST_L2, 3) for i in big]) nearest = np.argmin(dist, axis=0) return [mask & (nearest == j) for j in range(expected)] cuts = _cut_positions(mask.sum(0), centers) parts = [] for a, b in zip([x0] + cuts, cuts + [x1]): part = np.zeros_like(mask) part[:, a:b] = mask[:, a:b] parts.append(part) return parts def _reassign(parts): """切开后,脱离主体的碎块(被切断的指尖)归还给与它接触最多的那个人物""" mains, frags = [], [] for p in parts: n, lab, stats, _ = cv2.connectedComponentsWithStats(p.astype(np.uint8), connectivity=8) k = 1 + int(np.argmax(stats[1:, cv2.CC_STAT_AREA])) mains.append(lab == k) frags += [lab == j for j in range(1, n) if j != k] grown = [cv2.dilate(m.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0 for m in mains] for f in frags: contact = [int((g & f).sum()) for g in grown] if max(contact) > 0: best = int(np.argmax(contact)) mains[best] = mains[best] | f return mains def find_figures(rgba: np.ndarray, grid): """连通域切出每个人物(按阅读顺序);紧挨人物的碎块(发梢、指尖)并入,远处的噪点丢弃""" on = (rgba[..., 3] > ALPHA_ON).astype(np.uint8) n, lab, stats, _ = cv2.connectedComponentsWithStats(on, connectivity=8) comps = sorted(range(1, n), key=lambda i: -stats[i, cv2.CC_STAT_AREA]) big = stats[comps[0], cv2.CC_STAT_AREA] mains = [i for i in comps if stats[i, cv2.CC_STAT_AREA] >= big * 0.18] rows_n, cols_n = grid grounds = row_grounds(stats, mains, rows_n, rgba.shape[0]) figs = [] for i in mains: parts = [p for part in _split_rows(lab == i, rows_n, grounds) for p in _split_cols(part, cols_n)] figs += _reassign(parts) if len(parts) > 1 else parts union = np.logical_or.reduce(figs) box = [bbox(m) for m in figs] cy = [(b[1] + b[3]) / 2 for b in box] rows = [] for i in sorted(range(len(figs)), key=lambda i: cy[i]): if rows and cy[i] - np.mean([cy[j] for j in rows[-1]]) < (box[rows[-1][0]][3] - box[rows[-1][0]][1]) / 2: rows[-1].append(i) else: rows.append([i]) ordered = [i for row in rows for i in sorted(row, key=lambda i: box[i][0])] dist = [cv2.distanceTransform((~m).astype(np.uint8), cv2.DIST_L2, 3) for m in figs] extra = [np.zeros_like(m) for m in figs] for c in comps: if c in mains or stats[c, cv2.CC_STAT_AREA] < 12: continue pts = lab == c near = min(range(len(figs)), key=lambda k: dist[k][pts].min()) if dist[near][pts].min() <= 6: extra[near] |= pts result = [] for i in ordered: mask = cv2.dilate((figs[i] | extra[i]).astype(np.uint8), np.ones((5, 5), np.uint8)) > 0 mask &= ~(union & ~figs[i]) # 外扩时不要把相邻人物的像素带进来 ys, xs = np.where(mask) x0, y0, x1, y1 = xs.min(), ys.min(), xs.max() + 1, ys.max() + 1 fig = rgba[y0:y1, x0:x1].copy() fig[..., 3] = (fig[..., 3] * mask[y0:y1, x0:x1]).astype(np.uint8) ys, xs = np.where(fig[..., 3] > 8) result.append(fig[ys.min() : ys.max() + 1, xs.min() : xs.max() + 1]) return result def resize_rgba(arr: np.ndarray, size) -> np.ndarray: """在预乘 Alpha 空间缩放:直接缩放 RGBA 会把透明像素的黑色混进边缘""" im = Image.fromarray(arr, "RGBA").convert("RGBa").resize(size, Image.Resampling.LANCZOS) return np.array(im.convert("RGBA")) def metric(kind: str, kp: dict) -> float: if kind == "hip_drop": return (kp["lhip"][1] + kp["rhip"][1]) / 2 if kind == "arms": return (arm_angle(kp, "l") + arm_angle(kp, "r")) / 2 if kind == "knee_l": return -kp["lkn"][1] raise ValueError(kind) def select(values): """按幅度排序,去掉倒退、重复的帧;第一帧(站姿)= 0,幅度最大的帧 = 1""" v0, v1 = values[0], max(values) norm = [(v - v0) / (v1 - v0) for v in values] peak = max(range(len(values)), key=lambda i: norm[i]) chosen = [0] for i in sorted(range(1, len(values)), key=lambda i: (norm[i], i)): if norm[i] - norm[chosen[-1]] >= MIN_GAP: chosen.append(i) if chosen[-1] != peak: chosen[-1] = peak return chosen, [round(norm[i], 3) for i in chosen] def place(up: np.ndarray, kp_up: dict, scale: float): """把 4 倍超分后的人物放到主帧画布:肩部中点对齐中线,脚底对齐地平线;返回画布与画布坐标下的关键点""" f = scale / 4 w, h = max(1, round(up.shape[1] * f)), max(1, round(up.shape[0] * f)) small = resize_rgba(up, (w, h)) sx, _ = shoulder_center(kp_up) left = round(CANVAS / 2 - sx * f) top = GROUND_Y - h canvas = np.zeros((CANVAS, CANVAS, 4), np.uint8) dx0, dy0 = max(0, left), max(0, top) sx0, sy0 = dx0 - left, dy0 - top cw, ch = min(w - sx0, CANVAS - dx0), min(h - sy0, CANVAS - dy0) if cw < w or ch < h: print(f" ! 人物超出画布(left={left}, top={top}, {w}×{h})", file=sys.stderr) canvas[dy0 : dy0 + ch, dx0 : dx0 + cw] = small[sy0 : sy0 + ch, sx0 : sx0 + cw] kp = {k: (x * f + left, y * f + top, s) for k, (x, y, s) in kp_up.items()} return canvas, kp def strip_foreign_skin(canvas: np.ndarray, kp: dict): """脚踝以下只有鞋子,不会有皮肤:出现的肤色像素是下一行人物举起的指尖(设定稿里碰到了鞋),去掉""" y = int(max(kp["lan"][1], kp["ran"][1]) + 6) rgb = canvas[y:, :, :3].astype(np.int32) skin = (rgb[..., 0] > 150) & (rgb[..., 0] - rgb[..., 2] > 50) & (rgb[..., 0] - rgb[..., 1] > 20) & (canvas[y:, :, 3] > 0) skin = cv2.dilate(skin.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0 canvas[y:][skin] = 0 def process(move: str, cfg: dict, sequences: dict): rgba = np.array(Image.open(SHEETS / f"{move}.webp").convert("RGBA")) rgba[rgba[..., 3] < 24] = 0 # 透明背景里的低 alpha 彩色雾点 figures = find_figures(rgba, cfg["grid"]) take = list(cfg["take"]) print(f"{move}: 设定稿 {len(figures)} 个人物,取 {len(take)} 个") ups, kps = [], [] for i in take: up = upscale4(figures[i]) ups.append(up) kps.append(keypoints(up)) heights = [figures[i].shape[0] for i in take] placed = [] for up, kp_up, h in zip(ups, kps, heights): scale = STAND_H / (heights[0] if cfg["scale"] == "first" else h) canvas, kp = place(up, kp_up, scale) strip_foreign_skin(canvas, kp) placed.append((canvas, kp)) values = [metric(cfg["metric"], kp) for _, kp in placed] chosen, amounts = select(values) dropped = [take[i] for i in range(len(take)) if i not in chosen] print(f" 幅度 {[round(v, 1) for v in values]}") print(f" 保留 {len(chosen)} 帧,去掉第 {dropped} 个人物(倒退或重复)") for prefix in [cfg["prefix"]] + ([cfg["mirror"][1]] if "mirror" in cfg else []): for old in MASTER.glob(f"{prefix}_*.png"): old.unlink() # 上一次生成的帧数可能更多 frames, frame_kps = lock_faces([placed[i][0] for i in chosen], [placed[i][1] for i in chosen]) if cfg.get("densify"): frames, frame_kps, amounts, added = densify(frames, frame_kps, amounts, cfg["densify"]) print(f" 插帧 {added} 帧") if cfg.get("frames"): keep = thin(amounts, cfg["frames"]) frames, frame_kps, amounts = [frames[i] for i in keep], [frame_kps[i] for i in keep], [amounts[i] for i in keep] print(f" 最终 {len(frames)} 帧,幅度 {amounts}") names = [] for n, frame in enumerate(frames): name = f"{cfg['prefix']}_{n:02d}" Image.fromarray(frame, "RGBA").save(MASTER / f"{name}.png") names.append(name) sequences[cfg["seq"]] = {"frames": names, "amounts": amounts} if "mirror" in cfg: seq, prefix = cfg["mirror"] mirrored = [names[0]] # 共用起始站姿,换边时不会跳 for n in range(1, len(frames)): name = f"{prefix}_{n:02d}" Image.fromarray(mirror_frame(frames[n], frame_kps[n]), "RGBA").save(MASTER / f"{name}.png") mirrored.append(name) sequences[seq] = {"frames": mirrored, "amounts": amounts} contact([sequences[cfg["seq"]]["frames"]] + ([sequences[cfg["mirror"][0]]["frames"]] if "mirror" in cfg else []), PREVIEW / f"contact_{move}.jpg") def contact(rows, out: Path, cell=160): cols = max(len(r) for r in rows) sheet = Image.new("RGB", (cols * cell, len(rows) * cell), (236, 241, 246)) for r, names in enumerate(rows): for c, n in enumerate(names): im = Image.open(MASTER / f"{n}.png").crop((100, 120, 900, 970)).resize((cell * 800 // 850, cell), Image.Resampling.LANCZOS) sheet.paste(im, (c * cell + (cell - im.width) // 2, r * cell), im) sheet.save(out, quality=88) def main(): moves = sys.argv[1:] or list(MOVES) MASTER.mkdir(parents=True, exist_ok=True) PREVIEW.mkdir(parents=True, exist_ok=True) out = WORK / "frames" / "sequences.json" sequences = json.loads(out.read_text()) if out.exists() else {} for move in moves: process(move, MOVES[move], sequences) out.write_text(json.dumps(sequences, ensure_ascii=False, indent=2)) print(f"已写入 {out.relative_to(ROOT)}") if __name__ == "__main__": main()