AI 跟练的素材生产工具,产物写入 TUICallKit-Vue3/training: - 教练动作帧:AI 设定稿 → 切帧对齐、锁脸、插帧 → 导出小程序帧与 COS 高清帧 - 示范视频:Remotion 渲染四个动作的示范视频 - 背景音乐:程序合成的原创循环音乐(npm run bgm)
329 lines
15 KiB
Python
329 lines
15 KiB
Python
"""
|
||
教练动作帧流水线:连续动作设定稿 → 对齐的高清主帧 + 帧清单
|
||
|
||
输入:assets/sheets/<动作>.webp —— Codex 生成的透明背景设定稿,一张图里按阅读顺序画一个动作的全部帧
|
||
(同一次渲染,前后帧的脸和衣服完全一致,播放时不会“闪”)
|
||
处理:
|
||
1. 连通域切出每个人物,丢弃远处的彩色噪点
|
||
2. Real-ESRGAN 4 倍超分后再缩到主帧尺寸(设定稿里人物只有 300~450px 高)
|
||
3. Keypoint R-CNN 测关键点:肩部中点对齐画布中线,脚底对齐地平线
|
||
4. 按实测动作幅度挑帧:去掉倒退、重复的帧,幅度归一化到 0~1(播放时按幅度查帧)
|
||
5. 锁脸:所有帧都用第一帧的脸,并按头部对齐,播放时脸不会“一闪一闪”、头不会左右晃
|
||
6. 插帧:相邻两帧之间用 RIFE 补中间帧(有残影的跳过),再按幅度均匀挑出目标帧数(包体有上限)
|
||
7. 左右交替的动作只画左侧,右侧由镜像换边生成(头部保留原图方向,马尾不会左右跳)
|
||
输出:work/frames/master/*.png(1000px 主帧)、work/frames/sequences.json、work/preview/contact_*.jpg
|
||
|
||
用法:python3 coach_frames.py [动作 ...] (默认处理全部动作)
|
||
"""
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
import cv2
|
||
import numpy as np
|
||
from PIL import Image
|
||
|
||
from coach_facelock import lock_faces
|
||
from coach_interp import densify, thin
|
||
from coach_mirror import mirror_frame
|
||
from coach_vision import arm_angle, keypoints, shoulder_center, upscale4
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
SHEETS = ROOT / "assets" / "sheets"
|
||
WORK = ROOT / "work"
|
||
MASTER = WORK / "frames" / "master"
|
||
PREVIEW = WORK / "preview"
|
||
|
||
CANVAS = 1000 # 主帧画布边长
|
||
STAND_H = 720 # 标准站姿身高
|
||
GROUND_Y = 950 # 地平线
|
||
ALPHA_ON = 40
|
||
MIN_GAP = 0.015 # 幅度差小于该值的帧视为重复
|
||
|
||
# 每个动作:设定稿版式(行、列)、取第几个人物(阅读顺序)、幅度指标、帧名前缀、比例基准、镜像生成的另一侧
|
||
# scale = "first":整张设定稿按第一个人物(站姿)统一缩放(下蹲、开合跳的身高本来就会变)
|
||
# scale = "each":每个人物各自缩放到标准身高(全程站直的动作,顺带消除模型画的大小误差)
|
||
# densify:幅度差大于该值的相邻两帧之间插一帧;frames:最终按幅度均匀挑出的帧数
|
||
# 帧数按“动作过程约每秒 20 帧以上”定:深蹲、侧平举动作慢(约 1.1 秒)要多;开合跳、高抬腿快(约 0.4 秒)少一些
|
||
MOVES = {
|
||
"squat": dict(grid=(2, 9), take=range(18), metric="hip_drop", prefix="sq", seq="squat", scale="first", densify=0.03, frames=24),
|
||
"raise": dict(grid=(3, 6), take=range(18), metric="arms", prefix="ra", seq="raise", scale="each", densify=0.03, frames=24),
|
||
"jack": dict(grid=(3, 6), take=range(18), metric="arms", prefix="jl", seq="jack_l", scale="first", frames=10, mirror=("jack_r", "jr")),
|
||
"march": dict(grid=(2, 9), take=range(9), metric="knee_l", prefix="ml", seq="march_l", scale="each", densify=0.03, frames=13, mirror=("march_r", "mr")),
|
||
}
|
||
|
||
|
||
def bbox(mask: np.ndarray):
|
||
ys, xs = np.where(mask)
|
||
return xs.min(), ys.min(), xs.max() + 1, ys.max() + 1
|
||
|
||
|
||
def _centers_inside(lo, hi, n, size):
|
||
return [(k + 0.5) * size / n for k in range(n) if lo < (k + 0.5) * size / n < hi]
|
||
|
||
|
||
def _cut_positions(profile, centers):
|
||
"""相邻两个格子中心之间,像素最少的位置"""
|
||
cuts = []
|
||
for a, b in zip(centers, centers[1:]):
|
||
lo, hi = int(a + (b - a) * 0.25), int(b - (b - a) * 0.25)
|
||
cuts.append(lo + int(np.argmin(profile[lo:hi])))
|
||
return cuts
|
||
|
||
|
||
def row_grounds(stats, mains, rows: int, height: int):
|
||
"""每一行的地平线 = 该行没有粘连的人物脚底的中位数"""
|
||
row_h = height / rows
|
||
bottoms = {}
|
||
for i in mains:
|
||
y0, y1 = stats[i, 1], stats[i, 1] + stats[i, 3]
|
||
if y1 - y0 < row_h * 1.2:
|
||
bottoms.setdefault(int((y0 + y1) / 2 // row_h), []).append(y1)
|
||
return {r: int(np.median(b)) for r, b in bottoms.items()}
|
||
|
||
|
||
def _split_rows(mask: np.ndarray, rows: int, grounds: dict):
|
||
"""上下粘连(举过头顶的手碰到上一行的脚):沿上一行的地平线切开,鞋底以下一定属于下一行的人物"""
|
||
_, y0, _, y1 = bbox(mask)
|
||
height = mask.shape[0]
|
||
centers = _centers_inside(y0, y1, rows, height)
|
||
if len(centers) <= 1:
|
||
return [mask]
|
||
first = int(centers[0] // (height / rows))
|
||
fallback = _cut_positions(mask.sum(1), centers)
|
||
cuts = [grounds.get(first + k, fallback[k]) for k in range(len(centers) - 1)]
|
||
parts = []
|
||
for a, b in zip([y0] + cuts, cuts + [y1]):
|
||
part = np.zeros_like(mask)
|
||
part[a:b] = mask[a:b]
|
||
if part.any():
|
||
parts.append(part)
|
||
return parts
|
||
|
||
|
||
def _split_cols(mask: np.ndarray, cols: int):
|
||
"""
|
||
左右粘连(平举的指尖碰到隔壁):腐蚀断开细小的接触点得到各人物主体,每个像素归给最近的主体;
|
||
接触面太大腐蚀不开时,按格子中心之间像素最少的位置直线切开
|
||
"""
|
||
x0, _, x1, _ = bbox(mask)
|
||
centers = _centers_inside(x0, x1, cols, mask.shape[1])
|
||
expected = len(centers)
|
||
if expected <= 1:
|
||
return [mask]
|
||
m8 = mask.astype(np.uint8)
|
||
for k in (3, 5, 7, 9, 11):
|
||
n, lab, stats, _ = cv2.connectedComponentsWithStats(cv2.erode(m8, np.ones((k, k), np.uint8)), connectivity=8)
|
||
if n <= 1:
|
||
break
|
||
areas = stats[1:, cv2.CC_STAT_AREA]
|
||
big = [i + 1 for i in np.argsort(-areas)[:expected] if areas[i] > areas.max() * 0.2]
|
||
if len(big) == expected:
|
||
dist = np.stack([cv2.distanceTransform((lab != i).astype(np.uint8), cv2.DIST_L2, 3) for i in big])
|
||
nearest = np.argmin(dist, axis=0)
|
||
return [mask & (nearest == j) for j in range(expected)]
|
||
cuts = _cut_positions(mask.sum(0), centers)
|
||
parts = []
|
||
for a, b in zip([x0] + cuts, cuts + [x1]):
|
||
part = np.zeros_like(mask)
|
||
part[:, a:b] = mask[:, a:b]
|
||
parts.append(part)
|
||
return parts
|
||
|
||
|
||
def _reassign(parts):
|
||
"""切开后,脱离主体的碎块(被切断的指尖)归还给与它接触最多的那个人物"""
|
||
mains, frags = [], []
|
||
for p in parts:
|
||
n, lab, stats, _ = cv2.connectedComponentsWithStats(p.astype(np.uint8), connectivity=8)
|
||
k = 1 + int(np.argmax(stats[1:, cv2.CC_STAT_AREA]))
|
||
mains.append(lab == k)
|
||
frags += [lab == j for j in range(1, n) if j != k]
|
||
grown = [cv2.dilate(m.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0 for m in mains]
|
||
for f in frags:
|
||
contact = [int((g & f).sum()) for g in grown]
|
||
if max(contact) > 0:
|
||
best = int(np.argmax(contact))
|
||
mains[best] = mains[best] | f
|
||
return mains
|
||
|
||
|
||
def find_figures(rgba: np.ndarray, grid):
|
||
"""连通域切出每个人物(按阅读顺序);紧挨人物的碎块(发梢、指尖)并入,远处的噪点丢弃"""
|
||
on = (rgba[..., 3] > ALPHA_ON).astype(np.uint8)
|
||
n, lab, stats, _ = cv2.connectedComponentsWithStats(on, connectivity=8)
|
||
comps = sorted(range(1, n), key=lambda i: -stats[i, cv2.CC_STAT_AREA])
|
||
big = stats[comps[0], cv2.CC_STAT_AREA]
|
||
mains = [i for i in comps if stats[i, cv2.CC_STAT_AREA] >= big * 0.18]
|
||
rows_n, cols_n = grid
|
||
grounds = row_grounds(stats, mains, rows_n, rgba.shape[0])
|
||
figs = []
|
||
for i in mains:
|
||
parts = [p for part in _split_rows(lab == i, rows_n, grounds) for p in _split_cols(part, cols_n)]
|
||
figs += _reassign(parts) if len(parts) > 1 else parts
|
||
union = np.logical_or.reduce(figs)
|
||
box = [bbox(m) for m in figs]
|
||
cy = [(b[1] + b[3]) / 2 for b in box]
|
||
rows = []
|
||
for i in sorted(range(len(figs)), key=lambda i: cy[i]):
|
||
if rows and cy[i] - np.mean([cy[j] for j in rows[-1]]) < (box[rows[-1][0]][3] - box[rows[-1][0]][1]) / 2:
|
||
rows[-1].append(i)
|
||
else:
|
||
rows.append([i])
|
||
ordered = [i for row in rows for i in sorted(row, key=lambda i: box[i][0])]
|
||
dist = [cv2.distanceTransform((~m).astype(np.uint8), cv2.DIST_L2, 3) for m in figs]
|
||
extra = [np.zeros_like(m) for m in figs]
|
||
for c in comps:
|
||
if c in mains or stats[c, cv2.CC_STAT_AREA] < 12:
|
||
continue
|
||
pts = lab == c
|
||
near = min(range(len(figs)), key=lambda k: dist[k][pts].min())
|
||
if dist[near][pts].min() <= 6:
|
||
extra[near] |= pts
|
||
result = []
|
||
for i in ordered:
|
||
mask = cv2.dilate((figs[i] | extra[i]).astype(np.uint8), np.ones((5, 5), np.uint8)) > 0
|
||
mask &= ~(union & ~figs[i]) # 外扩时不要把相邻人物的像素带进来
|
||
ys, xs = np.where(mask)
|
||
x0, y0, x1, y1 = xs.min(), ys.min(), xs.max() + 1, ys.max() + 1
|
||
fig = rgba[y0:y1, x0:x1].copy()
|
||
fig[..., 3] = (fig[..., 3] * mask[y0:y1, x0:x1]).astype(np.uint8)
|
||
ys, xs = np.where(fig[..., 3] > 8)
|
||
result.append(fig[ys.min() : ys.max() + 1, xs.min() : xs.max() + 1])
|
||
return result
|
||
|
||
|
||
def resize_rgba(arr: np.ndarray, size) -> np.ndarray:
|
||
"""在预乘 Alpha 空间缩放:直接缩放 RGBA 会把透明像素的黑色混进边缘"""
|
||
im = Image.fromarray(arr, "RGBA").convert("RGBa").resize(size, Image.Resampling.LANCZOS)
|
||
return np.array(im.convert("RGBA"))
|
||
|
||
|
||
def metric(kind: str, kp: dict) -> float:
|
||
if kind == "hip_drop":
|
||
return (kp["lhip"][1] + kp["rhip"][1]) / 2
|
||
if kind == "arms":
|
||
return (arm_angle(kp, "l") + arm_angle(kp, "r")) / 2
|
||
if kind == "knee_l":
|
||
return -kp["lkn"][1]
|
||
raise ValueError(kind)
|
||
|
||
|
||
def select(values):
|
||
"""按幅度排序,去掉倒退、重复的帧;第一帧(站姿)= 0,幅度最大的帧 = 1"""
|
||
v0, v1 = values[0], max(values)
|
||
norm = [(v - v0) / (v1 - v0) for v in values]
|
||
peak = max(range(len(values)), key=lambda i: norm[i])
|
||
chosen = [0]
|
||
for i in sorted(range(1, len(values)), key=lambda i: (norm[i], i)):
|
||
if norm[i] - norm[chosen[-1]] >= MIN_GAP:
|
||
chosen.append(i)
|
||
if chosen[-1] != peak:
|
||
chosen[-1] = peak
|
||
return chosen, [round(norm[i], 3) for i in chosen]
|
||
|
||
|
||
def place(up: np.ndarray, kp_up: dict, scale: float):
|
||
"""把 4 倍超分后的人物放到主帧画布:肩部中点对齐中线,脚底对齐地平线;返回画布与画布坐标下的关键点"""
|
||
f = scale / 4
|
||
w, h = max(1, round(up.shape[1] * f)), max(1, round(up.shape[0] * f))
|
||
small = resize_rgba(up, (w, h))
|
||
sx, _ = shoulder_center(kp_up)
|
||
left = round(CANVAS / 2 - sx * f)
|
||
top = GROUND_Y - h
|
||
canvas = np.zeros((CANVAS, CANVAS, 4), np.uint8)
|
||
dx0, dy0 = max(0, left), max(0, top)
|
||
sx0, sy0 = dx0 - left, dy0 - top
|
||
cw, ch = min(w - sx0, CANVAS - dx0), min(h - sy0, CANVAS - dy0)
|
||
if cw < w or ch < h:
|
||
print(f" ! 人物超出画布(left={left}, top={top}, {w}×{h})", file=sys.stderr)
|
||
canvas[dy0 : dy0 + ch, dx0 : dx0 + cw] = small[sy0 : sy0 + ch, sx0 : sx0 + cw]
|
||
kp = {k: (x * f + left, y * f + top, s) for k, (x, y, s) in kp_up.items()}
|
||
return canvas, kp
|
||
|
||
|
||
def strip_foreign_skin(canvas: np.ndarray, kp: dict):
|
||
"""脚踝以下只有鞋子,不会有皮肤:出现的肤色像素是下一行人物举起的指尖(设定稿里碰到了鞋),去掉"""
|
||
y = int(max(kp["lan"][1], kp["ran"][1]) + 6)
|
||
rgb = canvas[y:, :, :3].astype(np.int32)
|
||
skin = (rgb[..., 0] > 150) & (rgb[..., 0] - rgb[..., 2] > 50) & (rgb[..., 0] - rgb[..., 1] > 20) & (canvas[y:, :, 3] > 0)
|
||
skin = cv2.dilate(skin.astype(np.uint8), np.ones((5, 5), np.uint8)) > 0
|
||
canvas[y:][skin] = 0
|
||
|
||
|
||
def process(move: str, cfg: dict, sequences: dict):
|
||
rgba = np.array(Image.open(SHEETS / f"{move}.webp").convert("RGBA"))
|
||
rgba[rgba[..., 3] < 24] = 0 # 透明背景里的低 alpha 彩色雾点
|
||
figures = find_figures(rgba, cfg["grid"])
|
||
take = list(cfg["take"])
|
||
print(f"{move}: 设定稿 {len(figures)} 个人物,取 {len(take)} 个")
|
||
ups, kps = [], []
|
||
for i in take:
|
||
up = upscale4(figures[i])
|
||
ups.append(up)
|
||
kps.append(keypoints(up))
|
||
heights = [figures[i].shape[0] for i in take]
|
||
placed = []
|
||
for up, kp_up, h in zip(ups, kps, heights):
|
||
scale = STAND_H / (heights[0] if cfg["scale"] == "first" else h)
|
||
canvas, kp = place(up, kp_up, scale)
|
||
strip_foreign_skin(canvas, kp)
|
||
placed.append((canvas, kp))
|
||
values = [metric(cfg["metric"], kp) for _, kp in placed]
|
||
chosen, amounts = select(values)
|
||
dropped = [take[i] for i in range(len(take)) if i not in chosen]
|
||
print(f" 幅度 {[round(v, 1) for v in values]}")
|
||
print(f" 保留 {len(chosen)} 帧,去掉第 {dropped} 个人物(倒退或重复)")
|
||
for prefix in [cfg["prefix"]] + ([cfg["mirror"][1]] if "mirror" in cfg else []):
|
||
for old in MASTER.glob(f"{prefix}_*.png"):
|
||
old.unlink() # 上一次生成的帧数可能更多
|
||
frames, frame_kps = lock_faces([placed[i][0] for i in chosen], [placed[i][1] for i in chosen])
|
||
if cfg.get("densify"):
|
||
frames, frame_kps, amounts, added = densify(frames, frame_kps, amounts, cfg["densify"])
|
||
print(f" 插帧 {added} 帧")
|
||
if cfg.get("frames"):
|
||
keep = thin(amounts, cfg["frames"])
|
||
frames, frame_kps, amounts = [frames[i] for i in keep], [frame_kps[i] for i in keep], [amounts[i] for i in keep]
|
||
print(f" 最终 {len(frames)} 帧,幅度 {amounts}")
|
||
names = []
|
||
for n, frame in enumerate(frames):
|
||
name = f"{cfg['prefix']}_{n:02d}"
|
||
Image.fromarray(frame, "RGBA").save(MASTER / f"{name}.png")
|
||
names.append(name)
|
||
sequences[cfg["seq"]] = {"frames": names, "amounts": amounts}
|
||
if "mirror" in cfg:
|
||
seq, prefix = cfg["mirror"]
|
||
mirrored = [names[0]] # 共用起始站姿,换边时不会跳
|
||
for n in range(1, len(frames)):
|
||
name = f"{prefix}_{n:02d}"
|
||
Image.fromarray(mirror_frame(frames[n], frame_kps[n]), "RGBA").save(MASTER / f"{name}.png")
|
||
mirrored.append(name)
|
||
sequences[seq] = {"frames": mirrored, "amounts": amounts}
|
||
contact([sequences[cfg["seq"]]["frames"]] + ([sequences[cfg["mirror"][0]]["frames"]] if "mirror" in cfg else []), PREVIEW / f"contact_{move}.jpg")
|
||
|
||
|
||
def contact(rows, out: Path, cell=160):
|
||
cols = max(len(r) for r in rows)
|
||
sheet = Image.new("RGB", (cols * cell, len(rows) * cell), (236, 241, 246))
|
||
for r, names in enumerate(rows):
|
||
for c, n in enumerate(names):
|
||
im = Image.open(MASTER / f"{n}.png").crop((100, 120, 900, 970)).resize((cell * 800 // 850, cell), Image.Resampling.LANCZOS)
|
||
sheet.paste(im, (c * cell + (cell - im.width) // 2, r * cell), im)
|
||
sheet.save(out, quality=88)
|
||
|
||
|
||
def main():
|
||
moves = sys.argv[1:] or list(MOVES)
|
||
MASTER.mkdir(parents=True, exist_ok=True)
|
||
PREVIEW.mkdir(parents=True, exist_ok=True)
|
||
out = WORK / "frames" / "sequences.json"
|
||
sequences = json.loads(out.read_text()) if out.exists() else {}
|
||
for move in moves:
|
||
process(move, MOVES[move], sequences)
|
||
out.write_text(json.dumps(sequences, ensure_ascii=False, indent=2))
|
||
print(f"已写入 {out.relative_to(ROOT)}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|