long
99388aa58b
诊断面板和 console 识别摘要改为编译时设置 VITE_POSE_DIAG=1 才开启（与远程日志 VITE_POSE_LOG_URL 一致）， 开发版平时不再显示，需要真机排查时再打开。
323 lines
13 KiB
TypeScript
323 lines
13 KiB
TypeScript
/**
|
||
* 摄像头取帧 + VisionKit 人体关键点检测
|
||
*
|
||
* 方案：<camera> 负责预览（原生、流畅），CameraContext.onCameraFrame 拿 RGBA 帧，
|
||
* 节流后交给 VKSession.detectBody（mode 2，手动传图），结果通过 updateAnchors 回调。
|
||
* 这样不需要自己用 WebGL 渲染相机画面；<camera> 放在哪个组件里都可以（一个页面只有一个摄像头）。
|
||
*
|
||
* 要求：微信基础库 >= 2.28.0；非微信端或不支持的机型会返回 unsupported。
|
||
*/
|
||
import { ref, shallowRef, onUnmounted } from 'vue'
|
||
import { anchorToPose, smoothPose, type Pose } from './keypoints.ts'
|
||
import { poseDiagEnabled, rlog } from './remoteLog.ts'
|
||
|
||
export type CameraStatus = 'idle' | 'starting' | 'running' | 'unsupported' | 'denied' | 'error'
|
||
|
||
/** 检测间隔：约 10fps，兼顾流畅与发热 */
|
||
const DETECT_INTERVAL_MS = 90
|
||
/** 检测请求超时：没有回调视为画面里没人 */
|
||
const DETECT_TIMEOUT_MS = 600
|
||
/** 连续多久没检测到人，清空姿态 */
|
||
const LOST_AFTER_MS = 800
|
||
|
||
/** 诊断面板刷新间隔；console 摘要间隔 */
|
||
const DIAG_REFRESH_MS = 500
|
||
const DIAG_LOG_MS = 2000
|
||
|
||
type FrameInfo = { width: number; height: number }
|
||
type AnchorSource = 'add' | 'update' | 'remove' | 'timeout'
|
||
|
||
/** 真机排查用：识别引擎的事件计数和最近一次识别到人的原始数据摘要（开发版跟练页显示） */
|
||
export type PoseDiag = {
|
||
events: Record<AnchorSource | 'error', number>
|
||
/** detectBody 发出到收到结果的耗时 */
|
||
latencyMs: number
|
||
/** 最近一次识别到人的原始数据摘要 */
|
||
raw: string
|
||
}
|
||
|
||
|
||
export function usePoseCamera(onPose?: (pose: Pose | null) => void) {
|
||
const status = ref<CameraStatus>('idle')
|
||
const errorText = ref('')
|
||
const pose = shallowRef<Pose | null>(null)
|
||
const frame = ref<FrameInfo>({ width: 480, height: 640 })
|
||
/** 实际检测帧率，调试用 */
|
||
const fps = ref(0)
|
||
const diag = shallowRef<PoseDiag>(emptyDiag())
|
||
const events = { ...emptyDiag().events }
|
||
let latencyMs = 0
|
||
let lastRaw = ''
|
||
let firstAnchorLogged = false
|
||
let lastDiagAt = 0
|
||
let lastLogAt = 0
|
||
|
||
let session: any = null
|
||
let listener: any = null
|
||
let pending = false
|
||
let pendingTimer: ReturnType<typeof setTimeout> | null = null
|
||
let lastSent = 0
|
||
let lastSeen = 0
|
||
let sentFrame: FrameInfo = { width: 0, height: 0 }
|
||
let fpsCount = 0
|
||
let fpsTimer: ReturnType<typeof setInterval> | null = null
|
||
|
||
function emit(next: Pose | null) {
|
||
pose.value = next
|
||
onPose?.(next)
|
||
}
|
||
|
||
function clearPending() {
|
||
pending = false
|
||
if (pendingTimer) {
|
||
clearTimeout(pendingTimer)
|
||
pendingTimer = null
|
||
}
|
||
}
|
||
|
||
function handleAnchors(anchors: any[], source: AnchorSource) {
|
||
// 已经停止（训练结束、切到领练）后迟到的识别结果直接丢弃
|
||
if (!session) return
|
||
const now = Date.now()
|
||
if (pending && (source === 'add' || source === 'update')) latencyMs = now - lastSent
|
||
events[source] += 1
|
||
clearPending()
|
||
fpsCount += 1
|
||
// 画面里多人时取检测框最大的那个
|
||
const best = (anchors || []).slice().sort((a, b) => area(b) - area(a))[0]
|
||
rlog('det', {
|
||
src: source,
|
||
lat: source === 'add' || source === 'update' ? latencyMs : undefined,
|
||
n: anchors?.length ?? 0,
|
||
// 原始坐标不是有效数字的点（画面外的点真机返回 NaN）
|
||
nan: best ? nanPoints(best) : undefined,
|
||
})
|
||
if (best) recordRaw(best)
|
||
const next = best ? anchorToPose(best, sentFrame.width, sentFrame.height, now) : null
|
||
if (next) {
|
||
lastSeen = now
|
||
emit(smoothPose(pose.value, next))
|
||
} else if (now - lastSeen > LOST_AFTER_MS) {
|
||
emit(null)
|
||
}
|
||
refreshDiag(now)
|
||
}
|
||
|
||
/** 记下原始 anchor 的摘要；第一次识别到人时把完整原始数据打到 console，便于核对字段格式 */
|
||
function recordRaw(anchor: any) {
|
||
if (poseDiagEnabled && !firstAnchorLogged) {
|
||
firstAnchorLogged = true
|
||
// anchor 是原生对象，字段不可枚举，直接 JSON.stringify 得到 {}；逐个取出再序列化，NaN 显式写出来
|
||
const json = JSON.stringify(plainAnchor(anchor))
|
||
console.log('[跟练诊断] 首个原始 anchor', `帧 ${sentFrame.width}x${sentFrame.height}`, json.slice(0, 1500))
|
||
rlog('first_anchor', { frame: `${sentFrame.width}x${sentFrame.height}`, anchor: json.slice(0, 4000) })
|
||
}
|
||
lastRaw = summarizeAnchor(anchor)
|
||
}
|
||
|
||
function refreshDiag(now: number) {
|
||
if (now - lastDiagAt < DIAG_REFRESH_MS) return
|
||
lastDiagAt = now
|
||
diag.value = { events: { ...events }, latencyMs, raw: lastRaw }
|
||
if (poseDiagEnabled && now - lastLogAt >= DIAG_LOG_MS) {
|
||
lastLogAt = now
|
||
const e = events
|
||
console.log(
|
||
`[跟练诊断] 帧 ${frame.value.width}x${frame.value.height} 延迟 ${latencyMs}ms 事件 add${e.add} upd${e.update} rm${e.remove} 超时${e.timeout} 错${e.error} | ${lastRaw || '尚未识别到人'}`,
|
||
)
|
||
}
|
||
}
|
||
|
||
function area(a: any) {
|
||
const s = a?.size
|
||
if (!s) return 0
|
||
return Array.isArray(s) ? s[0] * s[1] : (s.width || 0) * (s.height || 0)
|
||
}
|
||
|
||
/** 相机组件 initdone 后调用 */
|
||
function start() {
|
||
if (status.value === 'running' || status.value === 'starting') return
|
||
status.value = 'starting'
|
||
Object.assign(events, emptyDiag().events)
|
||
latencyMs = 0
|
||
lastRaw = ''
|
||
firstAnchorLogged = false
|
||
diag.value = emptyDiag()
|
||
rlog('camera_start', deviceInfo())
|
||
// #ifdef MP-WEIXIN
|
||
startVisionKit()
|
||
// #endif
|
||
// #ifndef MP-WEIXIN
|
||
status.value = 'unsupported'
|
||
// #endif
|
||
}
|
||
|
||
function startVisionKit() {
|
||
try {
|
||
const w: any = wx
|
||
if (typeof w.createVKSession !== 'function' || (w.isVKSupport && !w.isVKSupport('v1'))) {
|
||
status.value = 'unsupported'
|
||
rlog('vk_unsupported', { hasCreate: typeof w.createVKSession === 'function' })
|
||
return
|
||
}
|
||
session = w.createVKSession({ track: { body: { mode: 2 } }, version: 'v1' })
|
||
// 连续帧（sourceType 0）会跟踪人体：首次识别到人、丢失后重新出现走 addAnchors，之后每帧走 updateAnchors（同官方示例）
|
||
session.on('addAnchors', (anchors: any[]) => handleAnchors(anchors, 'add'))
|
||
session.on('updateAnchors', (anchors: any[]) => handleAnchors(anchors, 'update'))
|
||
session.on('removeAnchors', () => handleAnchors([], 'remove'))
|
||
session.start((err: any) => {
|
||
if (err) {
|
||
status.value = 'unsupported'
|
||
errorText.value = String(err?.errMsg || err)
|
||
rlog('vk_start_error', { err: errorText.value })
|
||
return
|
||
}
|
||
rlog('vk_started')
|
||
startFrames()
|
||
})
|
||
} catch (e: any) {
|
||
status.value = 'unsupported'
|
||
errorText.value = String(e?.message || e)
|
||
rlog('vk_exception', { err: errorText.value })
|
||
}
|
||
}
|
||
|
||
function startFrames() {
|
||
const cam: any = uni.createCameraContext()
|
||
if (!cam?.onCameraFrame) {
|
||
status.value = 'unsupported'
|
||
rlog('no_camera_frame_api')
|
||
return
|
||
}
|
||
listener = cam.onCameraFrame((f: { data: ArrayBuffer; width: number; height: number }) => {
|
||
const now = Date.now()
|
||
if (pending || now - lastSent < DETECT_INTERVAL_MS || !session) return
|
||
if (frame.value.width !== f.width || frame.value.height !== f.height) {
|
||
frame.value = { width: f.width, height: f.height }
|
||
rlog('frame_size', { w: f.width, h: f.height, bytes: f.data?.byteLength })
|
||
}
|
||
pending = true
|
||
lastSent = now
|
||
sentFrame = { width: f.width, height: f.height }
|
||
pendingTimer = setTimeout(() => handleAnchors([], 'timeout'), DETECT_TIMEOUT_MS)
|
||
try {
|
||
session.detectBody({
|
||
frameBuffer: f.data,
|
||
width: f.width,
|
||
height: f.height,
|
||
scoreThreshold: 0.5,
|
||
// 0 表示输入来自视频连续帧，算法会做帧间跟踪
|
||
sourceType: 0,
|
||
})
|
||
} catch (e) {
|
||
events.error += 1
|
||
if (poseDiagEnabled) console.warn('[跟练诊断] detectBody 异常', e)
|
||
rlog('detect_err', { err: String((e as any)?.message || e) })
|
||
clearPending()
|
||
}
|
||
})
|
||
listener.start({
|
||
success: () => {
|
||
status.value = 'running'
|
||
rlog('frames_started')
|
||
fpsTimer = setInterval(() => {
|
||
fps.value = fpsCount
|
||
fpsCount = 0
|
||
}, 1000)
|
||
},
|
||
fail: (e: any) => {
|
||
status.value = 'error'
|
||
errorText.value = String(e?.errMsg || e)
|
||
rlog('frames_fail', { err: errorText.value })
|
||
},
|
||
})
|
||
}
|
||
|
||
function stop() {
|
||
clearPending()
|
||
try {
|
||
listener?.stop?.()
|
||
} catch (e) {}
|
||
listener = null
|
||
const oldSession = session
|
||
session = null
|
||
rlog('camera_stop')
|
||
// 停止可能发生在识别引擎自己的回调里（第 N 次计数 → 结束 → 关摄像头）：在回调里同步销毁引擎，iOS 真机会卡死。
|
||
// 先断开引用（之后迟到的回调全部丢弃），等当前回调返回后再销毁
|
||
setTimeout(() => {
|
||
try {
|
||
oldSession?.stop?.()
|
||
oldSession?.destroy?.()
|
||
} catch (e) {}
|
||
}, 0)
|
||
if (fpsTimer) clearInterval(fpsTimer)
|
||
fpsTimer = null
|
||
if (status.value === 'running' || status.value === 'starting') status.value = 'idle'
|
||
pose.value = null
|
||
}
|
||
|
||
/** <camera @error>：通常是用户拒绝了摄像头权限 */
|
||
function onCameraError(e: any) {
|
||
status.value = 'denied'
|
||
errorText.value = String(e?.detail?.errMsg || '')
|
||
}
|
||
|
||
onUnmounted(stop)
|
||
|
||
return { status, errorText, pose, frame, fps, diag, start, stop, onCameraError }
|
||
}
|
||
|
||
const num = (v: unknown) => (typeof v === 'number' ? (Number.isFinite(v) ? +v.toFixed(4) : String(v)) : v)
|
||
const rawXY = (p: any) => (Array.isArray(p) ? [p[0], p[1]] : [p?.x, p?.y])
|
||
|
||
/** 原始坐标不是有效数字的点的序号，如 "13,14,15,16" */
|
||
function nanPoints(anchor: any): string {
|
||
const pts: any[] = Array.isArray(anchor?.points) ? anchor.points : []
|
||
return pts
|
||
.map((p, i) => (rawXY(p).every((v) => typeof v === 'number' && Number.isFinite(v)) ? -1 : i))
|
||
.filter((i) => i >= 0)
|
||
.join(',')
|
||
}
|
||
|
||
function plainAnchor(a: any) {
|
||
return {
|
||
keys: Object.keys(a ?? {}),
|
||
score: num(a?.score),
|
||
origin: a?.origin ? [num(a.origin.x), num(a.origin.y)] : a?.origin,
|
||
size: a?.size ? [num(a.size.width), num(a.size.height)] : a?.size,
|
||
points: Array.isArray(a?.points) ? a.points.map((p: any) => rawXY(p).map(num)) : typeof a?.points,
|
||
confidence: Array.isArray(a?.confidence) ? a.confidence.map(num) : typeof a?.confidence,
|
||
}
|
||
}
|
||
|
||
/** 机型与微信版本，排查机型差异用 */
|
||
function deviceInfo() {
|
||
try {
|
||
const s: any = uni.getSystemInfoSync()
|
||
return { brand: s.brand, model: s.model, system: s.system, platform: s.platform, wx: s.version, sdk: s.SDKVersion }
|
||
} catch (e) {
|
||
return {}
|
||
}
|
||
}
|
||
|
||
function emptyDiag(): PoseDiag {
|
||
return { events: { add: 0, update: 0, remove: 0, timeout: 0, error: 0 }, latencyMs: 0, raw: '' }
|
||
}
|
||
|
||
const f2 = (v: unknown) => (typeof v === 'number' ? v.toFixed(2) : String(v))
|
||
|
||
/** 原始 anchor 摘要：点数、坐标是比例还是像素、置信度范围，以及鼻子 / 髋 / 踝的原始坐标 */
|
||
function summarizeAnchor(a: any): string {
|
||
const pts: any[] = Array.isArray(a?.points) ? a.points : []
|
||
const xy = (i: number) => {
|
||
const p = pts[i]
|
||
if (!p) return '-'
|
||
return Array.isArray(p) ? `${f2(p[0])},${f2(p[1])}` : `${f2(p.x)},${f2(p.y)}`
|
||
}
|
||
const flat = pts.flatMap((p) => (Array.isArray(p) ? p : [p?.x, p?.y])).filter((v) => typeof v === 'number')
|
||
const unit = flat.some((v) => v > 1.5) ? '像素' : '比例'
|
||
const conf: number[] = Array.isArray(a?.confidence) ? a.confidence : []
|
||
const confText = conf.length ? `${f2(Math.min(...conf))}~${f2(Math.max(...conf))}` : '无'
|
||
return `${pts.length}点 ${unit} 框分${f2(a?.score)} 点置信${confText} 鼻${xy(0)} 左髋${xy(11)} 右髋${xy(12)} 左踝${xy(15)} 右踝${xy(16)}`
|
||
}
|