feat(training): 新增握力环 2.0 摄像头手势识别页
用前置摄像头 + VisionKit/ROI 管线驱动开合计数，复用跟练舞台与连击粒子，并加固结束收尾避免末段卡死无响应。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -0,0 +1,296 @@
|
||||
import type { CameraFrame, HandSample, Point2D } from './types'
|
||||
import { mergeOpenness } from './openness'
|
||||
|
||||
type VkSession = {
|
||||
start: (cb?: (errno: number | null) => void) => void
|
||||
stop?: () => void
|
||||
destroy?: () => void
|
||||
detectHand?: (opts: Record<string, unknown>) => void
|
||||
on?: (event: string, cb: (...args: any[]) => void) => void
|
||||
off?: (event: string, cb?: (...args: any[]) => void) => void
|
||||
}
|
||||
|
||||
type WxLike = {
|
||||
createVKSession?: (opts: Record<string, unknown>) => VkSession
|
||||
getSystemInfoSync?: () => { SDKVersion?: string; platform?: string; brand?: string; model?: string }
|
||||
getAppBaseInfo?: () => { SDKVersion?: string }
|
||||
getDeviceInfo?: () => { platform?: string; brand?: string; model?: string; system?: string }
|
||||
}
|
||||
|
||||
function getWx(): WxLike | null {
|
||||
// #ifdef MP-WEIXIN
|
||||
// @ts-ignore
|
||||
if (typeof wx !== 'undefined') return wx as WxLike
|
||||
// #endif
|
||||
return null
|
||||
}
|
||||
|
||||
function parseVersion(v?: string) {
|
||||
if (!v) return [0, 0, 0]
|
||||
return v.split('.').map((n) => Number(n) || 0)
|
||||
}
|
||||
|
||||
function gteVersion(current: string | undefined, need: string) {
|
||||
const a = parseVersion(current)
|
||||
const b = parseVersion(need)
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (a[i] > b[i]) return true
|
||||
if (a[i] < b[i]) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
function readRuntimeInfo() {
|
||||
const wxApi = getWx()
|
||||
let SDKVersion = ''
|
||||
let platform = ''
|
||||
let brand = ''
|
||||
let model = ''
|
||||
try {
|
||||
const base = wxApi?.getAppBaseInfo?.()
|
||||
if (base?.SDKVersion) SDKVersion = base.SDKVersion
|
||||
} catch (_) {}
|
||||
try {
|
||||
const device = wxApi?.getDeviceInfo?.()
|
||||
if (device?.platform) platform = String(device.platform)
|
||||
if (device?.brand) brand = String(device.brand)
|
||||
if (device?.model) model = String(device.model)
|
||||
} catch (_) {}
|
||||
// 兼容旧环境
|
||||
if (!SDKVersion || !platform) {
|
||||
try {
|
||||
const info = wxApi?.getSystemInfoSync?.() || {}
|
||||
SDKVersion = SDKVersion || info.SDKVersion || ''
|
||||
platform = platform || String(info.platform || '')
|
||||
brand = brand || String(info.brand || '')
|
||||
model = model || String(info.model || '')
|
||||
} catch (_) {}
|
||||
}
|
||||
return { SDKVersion, platform, brand, model }
|
||||
}
|
||||
|
||||
/** 开发者工具 / 模拟器通常不支持 VisionKit Hand */
|
||||
export function isLikelyVisionKitUnsupportedHost(): boolean {
|
||||
const { platform, brand, model } = readRuntimeInfo()
|
||||
const blob = `${platform} ${brand} ${model}`.toLowerCase()
|
||||
return (
|
||||
blob.includes('devtools') ||
|
||||
blob.includes('mac') ||
|
||||
blob.includes('windows') ||
|
||||
blob.includes('devtools') ||
|
||||
platform === 'devtools'
|
||||
)
|
||||
}
|
||||
|
||||
export function isVisionKitHandSupported(): boolean {
|
||||
const wxApi = getWx()
|
||||
if (!wxApi?.createVKSession) return false
|
||||
if (isLikelyVisionKitUnsupportedHost()) return false
|
||||
const { SDKVersion } = readRuntimeInfo()
|
||||
// VisionKit Hand: 基础库 2.28.0+
|
||||
return gteVersion(SDKVersion, '2.28.0')
|
||||
}
|
||||
|
||||
export function getVisionKitSupportHint(): string {
|
||||
if (isLikelyVisionKitUnsupportedHost()) {
|
||||
return '当前是开发者工具/电脑环境，VisionKit 不可用，请用真机预览；已自动降级为 ROI 联调模式'
|
||||
}
|
||||
if (!getWx()?.createVKSession) {
|
||||
return '当前微信不支持 VisionKit'
|
||||
}
|
||||
const { SDKVersion } = readRuntimeInfo()
|
||||
if (!gteVersion(SDKVersion, '2.28.0')) {
|
||||
return `基础库 ${SDKVersion || '未知'} 过低，VisionKit Hand 需 ≥ 2.28.0`
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function normalizePoints(raw: any): Point2D[] | undefined {
|
||||
if (!Array.isArray(raw) || raw.length === 0) return undefined
|
||||
return raw.map((p) => {
|
||||
if (Array.isArray(p)) return { x: Number(p[0]) || 0, y: Number(p[1]) || 0 }
|
||||
return { x: Number(p?.x) || 0, y: Number(p?.y) || 0 }
|
||||
})
|
||||
}
|
||||
|
||||
function isUnsupportedDeviceError(err: unknown) {
|
||||
const msg = String((err as any)?.message || err || '')
|
||||
return (
|
||||
msg.includes('does not support version') ||
|
||||
msg.includes('not support') ||
|
||||
msg.includes('不支持') ||
|
||||
msg.includes('v1')
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* VisionKit Hand mode=2：由业务喂入 onCameraFrame 的 RGBA。
|
||||
* 内部节流，保证同一时刻最多一次 detectHand。
|
||||
*/
|
||||
export function createVisionKitHandDetector(options?: {
|
||||
scoreThreshold?: number
|
||||
minIntervalMs?: number
|
||||
}) {
|
||||
const scoreThreshold = options?.scoreThreshold ?? 0.5
|
||||
const minIntervalMs = options?.minIntervalMs ?? 100 // ~10Hz
|
||||
// busy 卡死看门狗：detectHand 后超过该时长无回调则强制释放
|
||||
// 注意：手不在画面时 VK 可能整段不回调，这属于正常情况，不能当作会话挂死
|
||||
const BUSY_TIMEOUT_MS = 700
|
||||
|
||||
let session: VkSession | null = null
|
||||
let started = false
|
||||
let busy = false
|
||||
let busySince = 0
|
||||
let restarting = false
|
||||
let sessionStartedAt = 0
|
||||
let lastCallbackTs = 0
|
||||
let lastDetectTs = 0
|
||||
let latestSample: HandSample | null = null
|
||||
let onSample: ((sample: HandSample) => void) | null = null
|
||||
|
||||
function handleAnchors(anchors: any[]) {
|
||||
busy = false
|
||||
lastCallbackTs = Date.now()
|
||||
if (!anchors || !anchors.length) {
|
||||
latestSample = {
|
||||
ts: Date.now(),
|
||||
openness: latestSample?.openness ?? 0.5,
|
||||
score: 0,
|
||||
source: 'visionkit',
|
||||
}
|
||||
onSample?.(latestSample)
|
||||
return
|
||||
}
|
||||
|
||||
const sorted = [...anchors].sort((a, b) => (Number(b?.score) || 0) - (Number(a?.score) || 0))
|
||||
const best = sorted[0]
|
||||
const points = normalizePoints(best?.points)
|
||||
const gesture = Number(best?.gesture)
|
||||
const score = Number(best?.score) || Number(best?.confidence?.[0]) || 0
|
||||
const openness = mergeOpenness({ gesture, points })
|
||||
|
||||
latestSample = {
|
||||
ts: Date.now(),
|
||||
openness: openness == null ? (latestSample?.openness ?? 0.5) : openness,
|
||||
score,
|
||||
gesture: Number.isFinite(gesture) ? gesture : undefined,
|
||||
points,
|
||||
source: 'visionkit',
|
||||
}
|
||||
onSample?.(latestSample)
|
||||
}
|
||||
|
||||
function teardownSession() {
|
||||
busy = false
|
||||
try {
|
||||
session?.stop?.()
|
||||
} catch (_) {}
|
||||
try {
|
||||
session?.destroy?.()
|
||||
} catch (_) {}
|
||||
session = null
|
||||
}
|
||||
|
||||
function createAndStartSession(): Promise<void> {
|
||||
const wxApi = getWx()
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
session = wxApi!.createVKSession!({
|
||||
track: {
|
||||
hand: { mode: 2 },
|
||||
},
|
||||
})
|
||||
session.on?.('updateAnchors', (anchors: any[]) => {
|
||||
handleAnchors(anchors || [])
|
||||
})
|
||||
session.on?.('removeAnchors', () => {
|
||||
busy = false
|
||||
lastCallbackTs = Date.now()
|
||||
latestSample = {
|
||||
ts: Date.now(),
|
||||
openness: latestSample?.openness ?? 0.5,
|
||||
score: 0,
|
||||
source: 'visionkit',
|
||||
}
|
||||
onSample?.(latestSample)
|
||||
})
|
||||
session.start?.((errno) => {
|
||||
if (errno) {
|
||||
reject(new Error(`VisionKit start 失败: ${errno}`))
|
||||
return
|
||||
}
|
||||
const now = Date.now()
|
||||
started = true
|
||||
busy = false
|
||||
sessionStartedAt = now
|
||||
lastCallbackTs = now
|
||||
resolve()
|
||||
})
|
||||
} catch (e: any) {
|
||||
if (isUnsupportedDeviceError(e)) {
|
||||
reject(new Error(getVisionKitSupportHint() || '当前设备不支持 VisionKit Hand'))
|
||||
return
|
||||
}
|
||||
reject(e instanceof Error ? e : new Error(String(e)))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function start(cb: (sample: HandSample) => void): Promise<void> {
|
||||
stop()
|
||||
onSample = cb
|
||||
const wxApi = getWx()
|
||||
if (!wxApi?.createVKSession) {
|
||||
return Promise.reject(new Error('不支持 VisionKit'))
|
||||
}
|
||||
if (isLikelyVisionKitUnsupportedHost()) {
|
||||
return Promise.reject(new Error(getVisionKitSupportHint()))
|
||||
}
|
||||
return createAndStartSession()
|
||||
}
|
||||
|
||||
function pushFrame(frame: CameraFrame) {
|
||||
if (!started || restarting) return
|
||||
const now = Date.now()
|
||||
|
||||
// busy 看门狗：detectHand 无回调时强制释放，避免识别永久停摆
|
||||
if (busy && now - busySince > BUSY_TIMEOUT_MS) {
|
||||
busy = false
|
||||
}
|
||||
|
||||
// 不再定期重建 VKSession：长跑中 destroy/create 容易把微信原生桥卡死（表现为 29 次后整页无响应）
|
||||
if (!session?.detectHand || busy) return
|
||||
if (now - lastDetectTs < minIntervalMs) return
|
||||
lastDetectTs = now
|
||||
busy = true
|
||||
busySince = now
|
||||
try {
|
||||
session.detectHand({
|
||||
frameBuffer: frame.data,
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
scoreThreshold,
|
||||
algoMode: 2,
|
||||
})
|
||||
} catch (_) {
|
||||
busy = false
|
||||
}
|
||||
}
|
||||
|
||||
function stop() {
|
||||
started = false
|
||||
restarting = false
|
||||
onSample = null
|
||||
sessionStartedAt = 0
|
||||
lastCallbackTs = 0
|
||||
teardownSession()
|
||||
}
|
||||
|
||||
function getLatest() {
|
||||
return latestSample
|
||||
}
|
||||
|
||||
return { start, stop, pushFrame, getLatest }
|
||||
}
|
||||
|
||||
export type VisionKitHandDetector = ReturnType<typeof createVisionKitHandDetector>
|
||||
Reference in New Issue
Block a user