import type { CameraFrame, HandSample, Point2D } from './types' import { mergeOpenness } from './openness' type VkSession = { start: (cb?: (errno: number | null) => void) => void stop?: () => void destroy?: () => void detectHand?: (opts: Record) => void on?: (event: string, cb: (...args: any[]) => void) => void off?: (event: string, cb?: (...args: any[]) => void) => void } type WxLike = { createVKSession?: (opts: Record) => VkSession getSystemInfoSync?: () => { SDKVersion?: string; platform?: string; brand?: string; model?: string } getAppBaseInfo?: () => { SDKVersion?: string } getDeviceInfo?: () => { platform?: string; brand?: string; model?: string; system?: string } } function getWx(): WxLike | null { // #ifdef MP-WEIXIN // @ts-ignore if (typeof wx !== 'undefined') return wx as WxLike // #endif return null } function parseVersion(v?: string) { if (!v) return [0, 0, 0] return v.split('.').map((n) => Number(n) || 0) } function gteVersion(current: string | undefined, need: string) { const a = parseVersion(current) const b = parseVersion(need) for (let i = 0; i < 3; i++) { if (a[i] > b[i]) return true if (a[i] < b[i]) return false } return true } function readRuntimeInfo() { const wxApi = getWx() let SDKVersion = '' let platform = '' let brand = '' let model = '' try { const base = wxApi?.getAppBaseInfo?.() if (base?.SDKVersion) SDKVersion = base.SDKVersion } catch (_) {} try { const device = wxApi?.getDeviceInfo?.() if (device?.platform) platform = String(device.platform) if (device?.brand) brand = String(device.brand) if (device?.model) model = String(device.model) } catch (_) {} // 兼容旧环境 if (!SDKVersion || !platform) { try { const info = wxApi?.getSystemInfoSync?.() || {} SDKVersion = SDKVersion || info.SDKVersion || '' platform = platform || String(info.platform || '') brand = brand || String(info.brand || '') model = model || String(info.model || '') } catch (_) {} } return { SDKVersion, platform, brand, model } } /** 开发者工具 / 模拟器通常不支持 VisionKit Hand */ export function isLikelyVisionKitUnsupportedHost(): boolean { const { platform, brand, model } = readRuntimeInfo() const blob = `${platform} ${brand} ${model}`.toLowerCase() return ( blob.includes('devtools') || blob.includes('mac') || blob.includes('windows') || blob.includes('devtools') || platform === 'devtools' ) } export function isVisionKitHandSupported(): boolean { const wxApi = getWx() if (!wxApi?.createVKSession) return false if (isLikelyVisionKitUnsupportedHost()) return false const { SDKVersion } = readRuntimeInfo() // VisionKit Hand: 基础库 2.28.0+ return gteVersion(SDKVersion, '2.28.0') } export function getVisionKitSupportHint(): string { if (isLikelyVisionKitUnsupportedHost()) { return '当前是开发者工具/电脑环境,VisionKit 不可用,请用真机预览;已自动降级为 ROI 联调模式' } if (!getWx()?.createVKSession) { return '当前微信不支持 VisionKit' } const { SDKVersion } = readRuntimeInfo() if (!gteVersion(SDKVersion, '2.28.0')) { return `基础库 ${SDKVersion || '未知'} 过低,VisionKit Hand 需 ≥ 2.28.0` } return '' } function normalizePoints(raw: any): Point2D[] | undefined { if (!Array.isArray(raw) || raw.length === 0) return undefined return raw.map((p) => { if (Array.isArray(p)) return { x: Number(p[0]) || 0, y: Number(p[1]) || 0 } return { x: Number(p?.x) || 0, y: Number(p?.y) || 0 } }) } function isUnsupportedDeviceError(err: unknown) { const msg = String((err as any)?.message || err || '') return ( msg.includes('does not support version') || msg.includes('not support') || msg.includes('不支持') || msg.includes('v1') ) } /** * VisionKit Hand mode=2:由业务喂入 onCameraFrame 的 RGBA。 * 内部节流,保证同一时刻最多一次 detectHand。 */ export function createVisionKitHandDetector(options?: { scoreThreshold?: number minIntervalMs?: number }) { const scoreThreshold = options?.scoreThreshold ?? 0.5 const minIntervalMs = options?.minIntervalMs ?? 100 // ~10Hz // busy 卡死看门狗:detectHand 后超过该时长无回调则强制释放 // 注意:手不在画面时 VK 可能整段不回调,这属于正常情况,不能当作会话挂死 const BUSY_TIMEOUT_MS = 700 let session: VkSession | null = null let started = false let busy = false let busySince = 0 let restarting = false let sessionStartedAt = 0 let lastCallbackTs = 0 let lastDetectTs = 0 let latestSample: HandSample | null = null let onSample: ((sample: HandSample) => void) | null = null function handleAnchors(anchors: any[]) { busy = false lastCallbackTs = Date.now() if (!anchors || !anchors.length) { latestSample = { ts: Date.now(), openness: latestSample?.openness ?? 0.5, score: 0, source: 'visionkit', } onSample?.(latestSample) return } const sorted = [...anchors].sort((a, b) => (Number(b?.score) || 0) - (Number(a?.score) || 0)) const best = sorted[0] const points = normalizePoints(best?.points) const gesture = Number(best?.gesture) const score = Number(best?.score) || Number(best?.confidence?.[0]) || 0 const openness = mergeOpenness({ gesture, points }) latestSample = { ts: Date.now(), openness: openness == null ? (latestSample?.openness ?? 0.5) : openness, score, gesture: Number.isFinite(gesture) ? gesture : undefined, points, source: 'visionkit', } onSample?.(latestSample) } function teardownSession() { busy = false try { session?.stop?.() } catch (_) {} try { session?.destroy?.() } catch (_) {} session = null } function createAndStartSession(): Promise { const wxApi = getWx() return new Promise((resolve, reject) => { try { session = wxApi!.createVKSession!({ track: { hand: { mode: 2 }, }, }) session.on?.('updateAnchors', (anchors: any[]) => { handleAnchors(anchors || []) }) session.on?.('removeAnchors', () => { busy = false lastCallbackTs = Date.now() latestSample = { ts: Date.now(), openness: latestSample?.openness ?? 0.5, score: 0, source: 'visionkit', } onSample?.(latestSample) }) session.start?.((errno) => { if (errno) { reject(new Error(`VisionKit start 失败: ${errno}`)) return } const now = Date.now() started = true busy = false sessionStartedAt = now lastCallbackTs = now resolve() }) } catch (e: any) { if (isUnsupportedDeviceError(e)) { reject(new Error(getVisionKitSupportHint() || '当前设备不支持 VisionKit Hand')) return } reject(e instanceof Error ? e : new Error(String(e))) } }) } function start(cb: (sample: HandSample) => void): Promise { stop() onSample = cb const wxApi = getWx() if (!wxApi?.createVKSession) { return Promise.reject(new Error('不支持 VisionKit')) } if (isLikelyVisionKitUnsupportedHost()) { return Promise.reject(new Error(getVisionKitSupportHint())) } return createAndStartSession() } function pushFrame(frame: CameraFrame) { if (!started || restarting) return const now = Date.now() // busy 看门狗:detectHand 无回调时强制释放,避免识别永久停摆 if (busy && now - busySince > BUSY_TIMEOUT_MS) { busy = false } // 不再定期重建 VKSession:长跑中 destroy/create 容易把微信原生桥卡死(表现为 29 次后整页无响应) if (!session?.detectHand || busy) return if (now - lastDetectTs < minIntervalMs) return lastDetectTs = now busy = true busySince = now try { session.detectHand({ frameBuffer: frame.data, width: frame.width, height: frame.height, scoreThreshold, algoMode: 2, }) } catch (_) { busy = false } } function stop() { started = false restarting = false onSample = null sessionStartedAt = 0 lastCallbackTs = 0 teardownSession() } function getLatest() { return latestSample } return { start, stop, pushFrame, getLatest } } export type VisionKitHandDetector = ReturnType