zyt/TUICallKit-Vue3/training/motion/visionkit-hand.ts

import type { CameraFrame, HandSample, Point2D } from './types'
import { mergeOpenness } from './openness'

type VkSession = {
    start: (cb?: (errno: number | null) => void) => void
    stop?: () => void
    destroy?: () => void
    detectHand?: (opts: Record<string, unknown>) => void
    on?: (event: string, cb: (...args: any[]) => void) => void
    off?: (event: string, cb?: (...args: any[]) => void) => void
}

type WxLike = {
    createVKSession?: (opts: Record<string, unknown>) => VkSession
    getSystemInfoSync?: () => { SDKVersion?: string; platform?: string; brand?: string; model?: string }
    getAppBaseInfo?: () => { SDKVersion?: string }
    getDeviceInfo?: () => { platform?: string; brand?: string; model?: string; system?: string }
}

function getWx(): WxLike | null {
    // #ifdef MP-WEIXIN
    // @ts-ignore
    if (typeof wx !== 'undefined') return wx as WxLike
    // #endif
    return null
}

function parseVersion(v?: string) {
    if (!v) return [0, 0, 0]
    return v.split('.').map((n) => Number(n) || 0)
}

function gteVersion(current: string | undefined, need: string) {
    const a = parseVersion(current)
    const b = parseVersion(need)
    for (let i = 0; i < 3; i++) {
        if (a[i] > b[i]) return true
        if (a[i] < b[i]) return false
    }
    return true
}

function readRuntimeInfo() {
    const wxApi = getWx()
    let SDKVersion = ''
    let platform = ''
    let brand = ''
    let model = ''
    try {
        const base = wxApi?.getAppBaseInfo?.()
        if (base?.SDKVersion) SDKVersion = base.SDKVersion
    } catch (_) {}
    try {
        const device = wxApi?.getDeviceInfo?.()
        if (device?.platform) platform = String(device.platform)
        if (device?.brand) brand = String(device.brand)
        if (device?.model) model = String(device.model)
    } catch (_) {}
    // 兼容旧环境
    if (!SDKVersion || !platform) {
        try {
            const info = wxApi?.getSystemInfoSync?.() || {}
            SDKVersion = SDKVersion || info.SDKVersion || ''
            platform = platform || String(info.platform || '')
            brand = brand || String(info.brand || '')
            model = model || String(info.model || '')
        } catch (_) {}
    }
    return { SDKVersion, platform, brand, model }
}

/** 开发者工具 / 模拟器通常不支持 VisionKit Hand */
export function isLikelyVisionKitUnsupportedHost(): boolean {
    const { platform, brand, model } = readRuntimeInfo()
    const blob = `${platform} ${brand} ${model}`.toLowerCase()
    return (
        blob.includes('devtools') ||
        blob.includes('mac') ||
        blob.includes('windows') ||
        blob.includes('devtools') ||
        platform === 'devtools'
    )
}

export function isVisionKitHandSupported(): boolean {
    const wxApi = getWx()
    if (!wxApi?.createVKSession) return false
    if (isLikelyVisionKitUnsupportedHost()) return false
    const { SDKVersion } = readRuntimeInfo()
    // VisionKit Hand: 基础库 2.28.0+
    return gteVersion(SDKVersion, '2.28.0')
}

export function getVisionKitSupportHint(): string {
    if (isLikelyVisionKitUnsupportedHost()) {
        return '当前是开发者工具/电脑环境，VisionKit 不可用，请用真机预览；已自动降级为 ROI 联调模式'
    }
    if (!getWx()?.createVKSession) {
        return '当前微信不支持 VisionKit'
    }
    const { SDKVersion } = readRuntimeInfo()
    if (!gteVersion(SDKVersion, '2.28.0')) {
        return `基础库 ${SDKVersion || '未知'} 过低，VisionKit Hand 需 ≥ 2.28.0`
    }
    return ''
}

function normalizePoints(raw: any): Point2D[] | undefined {
    if (!Array.isArray(raw) || raw.length === 0) return undefined
    return raw.map((p) => {
        if (Array.isArray(p)) return { x: Number(p[0]) || 0, y: Number(p[1]) || 0 }
        return { x: Number(p?.x) || 0, y: Number(p?.y) || 0 }
    })
}

function isUnsupportedDeviceError(err: unknown) {
    const msg = String((err as any)?.message || err || '')
    return (
        msg.includes('does not support version') ||
        msg.includes('not support') ||
        msg.includes('不支持') ||
        msg.includes('v1')
    )
}

/**
 * VisionKit Hand mode=2：由业务喂入 onCameraFrame 的 RGBA。
 * 内部节流，保证同一时刻最多一次 detectHand。
 */
export function createVisionKitHandDetector(options?: {
    scoreThreshold?: number
    minIntervalMs?: number
}) {
    const scoreThreshold = options?.scoreThreshold ?? 0.5
    const minIntervalMs = options?.minIntervalMs ?? 100 // ~10Hz
    // busy 卡死看门狗：detectHand 后超过该时长无回调则强制释放
    // 注意：手不在画面时 VK 可能整段不回调，这属于正常情况，不能当作会话挂死
    const BUSY_TIMEOUT_MS = 700

    let session: VkSession | null = null
    let started = false
    let busy = false
    let busySince = 0
    let restarting = false
    let sessionStartedAt = 0
    let lastCallbackTs = 0
    let lastDetectTs = 0
    let latestSample: HandSample | null = null
    let onSample: ((sample: HandSample) => void) | null = null

    function handleAnchors(anchors: any[]) {
        busy = false
        lastCallbackTs = Date.now()
        if (!anchors || !anchors.length) {
            latestSample = {
                ts: Date.now(),
                openness: latestSample?.openness ?? 0.5,
                score: 0,
                source: 'visionkit',
            }
            onSample?.(latestSample)
            return
        }

        const sorted = [...anchors].sort((a, b) => (Number(b?.score) || 0) - (Number(a?.score) || 0))
        const best = sorted[0]
        const points = normalizePoints(best?.points)
        const gesture = Number(best?.gesture)
        const score = Number(best?.score) || Number(best?.confidence?.[0]) || 0
        const openness = mergeOpenness({ gesture, points })

        latestSample = {
            ts: Date.now(),
            openness: openness == null ? (latestSample?.openness ?? 0.5) : openness,
            score,
            gesture: Number.isFinite(gesture) ? gesture : undefined,
            points,
            source: 'visionkit',
        }
        onSample?.(latestSample)
    }

    function teardownSession() {
        busy = false
        try {
            session?.stop?.()
        } catch (_) {}
        try {
            session?.destroy?.()
        } catch (_) {}
        session = null
    }

    function createAndStartSession(): Promise<void> {
        const wxApi = getWx()
        return new Promise((resolve, reject) => {
            try {
                session = wxApi!.createVKSession!({
                    track: {
                        hand: { mode: 2 },
                    },
                })
                session.on?.('updateAnchors', (anchors: any[]) => {
                    handleAnchors(anchors || [])
                })
                session.on?.('removeAnchors', () => {
                    busy = false
                    lastCallbackTs = Date.now()
                    latestSample = {
                        ts: Date.now(),
                        openness: latestSample?.openness ?? 0.5,
                        score: 0,
                        source: 'visionkit',
                    }
                    onSample?.(latestSample)
                })
                session.start?.((errno) => {
                    if (errno) {
                        reject(new Error(`VisionKit start 失败: ${errno}`))
                        return
                    }
                    const now = Date.now()
                    started = true
                    busy = false
                    sessionStartedAt = now
                    lastCallbackTs = now
                    resolve()
                })
            } catch (e: any) {
                if (isUnsupportedDeviceError(e)) {
                    reject(new Error(getVisionKitSupportHint() || '当前设备不支持 VisionKit Hand'))
                    return
                }
                reject(e instanceof Error ? e : new Error(String(e)))
            }
        })
    }

    function start(cb: (sample: HandSample) => void): Promise<void> {
        stop()
        onSample = cb
        const wxApi = getWx()
        if (!wxApi?.createVKSession) {
            return Promise.reject(new Error('不支持 VisionKit'))
        }
        if (isLikelyVisionKitUnsupportedHost()) {
            return Promise.reject(new Error(getVisionKitSupportHint()))
        }
        return createAndStartSession()
    }

    function pushFrame(frame: CameraFrame) {
        if (!started || restarting) return
        const now = Date.now()

        // busy 看门狗：detectHand 无回调时强制释放，避免识别永久停摆
        if (busy && now - busySince > BUSY_TIMEOUT_MS) {
            busy = false
        }

        // 不再定期重建 VKSession：长跑中 destroy/create 容易把微信原生桥卡死（表现为 29 次后整页无响应）
        if (!session?.detectHand || busy) return
        if (now - lastDetectTs < minIntervalMs) return
        lastDetectTs = now
        busy = true
        busySince = now
        try {
            session.detectHand({
                frameBuffer: frame.data,
                width: frame.width,
                height: frame.height,
                scoreThreshold,
                algoMode: 2,
            })
        } catch (_) {
            busy = false
        }
    }

    function stop() {
        started = false
        restarting = false
        onSample = null
        sessionStartedAt = 0
        lastCallbackTs = 0
        teardownSession()
    }

    function getLatest() {
        return latestSample
    }

    return { start, stop, pushFrame, getLatest }
}

export type VisionKitHandDetector = ReturnType<typeof createVisionKitHandDetector>