zyt/TUICallKit-Vue3/training/motion/gesture-pipeline.ts

import { createCameraFramePump } from './camera-frames'
import { createOpenCloseClassifier } from './open-close-classifier'
import { estimateOpennessFromRoi } from './roi-openness'
import type { GestureObservation, GestureSource, RepCompletedInfo } from './types'
import {
    createVisionKitHandDetector,
    getVisionKitSupportHint,
    isLikelyVisionKitUnsupportedHost,
    isVisionKitHandSupported,
} from './visionkit-hand'

export interface GesturePipelineHandlers {
    onObservation: (obs: GestureObservation) => void
    onRepCompleted?: (info: RepCompletedInfo) => void
    /** 非致命提示（如已降级 ROI） */
    onWarning?: (message: string) => void
    /** 致命错误（相机帧都起不来） */
    onError?: (message: string) => void
}

/**
 * 编排：优先 VisionKit；开发者工具/不支持设备自动降级 ROI 联调。
 */
export function createGesturePipeline(handlers: GesturePipelineHandlers) {
    const pump = createCameraFramePump()
    // ~10Hz；置信度适中，减少噪声帧
    const detector = createVisionKitHandDetector({ minIntervalMs: 100, scoreThreshold: 0.28 })
    // 握力环校准：半开即可计张开，相对抬升即可完成松开
    const classifier = createOpenCloseClassifier({
        openEnter: 0.45,
        openExit: 0.38,
        closeEnter: 0.36,
        closeExit: 0.42,
        emaAlpha: 0.42,
        minHoldMs: 120,
        lostMs: 1400,
        reopenRise: 0.08,
    })

    let active = false
    let paused = false
    let mode: GestureSource = 'none'
    let lastRoiTs = 0
    let lastUiEmitTs = 0
    let lastEmittedPhase = ''
    let lastEmittedCompression = -1
    let lastEmittedHold = -1
    let lastSyntheticTs = 0
    const UI_EMIT_MIN_MS = 100

    function emitFromSample(sample: Parameters<typeof classifier.update>[0]) {
        if (!active || paused) return
        const out = classifier.update(sample)
        const now = Date.now()
        const phaseChanged = out.phase !== lastEmittedPhase
        const due = now - lastUiEmitTs >= UI_EMIT_MIN_MS
        const compressionDelta = Math.abs(out.compression - lastEmittedCompression) >= 0.08
        const holdDelta = Math.abs(out.holdProgress - lastEmittedHold) >= 0.12

        // 计次始终即时；UI 观察节流，避免小程序每帧 setData 卡顿
        if (out.rep) {
            handlers.onRepCompleted?.(out.rep)
        }

        if (!phaseChanged && !due && !compressionDelta && !holdDelta && !out.rep) {
            return
        }

        lastUiEmitTs = now
        lastEmittedPhase = out.phase
        lastEmittedCompression = out.compression
        lastEmittedHold = out.holdProgress

        handlers.onObservation({
            ts: now,
            openness: out.openness,
            compression: out.compression,
            phase: out.phase,
            trackingQuality: out.trackingQuality,
            source: out.source,
            holdProgress: out.holdProgress,
            fps: pump.getFps(),
        })
    }

    async function startVisionKit(): Promise<boolean> {
        if (!isVisionKitHandSupported()) return false
        try {
            await detector.start((sample) => emitFromSample(sample))
            mode = 'visionkit'
            return true
        } catch (_) {
            detector.stop()
            return false
        }
    }

    function startRoiFallback(reason?: string) {
        mode = 'roi'
        const hint = reason || getVisionKitSupportHint() || '已降级为 ROI 联调模式（非真机手势识别）'
        handlers.onWarning?.(hint)
        pump.start((frame) => {
            if (!active || paused) return
            const now = Date.now()
            if (now - lastRoiTs < 160) return
            lastRoiTs = now
            emitFromSample(estimateOpennessFromRoi(frame))
        })
    }

    async function start() {
        stop()

        // 开发者工具直接走 ROI，避免 createVKSession 抛 “does not support version v1”
        const forceRoi = isLikelyVisionKitUnsupportedHost()

        try {
            if (!forceRoi) {
                const vkOk = await startVisionKit()
                if (vkOk) {
                    active = true
                    paused = false
                    pump.start((frame) => {
                        if (!active || paused) return
                        detector.pushFrame(frame)
                        // 手离开画面时 VK 可能完全静默：补发“无手”样本，
                        // 让状态机能正常走丢失→UNTRACKED，回来后能重新武装
                        const latest = detector.getLatest()
                        const now = Date.now()
                        if ((!latest || now - latest.ts > 500) && now - lastSyntheticTs > 250) {
                            lastSyntheticTs = now
                            emitFromSample(null)
                        }
                    })
                    return true
                }
            }

            // VisionKit 不可用：ROI 回退，保证链路可联调
            active = true
            paused = false
            startRoiFallback(forceRoi ? getVisionKitSupportHint() : 'VisionKit 启动失败，已降级 ROI 联调')
            return true
        } catch (e: any) {
            handlers.onError?.(e?.message || '手势管线启动失败')
            stop()
            return false
        }
    }

    function setPaused(v: boolean) {
        paused = v
    }

    function setNearGoal(enabled: boolean) {
        classifier.setNearGoal(enabled)
    }

    function stop() {
        active = false
        paused = false
        mode = 'none'
        lastRoiTs = 0
        lastUiEmitTs = 0
        lastEmittedPhase = ''
        lastEmittedCompression = -1
        lastEmittedHold = -1
        lastSyntheticTs = 0
        pump.stop()
        detector.stop()
        classifier.setNearGoal(false)
        classifier.reset()
    }

    function isActive() {
        return active
    }

    function getMode() {
        return mode
    }

    return { start, stop, setPaused, setNearGoal, isActive, getMode }
}

export type GesturePipeline = ReturnType<typeof createGesturePipeline>