import { createCameraFramePump } from './camera-frames' import { createOpenCloseClassifier } from './open-close-classifier' import { estimateOpennessFromRoi } from './roi-openness' import type { GestureObservation, GestureSource, RepCompletedInfo } from './types' import { createVisionKitHandDetector, getVisionKitSupportHint, isLikelyVisionKitUnsupportedHost, isVisionKitHandSupported, } from './visionkit-hand' export interface GesturePipelineHandlers { onObservation: (obs: GestureObservation) => void onRepCompleted?: (info: RepCompletedInfo) => void /** 非致命提示(如已降级 ROI) */ onWarning?: (message: string) => void /** 致命错误(相机帧都起不来) */ onError?: (message: string) => void } /** * 编排:优先 VisionKit;开发者工具/不支持设备自动降级 ROI 联调。 */ export function createGesturePipeline(handlers: GesturePipelineHandlers) { const pump = createCameraFramePump() // ~10Hz;置信度适中,减少噪声帧 const detector = createVisionKitHandDetector({ minIntervalMs: 100, scoreThreshold: 0.28 }) // 握力环校准:半开即可计张开,相对抬升即可完成松开 const classifier = createOpenCloseClassifier({ openEnter: 0.45, openExit: 0.38, closeEnter: 0.36, closeExit: 0.42, emaAlpha: 0.42, minHoldMs: 120, lostMs: 1400, reopenRise: 0.08, }) let active = false let paused = false let mode: GestureSource = 'none' let lastRoiTs = 0 let lastUiEmitTs = 0 let lastEmittedPhase = '' let lastEmittedCompression = -1 let lastEmittedHold = -1 let lastSyntheticTs = 0 const UI_EMIT_MIN_MS = 100 function emitFromSample(sample: Parameters[0]) { if (!active || paused) return const out = classifier.update(sample) const now = Date.now() const phaseChanged = out.phase !== lastEmittedPhase const due = now - lastUiEmitTs >= UI_EMIT_MIN_MS const compressionDelta = Math.abs(out.compression - lastEmittedCompression) >= 0.08 const holdDelta = Math.abs(out.holdProgress - lastEmittedHold) >= 0.12 // 计次始终即时;UI 观察节流,避免小程序每帧 setData 卡顿 if (out.rep) { handlers.onRepCompleted?.(out.rep) } if (!phaseChanged && !due && !compressionDelta && !holdDelta && !out.rep) { return } lastUiEmitTs = now lastEmittedPhase = out.phase lastEmittedCompression = out.compression lastEmittedHold = out.holdProgress handlers.onObservation({ ts: now, openness: out.openness, compression: out.compression, phase: out.phase, trackingQuality: out.trackingQuality, source: out.source, holdProgress: out.holdProgress, fps: pump.getFps(), }) } async function startVisionKit(): Promise { if (!isVisionKitHandSupported()) return false try { await detector.start((sample) => emitFromSample(sample)) mode = 'visionkit' return true } catch (_) { detector.stop() return false } } function startRoiFallback(reason?: string) { mode = 'roi' const hint = reason || getVisionKitSupportHint() || '已降级为 ROI 联调模式(非真机手势识别)' handlers.onWarning?.(hint) pump.start((frame) => { if (!active || paused) return const now = Date.now() if (now - lastRoiTs < 160) return lastRoiTs = now emitFromSample(estimateOpennessFromRoi(frame)) }) } async function start() { stop() // 开发者工具直接走 ROI,避免 createVKSession 抛 “does not support version v1” const forceRoi = isLikelyVisionKitUnsupportedHost() try { if (!forceRoi) { const vkOk = await startVisionKit() if (vkOk) { active = true paused = false pump.start((frame) => { if (!active || paused) return detector.pushFrame(frame) // 手离开画面时 VK 可能完全静默:补发“无手”样本, // 让状态机能正常走丢失→UNTRACKED,回来后能重新武装 const latest = detector.getLatest() const now = Date.now() if ((!latest || now - latest.ts > 500) && now - lastSyntheticTs > 250) { lastSyntheticTs = now emitFromSample(null) } }) return true } } // VisionKit 不可用:ROI 回退,保证链路可联调 active = true paused = false startRoiFallback(forceRoi ? getVisionKitSupportHint() : 'VisionKit 启动失败,已降级 ROI 联调') return true } catch (e: any) { handlers.onError?.(e?.message || '手势管线启动失败') stop() return false } } function setPaused(v: boolean) { paused = v } function setNearGoal(enabled: boolean) { classifier.setNearGoal(enabled) } function stop() { active = false paused = false mode = 'none' lastRoiTs = 0 lastUiEmitTs = 0 lastEmittedPhase = '' lastEmittedCompression = -1 lastEmittedHold = -1 lastSyntheticTs = 0 pump.stop() detector.stop() classifier.setNearGoal(false) classifier.reset() } function isActive() { return active } function getMode() { return mode } return { start, stop, setPaused, setNearGoal, isActive, getMode } } export type GesturePipeline = ReturnType