feat(training): 新增握力环 2.0 摄像头手势识别页
用前置摄像头 + VisionKit/ROI 管线驱动开合计数，复用跟练舞台与连击粒子，并加固结束收尾避免末段卡死无响应。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Cursor
@@ -116,6 +116,17 @@
|
||||
<view class="menu-arrow">›</view>
|
||||
</view>
|
||||
|
||||
<view class="menu-item" @click="goto('/training/pages/camera-grip-ring')">
|
||||
<view class="menu-icon menu-icon--grip">
|
||||
<view class="grip-ring-shape" />
|
||||
</view>
|
||||
<view class="menu-info">
|
||||
<text class="menu-title">握力环 2.0</text>
|
||||
<text class="menu-sub">摄像头识别 · 试玩</text>
|
||||
</view>
|
||||
<view class="menu-arrow">›</view>
|
||||
</view>
|
||||
|
||||
<view class="menu-item" @click="goto('/training/pages/metronome')">
|
||||
<view class="menu-icon menu-icon--metro">
|
||||
<view class="metro-bar metro-bar-1" />
|
||||
@@ -163,6 +174,7 @@ const trainingItems: TrainingItem[] = [
|
||||
{ url: '/training/pages/foot-pedal', title: '脚蹬器', sub: '下肢有氧', icon: 'pedal', iconClass: 'menu-icon--pedal' },
|
||||
{ url: '/training/pages/pilates-ring', title: '瑜伽环', sub: '核心塑形', icon: 'pilates', iconClass: 'menu-icon--pilates' },
|
||||
{ url: '/training/pages/grip-ring', title: '握力环', sub: '握力训练', icon: 'grip', iconClass: 'menu-icon--grip' },
|
||||
{ url: '/training/pages/camera-grip-ring', title: '握力环 2.0', sub: '摄像头识别', icon: 'grip', iconClass: 'menu-icon--grip' },
|
||||
{ url: '/training/pages/metronome', title: '耗糖节拍器', sub: '健走配速', icon: 'metro', iconClass: 'menu-icon--metro', featured: true },
|
||||
]
|
||||
|
||||
|
||||
@@ -122,6 +122,17 @@
|
||||
"backgroundColor": "#f8fafc"
|
||||
}
|
||||
},
|
||||
{
|
||||
"path": "pages/camera-grip-ring",
|
||||
"style": {
|
||||
"navigationStyle": "custom",
|
||||
"navigationBarTitleText": "握力环训练 2.0",
|
||||
"navigationBarBackgroundColor": "#F8FAFC",
|
||||
"navigationBarTextStyle": "black",
|
||||
"backgroundColor": "#F8FAFC",
|
||||
"disableScroll": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"path": "pages/pilates-ring",
|
||||
"style": {
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import type { CameraFrame } from './types'
|
||||
|
||||
type FrameListener = {
|
||||
start: () => void
|
||||
stop: () => void
|
||||
}
|
||||
|
||||
type CameraContextLike = {
|
||||
onCameraFrame?: (cb: (frame: { width: number; height: number; data: ArrayBuffer }) => void) => FrameListener
|
||||
}
|
||||
|
||||
function createContext(cameraId?: string): CameraContextLike | null {
|
||||
try {
|
||||
// uni-app 优先：可按 camera 组件 id 绑定
|
||||
if (typeof uni !== 'undefined' && typeof uni.createCameraContext === 'function') {
|
||||
return uni.createCameraContext(cameraId || 'gripCamera') as CameraContextLike
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
// #ifdef MP-WEIXIN
|
||||
try {
|
||||
// @ts-ignore
|
||||
if (typeof wx !== 'undefined' && typeof wx.createCameraContext === 'function') {
|
||||
// @ts-ignore
|
||||
return wx.createCameraContext() as CameraContextLike
|
||||
}
|
||||
} catch (_) {}
|
||||
// #endif
|
||||
return null
|
||||
}
|
||||
|
||||
export function isCameraFrameSupported(): boolean {
|
||||
try {
|
||||
const ctx = createContext('gripCamera')
|
||||
return typeof ctx?.onCameraFrame === 'function'
|
||||
} catch (_) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 管理 CameraContext.onCameraFrame：只保留最新一帧，不入 Vue 响应式。
|
||||
*/
|
||||
export function createCameraFramePump(options?: { cameraId?: string }) {
|
||||
let listener: FrameListener | null = null
|
||||
let latest: CameraFrame | null = null
|
||||
let running = false
|
||||
let callbackCount = 0
|
||||
let fpsWindowStart = Date.now()
|
||||
let fps = 0
|
||||
let onFrame: ((frame: CameraFrame) => void) | null = null
|
||||
|
||||
function start(cb: (frame: CameraFrame) => void) {
|
||||
stop()
|
||||
onFrame = cb
|
||||
const ctx = createContext(options?.cameraId || 'gripCamera')
|
||||
if (!ctx?.onCameraFrame) {
|
||||
throw new Error('当前基础库不支持 onCameraFrame')
|
||||
}
|
||||
|
||||
listener = ctx.onCameraFrame((raw) => {
|
||||
if (!running) return
|
||||
const frame: CameraFrame = {
|
||||
width: raw.width,
|
||||
height: raw.height,
|
||||
data: raw.data,
|
||||
ts: Date.now(),
|
||||
}
|
||||
latest = frame
|
||||
callbackCount += 1
|
||||
const now = Date.now()
|
||||
if (now - fpsWindowStart >= 1000) {
|
||||
fps = callbackCount
|
||||
callbackCount = 0
|
||||
fpsWindowStart = now
|
||||
}
|
||||
onFrame?.(frame)
|
||||
})
|
||||
running = true
|
||||
listener.start()
|
||||
}
|
||||
|
||||
function stop() {
|
||||
running = false
|
||||
try {
|
||||
listener?.stop()
|
||||
} catch (_) {}
|
||||
listener = null
|
||||
onFrame = null
|
||||
}
|
||||
|
||||
function getLatest() {
|
||||
return latest
|
||||
}
|
||||
|
||||
function getFps() {
|
||||
return fps
|
||||
}
|
||||
|
||||
function isRunning() {
|
||||
return running
|
||||
}
|
||||
|
||||
return { start, stop, getLatest, getFps, isRunning }
|
||||
}
|
||||
|
||||
export type CameraFramePump = ReturnType<typeof createCameraFramePump>
|
||||
@@ -0,0 +1,186 @@
|
||||
import { createCameraFramePump } from './camera-frames'
|
||||
import { createOpenCloseClassifier } from './open-close-classifier'
|
||||
import { estimateOpennessFromRoi } from './roi-openness'
|
||||
import type { GestureObservation, GestureSource, RepCompletedInfo } from './types'
|
||||
import {
|
||||
createVisionKitHandDetector,
|
||||
getVisionKitSupportHint,
|
||||
isLikelyVisionKitUnsupportedHost,
|
||||
isVisionKitHandSupported,
|
||||
} from './visionkit-hand'
|
||||
|
||||
export interface GesturePipelineHandlers {
|
||||
onObservation: (obs: GestureObservation) => void
|
||||
onRepCompleted?: (info: RepCompletedInfo) => void
|
||||
/** 非致命提示（如已降级 ROI） */
|
||||
onWarning?: (message: string) => void
|
||||
/** 致命错误（相机帧都起不来） */
|
||||
onError?: (message: string) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* 编排：优先 VisionKit；开发者工具/不支持设备自动降级 ROI 联调。
|
||||
*/
|
||||
export function createGesturePipeline(handlers: GesturePipelineHandlers) {
|
||||
const pump = createCameraFramePump()
|
||||
// ~10Hz；置信度适中，减少噪声帧
|
||||
const detector = createVisionKitHandDetector({ minIntervalMs: 100, scoreThreshold: 0.28 })
|
||||
// 握力环校准：半开即可计张开，相对抬升即可完成松开
|
||||
const classifier = createOpenCloseClassifier({
|
||||
openEnter: 0.45,
|
||||
openExit: 0.38,
|
||||
closeEnter: 0.36,
|
||||
closeExit: 0.42,
|
||||
emaAlpha: 0.42,
|
||||
minHoldMs: 120,
|
||||
lostMs: 1400,
|
||||
reopenRise: 0.08,
|
||||
})
|
||||
|
||||
let active = false
|
||||
let paused = false
|
||||
let mode: GestureSource = 'none'
|
||||
let lastRoiTs = 0
|
||||
let lastUiEmitTs = 0
|
||||
let lastEmittedPhase = ''
|
||||
let lastEmittedCompression = -1
|
||||
let lastEmittedHold = -1
|
||||
let lastSyntheticTs = 0
|
||||
const UI_EMIT_MIN_MS = 100
|
||||
|
||||
function emitFromSample(sample: Parameters<typeof classifier.update>[0]) {
|
||||
if (!active || paused) return
|
||||
const out = classifier.update(sample)
|
||||
const now = Date.now()
|
||||
const phaseChanged = out.phase !== lastEmittedPhase
|
||||
const due = now - lastUiEmitTs >= UI_EMIT_MIN_MS
|
||||
const compressionDelta = Math.abs(out.compression - lastEmittedCompression) >= 0.08
|
||||
const holdDelta = Math.abs(out.holdProgress - lastEmittedHold) >= 0.12
|
||||
|
||||
// 计次始终即时；UI 观察节流，避免小程序每帧 setData 卡顿
|
||||
if (out.rep) {
|
||||
handlers.onRepCompleted?.(out.rep)
|
||||
}
|
||||
|
||||
if (!phaseChanged && !due && !compressionDelta && !holdDelta && !out.rep) {
|
||||
return
|
||||
}
|
||||
|
||||
lastUiEmitTs = now
|
||||
lastEmittedPhase = out.phase
|
||||
lastEmittedCompression = out.compression
|
||||
lastEmittedHold = out.holdProgress
|
||||
|
||||
handlers.onObservation({
|
||||
ts: now,
|
||||
openness: out.openness,
|
||||
compression: out.compression,
|
||||
phase: out.phase,
|
||||
trackingQuality: out.trackingQuality,
|
||||
source: out.source,
|
||||
holdProgress: out.holdProgress,
|
||||
fps: pump.getFps(),
|
||||
})
|
||||
}
|
||||
|
||||
async function startVisionKit(): Promise<boolean> {
|
||||
if (!isVisionKitHandSupported()) return false
|
||||
try {
|
||||
await detector.start((sample) => emitFromSample(sample))
|
||||
mode = 'visionkit'
|
||||
return true
|
||||
} catch (_) {
|
||||
detector.stop()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function startRoiFallback(reason?: string) {
|
||||
mode = 'roi'
|
||||
const hint = reason || getVisionKitSupportHint() || '已降级为 ROI 联调模式（非真机手势识别）'
|
||||
handlers.onWarning?.(hint)
|
||||
pump.start((frame) => {
|
||||
if (!active || paused) return
|
||||
const now = Date.now()
|
||||
if (now - lastRoiTs < 160) return
|
||||
lastRoiTs = now
|
||||
emitFromSample(estimateOpennessFromRoi(frame))
|
||||
})
|
||||
}
|
||||
|
||||
async function start() {
|
||||
stop()
|
||||
|
||||
// 开发者工具直接走 ROI，避免 createVKSession 抛 “does not support version v1”
|
||||
const forceRoi = isLikelyVisionKitUnsupportedHost()
|
||||
|
||||
try {
|
||||
if (!forceRoi) {
|
||||
const vkOk = await startVisionKit()
|
||||
if (vkOk) {
|
||||
active = true
|
||||
paused = false
|
||||
pump.start((frame) => {
|
||||
if (!active || paused) return
|
||||
detector.pushFrame(frame)
|
||||
// 手离开画面时 VK 可能完全静默：补发“无手”样本，
|
||||
// 让状态机能正常走丢失→UNTRACKED，回来后能重新武装
|
||||
const latest = detector.getLatest()
|
||||
const now = Date.now()
|
||||
if ((!latest || now - latest.ts > 500) && now - lastSyntheticTs > 250) {
|
||||
lastSyntheticTs = now
|
||||
emitFromSample(null)
|
||||
}
|
||||
})
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// VisionKit 不可用：ROI 回退，保证链路可联调
|
||||
active = true
|
||||
paused = false
|
||||
startRoiFallback(forceRoi ? getVisionKitSupportHint() : 'VisionKit 启动失败，已降级 ROI 联调')
|
||||
return true
|
||||
} catch (e: any) {
|
||||
handlers.onError?.(e?.message || '手势管线启动失败')
|
||||
stop()
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
function setPaused(v: boolean) {
|
||||
paused = v
|
||||
}
|
||||
|
||||
function setNearGoal(enabled: boolean) {
|
||||
classifier.setNearGoal(enabled)
|
||||
}
|
||||
|
||||
function stop() {
|
||||
active = false
|
||||
paused = false
|
||||
mode = 'none'
|
||||
lastRoiTs = 0
|
||||
lastUiEmitTs = 0
|
||||
lastEmittedPhase = ''
|
||||
lastEmittedCompression = -1
|
||||
lastEmittedHold = -1
|
||||
lastSyntheticTs = 0
|
||||
pump.stop()
|
||||
detector.stop()
|
||||
classifier.setNearGoal(false)
|
||||
classifier.reset()
|
||||
}
|
||||
|
||||
function isActive() {
|
||||
return active
|
||||
}
|
||||
|
||||
function getMode() {
|
||||
return mode
|
||||
}
|
||||
|
||||
return { start, stop, setPaused, setNearGoal, isActive, getMode }
|
||||
}
|
||||
|
||||
export type GesturePipeline = ReturnType<typeof createGesturePipeline>
|
||||
@@ -0,0 +1,189 @@
|
||||
import type { GesturePhase, GestureSource, HandSample, RepCompletedInfo, TrackingQuality } from './types'
|
||||
|
||||
export interface ClassifierOutput {
|
||||
phase: GesturePhase
|
||||
trackingQuality: TrackingQuality
|
||||
openness: number
|
||||
compression: number
|
||||
holdProgress: number
|
||||
source: GestureSource
|
||||
rep?: RepCompletedInfo
|
||||
}
|
||||
|
||||
/**
|
||||
* 开合状态机（握力环校准版）
|
||||
* 计数：OPEN → CLOSING → CLOSED(保持) → REOPENING → 相对张开，才 +1
|
||||
* 松开不必完全张开：相对握紧谷底抬升即可计次，避免做到后期张不开导致卡在 29
|
||||
*/
|
||||
export function createOpenCloseClassifier(options?: {
|
||||
openEnter?: number
|
||||
openExit?: number
|
||||
closeEnter?: number
|
||||
closeExit?: number
|
||||
minHoldMs?: number
|
||||
lostMs?: number
|
||||
emaAlpha?: number
|
||||
/** 相对松开：相对握紧谷底至少抬升这么多才计 1 次 */
|
||||
reopenRise?: number
|
||||
}) {
|
||||
const openEnter = options?.openEnter ?? 0.48
|
||||
const openExit = options?.openExit ?? 0.4
|
||||
const closeEnter = options?.closeEnter ?? 0.38
|
||||
const closeExit = options?.closeExit ?? 0.44
|
||||
const minHoldMs = options?.minHoldMs ?? 140
|
||||
const lostMs = options?.lostMs ?? 1200
|
||||
const emaAlpha = options?.emaAlpha ?? 0.4
|
||||
const reopenRise = options?.reopenRise ?? 0.1
|
||||
|
||||
let phase: GesturePhase = 'UNTRACKED'
|
||||
let openness = 0.5
|
||||
let holdStartedAt = 0
|
||||
let holdProgress = 0
|
||||
let lastSeenAt = 0
|
||||
let opennessPeak = 0
|
||||
let closedTrough = 1
|
||||
let source: GestureSource = 'none'
|
||||
let lastRepTs = 0
|
||||
|
||||
let reopenRiseActive = reopenRise
|
||||
|
||||
function resetPartial() {
|
||||
holdStartedAt = 0
|
||||
holdProgress = 0
|
||||
opennessPeak = 0
|
||||
closedTrough = 1
|
||||
}
|
||||
|
||||
/** 接近目标次数时进一步放宽松开判定 */
|
||||
function setNearGoal(enabled: boolean) {
|
||||
reopenRiseActive = enabled ? 0.045 : reopenRise
|
||||
}
|
||||
|
||||
function canCountRep(now: number) {
|
||||
// 防连发：两次有效计次至少间隔 280ms
|
||||
return now - lastRepTs >= 280
|
||||
}
|
||||
|
||||
function update(sample: HandSample | null, now = Date.now()): ClassifierOutput {
|
||||
let rep: RepCompletedInfo | undefined
|
||||
|
||||
if (!sample || sample.score <= 0.05) {
|
||||
if (lastSeenAt && now - lastSeenAt > lostMs) {
|
||||
phase = 'UNTRACKED'
|
||||
resetPartial()
|
||||
}
|
||||
const withinGrace = lastSeenAt > 0 && now - lastSeenAt <= 600
|
||||
return {
|
||||
phase,
|
||||
trackingQuality: phase === 'UNTRACKED' ? 'lost' : withinGrace ? 'good' : 'weak',
|
||||
openness,
|
||||
compression: 1 - openness,
|
||||
holdProgress,
|
||||
source,
|
||||
rep,
|
||||
}
|
||||
}
|
||||
|
||||
lastSeenAt = now
|
||||
source = sample.source
|
||||
openness = openness * (1 - emaAlpha) + sample.openness * emaAlpha
|
||||
const quality: TrackingQuality = sample.score >= 0.22 ? 'good' : 'weak'
|
||||
|
||||
switch (phase) {
|
||||
case 'UNTRACKED':
|
||||
if (openness >= openEnter) {
|
||||
phase = 'OPEN_READY'
|
||||
opennessPeak = openness
|
||||
resetPartial()
|
||||
}
|
||||
break
|
||||
|
||||
case 'OPEN_READY':
|
||||
opennessPeak = Math.max(opennessPeak, openness)
|
||||
if (openness <= closeEnter) {
|
||||
phase = 'CLOSING'
|
||||
closedTrough = openness
|
||||
}
|
||||
break
|
||||
|
||||
case 'CLOSING':
|
||||
closedTrough = Math.min(closedTrough, openness)
|
||||
if (openness <= closeEnter) {
|
||||
phase = 'CLOSED_CONFIRMED'
|
||||
holdStartedAt = now
|
||||
holdProgress = 0
|
||||
closedTrough = openness
|
||||
} else if (openness >= openEnter) {
|
||||
phase = 'OPEN_READY'
|
||||
}
|
||||
break
|
||||
|
||||
case 'CLOSED_CONFIRMED': {
|
||||
closedTrough = Math.min(closedTrough, openness)
|
||||
const held = now - holdStartedAt
|
||||
holdProgress = Math.min(1, held / minHoldMs)
|
||||
if (held >= minHoldMs && openness >= closeExit) {
|
||||
phase = 'REOPENING'
|
||||
} else if (openness >= openEnter && held < minHoldMs) {
|
||||
// 保持不足就完全张开：作废本轮
|
||||
phase = 'OPEN_READY'
|
||||
resetPartial()
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
case 'REOPENING': {
|
||||
closedTrough = Math.min(closedTrough, openness)
|
||||
const rise = openness - closedTrough
|
||||
// 绝对张开，或相对握紧谷底抬升足够 → 计 1 次
|
||||
const reopened = openness >= openEnter || (rise >= reopenRiseActive && openness >= closeExit)
|
||||
if (reopened && canCountRep(now)) {
|
||||
lastRepTs = now
|
||||
rep = {
|
||||
ts: now,
|
||||
holdMs: holdStartedAt ? now - holdStartedAt : 0,
|
||||
opennessPeak,
|
||||
source,
|
||||
}
|
||||
phase = 'OPEN_READY'
|
||||
resetPartial()
|
||||
opennessPeak = openness
|
||||
} else if (openness <= closeEnter) {
|
||||
// 松开中又握回去：回到保持，不直接作废
|
||||
phase = 'CLOSED_CONFIRMED'
|
||||
holdStartedAt = now
|
||||
holdProgress = 0
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
phase,
|
||||
trackingQuality: quality,
|
||||
openness,
|
||||
compression: 1 - openness,
|
||||
holdProgress,
|
||||
source,
|
||||
rep,
|
||||
}
|
||||
}
|
||||
|
||||
function reset() {
|
||||
phase = 'UNTRACKED'
|
||||
openness = 0.5
|
||||
source = 'none'
|
||||
lastSeenAt = 0
|
||||
lastRepTs = 0
|
||||
reopenRiseActive = reopenRise
|
||||
resetPartial()
|
||||
}
|
||||
|
||||
function getPhase() {
|
||||
return phase
|
||||
}
|
||||
|
||||
return { update, reset, getPhase, setNearGoal }
|
||||
}
|
||||
|
||||
export type OpenCloseClassifier = ReturnType<typeof createOpenCloseClassifier>
|
||||
@@ -0,0 +1,63 @@
|
||||
import type { HandSample, Point2D } from './types'
|
||||
|
||||
/** VisionKit 文档：1=布/开掌，3=握拳 */
|
||||
const GESTURE_OPEN = 1
|
||||
const GESTURE_FIST = 3
|
||||
|
||||
/**
|
||||
* OpenPose-21 指尖索引（相对手腕距离用于连续 openness）
|
||||
* 0 wrist, 4 thumb tip, 8 index, 12 middle, 16 ring, 20 pinky
|
||||
*/
|
||||
const TIP_INDEXES = [4, 8, 12, 16, 20]
|
||||
|
||||
function dist(a: Point2D, b: Point2D) {
|
||||
const dx = a.x - b.x
|
||||
const dy = a.y - b.y
|
||||
return Math.sqrt(dx * dx + dy * dy)
|
||||
}
|
||||
|
||||
function clamp01(v: number) {
|
||||
return Math.max(0, Math.min(1, v))
|
||||
}
|
||||
|
||||
/** 离散手势 → 开合度；未知手势返回 null（交给几何或保持） */
|
||||
export function opennessFromGesture(gesture?: number): number | null {
|
||||
if (gesture === GESTURE_OPEN) return 1
|
||||
if (gesture === GESTURE_FIST) return 0
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 用指尖到手腕的平均距离 / 掌宽 估计 openness。
|
||||
* 握力环场景开合幅度偏小，映射区间比裸掌更紧一些。
|
||||
*/
|
||||
export function opennessFromLandmarks(points?: Point2D[]): number | null {
|
||||
if (!points || points.length < 21) return null
|
||||
const wrist = points[0]
|
||||
const mcpIndex = points[5]
|
||||
const mcpPinky = points[17]
|
||||
const palmWidth = dist(mcpIndex, mcpPinky)
|
||||
if (palmWidth < 1e-6) return null
|
||||
|
||||
let tipSum = 0
|
||||
for (const i of TIP_INDEXES) {
|
||||
tipSum += dist(points[i], wrist)
|
||||
}
|
||||
const tipMean = tipSum / TIP_INDEXES.length
|
||||
// 握力环：握紧约 0.9~1.35 掌宽，半开约 1.5~2.2 掌宽
|
||||
const ratio = tipMean / palmWidth
|
||||
return clamp01((ratio - 0.95) / 1.15)
|
||||
}
|
||||
|
||||
export function mergeOpenness(sample: Pick<HandSample, 'gesture' | 'points'>): number | null {
|
||||
const fromGeom = opennessFromLandmarks(sample.points)
|
||||
const fromGesture = opennessFromGesture(sample.gesture)
|
||||
|
||||
if (fromGeom != null && fromGesture != null) {
|
||||
// 几何为主：离散手势（布/拳）在捏环时容易误判，只做轻牵引
|
||||
return clamp01(fromGeom * 0.85 + fromGesture * 0.15)
|
||||
}
|
||||
if (fromGeom != null) return fromGeom
|
||||
if (fromGesture != null) return fromGesture
|
||||
return null
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { CameraFrame, HandSample } from './types'
|
||||
|
||||
function clamp01(v: number) {
|
||||
return Math.max(0, Math.min(1, v))
|
||||
}
|
||||
|
||||
/**
|
||||
* 开发者工具 / 不支持 VisionKit 时的粗开合估计：
|
||||
* 取画面中心 ROI 的亮度方差与边缘能量，映射到 openness。
|
||||
* 仅用于打通链路与 UI 联调，不能作为产品识别。
|
||||
*/
|
||||
export function estimateOpennessFromRoi(frame: CameraFrame): HandSample {
|
||||
const { width, height, data } = frame
|
||||
const u8 = new Uint8Array(data)
|
||||
const cx0 = Math.floor(width * 0.28)
|
||||
const cy0 = Math.floor(height * 0.22)
|
||||
const cx1 = Math.floor(width * 0.72)
|
||||
const cy1 = Math.floor(height * 0.78)
|
||||
const step = Math.max(4, Math.floor(Math.min(width, height) / 40))
|
||||
|
||||
let count = 0
|
||||
let sum = 0
|
||||
let sumSq = 0
|
||||
let edge = 0
|
||||
let prev = 0
|
||||
|
||||
for (let y = cy0; y < cy1; y += step) {
|
||||
for (let x = cx0; x < cx1; x += step) {
|
||||
const i = (y * width + x) * 4
|
||||
const r = u8[i]
|
||||
const g = u8[i + 1]
|
||||
const b = u8[i + 2]
|
||||
// 粗略肤色门控 + 亮度
|
||||
const luma = 0.299 * r + 0.587 * g + 0.114 * b
|
||||
const skinish = r > 60 && g > 30 && b > 20 && r >= g && Math.abs(r - g) > 8
|
||||
if (!skinish && luma < 40) continue
|
||||
sum += luma
|
||||
sumSq += luma * luma
|
||||
edge += Math.abs(luma - prev)
|
||||
prev = luma
|
||||
count += 1
|
||||
}
|
||||
}
|
||||
|
||||
if (count < 20) {
|
||||
return {
|
||||
ts: Date.now(),
|
||||
openness: 0.5,
|
||||
score: 0.05,
|
||||
source: 'roi',
|
||||
}
|
||||
}
|
||||
|
||||
const mean = sum / count
|
||||
const variance = Math.max(0, sumSq / count - mean * mean)
|
||||
const edgeNorm = edge / count
|
||||
|
||||
// 张开手掌通常带来更大轮廓/边缘；握拳更紧凑。经验映射，仅联调。
|
||||
const openness = clamp01((Math.sqrt(variance) / 55 + edgeNorm / 40) * 0.85)
|
||||
const score = clamp01(0.25 + count / 800)
|
||||
|
||||
return {
|
||||
ts: Date.now(),
|
||||
openness,
|
||||
score,
|
||||
source: 'roi',
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
/** 握力环 2.0 视觉观测与状态机共享类型 */
|
||||
|
||||
export type TrackingQuality = 'good' | 'weak' | 'lost'
|
||||
|
||||
export type GesturePhase =
|
||||
| 'UNTRACKED'
|
||||
| 'OPEN_READY'
|
||||
| 'CLOSING'
|
||||
| 'CLOSED_CONFIRMED'
|
||||
| 'REOPENING'
|
||||
|
||||
export type GestureSource = 'visionkit' | 'roi' | 'mock' | 'none'
|
||||
|
||||
export interface Point2D {
|
||||
x: number
|
||||
y: number
|
||||
}
|
||||
|
||||
export interface CameraFrame {
|
||||
width: number
|
||||
height: number
|
||||
data: ArrayBuffer
|
||||
ts: number
|
||||
}
|
||||
|
||||
export interface HandSample {
|
||||
ts: number
|
||||
/** 0=握紧 … 1=张开 */
|
||||
openness: number
|
||||
score: number
|
||||
gesture?: number
|
||||
points?: Point2D[]
|
||||
source: GestureSource
|
||||
}
|
||||
|
||||
export interface GestureObservation {
|
||||
ts: number
|
||||
openness: number
|
||||
/** 0=张开 … 1=握紧，供能量环使用 */
|
||||
compression: number
|
||||
phase: GesturePhase
|
||||
trackingQuality: TrackingQuality
|
||||
source: GestureSource
|
||||
holdProgress: number
|
||||
fps: number
|
||||
}
|
||||
|
||||
export interface RepCompletedInfo {
|
||||
ts: number
|
||||
holdMs: number
|
||||
opennessPeak: number
|
||||
source: GestureSource
|
||||
}
|
||||
@@ -0,0 +1,296 @@
|
||||
import type { CameraFrame, HandSample, Point2D } from './types'
|
||||
import { mergeOpenness } from './openness'
|
||||
|
||||
type VkSession = {
|
||||
start: (cb?: (errno: number | null) => void) => void
|
||||
stop?: () => void
|
||||
destroy?: () => void
|
||||
detectHand?: (opts: Record<string, unknown>) => void
|
||||
on?: (event: string, cb: (...args: any[]) => void) => void
|
||||
off?: (event: string, cb?: (...args: any[]) => void) => void
|
||||
}
|
||||
|
||||
type WxLike = {
|
||||
createVKSession?: (opts: Record<string, unknown>) => VkSession
|
||||
getSystemInfoSync?: () => { SDKVersion?: string; platform?: string; brand?: string; model?: string }
|
||||
getAppBaseInfo?: () => { SDKVersion?: string }
|
||||
getDeviceInfo?: () => { platform?: string; brand?: string; model?: string; system?: string }
|
||||
}
|
||||
|
||||
function getWx(): WxLike | null {
|
||||
// #ifdef MP-WEIXIN
|
||||
// @ts-ignore
|
||||
if (typeof wx !== 'undefined') return wx as WxLike
|
||||
// #endif
|
||||
return null
|
||||
}
|
||||
|
||||
function parseVersion(v?: string) {
|
||||
if (!v) return [0, 0, 0]
|
||||
return v.split('.').map((n) => Number(n) || 0)
|
||||
}
|
||||
|
||||
function gteVersion(current: string | undefined, need: string) {
|
||||
const a = parseVersion(current)
|
||||
const b = parseVersion(need)
|
||||
for (let i = 0; i < 3; i++) {
|
||||
if (a[i] > b[i]) return true
|
||||
if (a[i] < b[i]) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
function readRuntimeInfo() {
|
||||
const wxApi = getWx()
|
||||
let SDKVersion = ''
|
||||
let platform = ''
|
||||
let brand = ''
|
||||
let model = ''
|
||||
try {
|
||||
const base = wxApi?.getAppBaseInfo?.()
|
||||
if (base?.SDKVersion) SDKVersion = base.SDKVersion
|
||||
} catch (_) {}
|
||||
try {
|
||||
const device = wxApi?.getDeviceInfo?.()
|
||||
if (device?.platform) platform = String(device.platform)
|
||||
if (device?.brand) brand = String(device.brand)
|
||||
if (device?.model) model = String(device.model)
|
||||
} catch (_) {}
|
||||
// 兼容旧环境
|
||||
if (!SDKVersion || !platform) {
|
||||
try {
|
||||
const info = wxApi?.getSystemInfoSync?.() || {}
|
||||
SDKVersion = SDKVersion || info.SDKVersion || ''
|
||||
platform = platform || String(info.platform || '')
|
||||
brand = brand || String(info.brand || '')
|
||||
model = model || String(info.model || '')
|
||||
} catch (_) {}
|
||||
}
|
||||
return { SDKVersion, platform, brand, model }
|
||||
}
|
||||
|
||||
/** 开发者工具 / 模拟器通常不支持 VisionKit Hand */
|
||||
export function isLikelyVisionKitUnsupportedHost(): boolean {
|
||||
const { platform, brand, model } = readRuntimeInfo()
|
||||
const blob = `${platform} ${brand} ${model}`.toLowerCase()
|
||||
return (
|
||||
blob.includes('devtools') ||
|
||||
blob.includes('mac') ||
|
||||
blob.includes('windows') ||
|
||||
blob.includes('devtools') ||
|
||||
platform === 'devtools'
|
||||
)
|
||||
}
|
||||
|
||||
export function isVisionKitHandSupported(): boolean {
|
||||
const wxApi = getWx()
|
||||
if (!wxApi?.createVKSession) return false
|
||||
if (isLikelyVisionKitUnsupportedHost()) return false
|
||||
const { SDKVersion } = readRuntimeInfo()
|
||||
// VisionKit Hand: 基础库 2.28.0+
|
||||
return gteVersion(SDKVersion, '2.28.0')
|
||||
}
|
||||
|
||||
export function getVisionKitSupportHint(): string {
|
||||
if (isLikelyVisionKitUnsupportedHost()) {
|
||||
return '当前是开发者工具/电脑环境，VisionKit 不可用，请用真机预览；已自动降级为 ROI 联调模式'
|
||||
}
|
||||
if (!getWx()?.createVKSession) {
|
||||
return '当前微信不支持 VisionKit'
|
||||
}
|
||||
const { SDKVersion } = readRuntimeInfo()
|
||||
if (!gteVersion(SDKVersion, '2.28.0')) {
|
||||
return `基础库 ${SDKVersion || '未知'} 过低，VisionKit Hand 需 ≥ 2.28.0`
|
||||
}
|
||||
return ''
|
||||
}
|
||||
|
||||
function normalizePoints(raw: any): Point2D[] | undefined {
|
||||
if (!Array.isArray(raw) || raw.length === 0) return undefined
|
||||
return raw.map((p) => {
|
||||
if (Array.isArray(p)) return { x: Number(p[0]) || 0, y: Number(p[1]) || 0 }
|
||||
return { x: Number(p?.x) || 0, y: Number(p?.y) || 0 }
|
||||
})
|
||||
}
|
||||
|
||||
function isUnsupportedDeviceError(err: unknown) {
|
||||
const msg = String((err as any)?.message || err || '')
|
||||
return (
|
||||
msg.includes('does not support version') ||
|
||||
msg.includes('not support') ||
|
||||
msg.includes('不支持') ||
|
||||
msg.includes('v1')
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* VisionKit Hand mode=2：由业务喂入 onCameraFrame 的 RGBA。
|
||||
* 内部节流，保证同一时刻最多一次 detectHand。
|
||||
*/
|
||||
export function createVisionKitHandDetector(options?: {
|
||||
scoreThreshold?: number
|
||||
minIntervalMs?: number
|
||||
}) {
|
||||
const scoreThreshold = options?.scoreThreshold ?? 0.5
|
||||
const minIntervalMs = options?.minIntervalMs ?? 100 // ~10Hz
|
||||
// busy 卡死看门狗：detectHand 后超过该时长无回调则强制释放
|
||||
// 注意：手不在画面时 VK 可能整段不回调，这属于正常情况，不能当作会话挂死
|
||||
const BUSY_TIMEOUT_MS = 700
|
||||
|
||||
let session: VkSession | null = null
|
||||
let started = false
|
||||
let busy = false
|
||||
let busySince = 0
|
||||
let restarting = false
|
||||
let sessionStartedAt = 0
|
||||
let lastCallbackTs = 0
|
||||
let lastDetectTs = 0
|
||||
let latestSample: HandSample | null = null
|
||||
let onSample: ((sample: HandSample) => void) | null = null
|
||||
|
||||
function handleAnchors(anchors: any[]) {
|
||||
busy = false
|
||||
lastCallbackTs = Date.now()
|
||||
if (!anchors || !anchors.length) {
|
||||
latestSample = {
|
||||
ts: Date.now(),
|
||||
openness: latestSample?.openness ?? 0.5,
|
||||
score: 0,
|
||||
source: 'visionkit',
|
||||
}
|
||||
onSample?.(latestSample)
|
||||
return
|
||||
}
|
||||
|
||||
const sorted = [...anchors].sort((a, b) => (Number(b?.score) || 0) - (Number(a?.score) || 0))
|
||||
const best = sorted[0]
|
||||
const points = normalizePoints(best?.points)
|
||||
const gesture = Number(best?.gesture)
|
||||
const score = Number(best?.score) || Number(best?.confidence?.[0]) || 0
|
||||
const openness = mergeOpenness({ gesture, points })
|
||||
|
||||
latestSample = {
|
||||
ts: Date.now(),
|
||||
openness: openness == null ? (latestSample?.openness ?? 0.5) : openness,
|
||||
score,
|
||||
gesture: Number.isFinite(gesture) ? gesture : undefined,
|
||||
points,
|
||||
source: 'visionkit',
|
||||
}
|
||||
onSample?.(latestSample)
|
||||
}
|
||||
|
||||
function teardownSession() {
|
||||
busy = false
|
||||
try {
|
||||
session?.stop?.()
|
||||
} catch (_) {}
|
||||
try {
|
||||
session?.destroy?.()
|
||||
} catch (_) {}
|
||||
session = null
|
||||
}
|
||||
|
||||
function createAndStartSession(): Promise<void> {
|
||||
const wxApi = getWx()
|
||||
return new Promise((resolve, reject) => {
|
||||
try {
|
||||
session = wxApi!.createVKSession!({
|
||||
track: {
|
||||
hand: { mode: 2 },
|
||||
},
|
||||
})
|
||||
session.on?.('updateAnchors', (anchors: any[]) => {
|
||||
handleAnchors(anchors || [])
|
||||
})
|
||||
session.on?.('removeAnchors', () => {
|
||||
busy = false
|
||||
lastCallbackTs = Date.now()
|
||||
latestSample = {
|
||||
ts: Date.now(),
|
||||
openness: latestSample?.openness ?? 0.5,
|
||||
score: 0,
|
||||
source: 'visionkit',
|
||||
}
|
||||
onSample?.(latestSample)
|
||||
})
|
||||
session.start?.((errno) => {
|
||||
if (errno) {
|
||||
reject(new Error(`VisionKit start 失败: ${errno}`))
|
||||
return
|
||||
}
|
||||
const now = Date.now()
|
||||
started = true
|
||||
busy = false
|
||||
sessionStartedAt = now
|
||||
lastCallbackTs = now
|
||||
resolve()
|
||||
})
|
||||
} catch (e: any) {
|
||||
if (isUnsupportedDeviceError(e)) {
|
||||
reject(new Error(getVisionKitSupportHint() || '当前设备不支持 VisionKit Hand'))
|
||||
return
|
||||
}
|
||||
reject(e instanceof Error ? e : new Error(String(e)))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function start(cb: (sample: HandSample) => void): Promise<void> {
|
||||
stop()
|
||||
onSample = cb
|
||||
const wxApi = getWx()
|
||||
if (!wxApi?.createVKSession) {
|
||||
return Promise.reject(new Error('不支持 VisionKit'))
|
||||
}
|
||||
if (isLikelyVisionKitUnsupportedHost()) {
|
||||
return Promise.reject(new Error(getVisionKitSupportHint()))
|
||||
}
|
||||
return createAndStartSession()
|
||||
}
|
||||
|
||||
function pushFrame(frame: CameraFrame) {
|
||||
if (!started || restarting) return
|
||||
const now = Date.now()
|
||||
|
||||
// busy 看门狗：detectHand 无回调时强制释放，避免识别永久停摆
|
||||
if (busy && now - busySince > BUSY_TIMEOUT_MS) {
|
||||
busy = false
|
||||
}
|
||||
|
||||
// 不再定期重建 VKSession：长跑中 destroy/create 容易把微信原生桥卡死（表现为 29 次后整页无响应）
|
||||
if (!session?.detectHand || busy) return
|
||||
if (now - lastDetectTs < minIntervalMs) return
|
||||
lastDetectTs = now
|
||||
busy = true
|
||||
busySince = now
|
||||
try {
|
||||
session.detectHand({
|
||||
frameBuffer: frame.data,
|
||||
width: frame.width,
|
||||
height: frame.height,
|
||||
scoreThreshold,
|
||||
algoMode: 2,
|
||||
})
|
||||
} catch (_) {
|
||||
busy = false
|
||||
}
|
||||
}
|
||||
|
||||
function stop() {
|
||||
started = false
|
||||
restarting = false
|
||||
onSample = null
|
||||
sessionStartedAt = 0
|
||||
lastCallbackTs = 0
|
||||
teardownSession()
|
||||
}
|
||||
|
||||
function getLatest() {
|
||||
return latestSample
|
||||
}
|
||||
|
||||
return { start, stop, pushFrame, getLatest }
|
||||
}
|
||||
|
||||
export type VisionKitHandDetector = ReturnType<typeof createVisionKitHandDetector>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -204,13 +204,13 @@ const instance = getCurrentInstance()
|
||||
|
||||
// 由父级 prop 信号驱动触发特效(避免跨组件 ref 调用, 小程序端更稳)
|
||||
const props = defineProps<{
|
||||
crushSignal?: { type: CrushItemType; nonce: number } | null
|
||||
crushSignal?: { type: CrushItemType; nonce: number; intensity?: number } | null
|
||||
}>()
|
||||
|
||||
watch(
|
||||
() => props.crushSignal,
|
||||
(sig) => {
|
||||
if (sig) triggerCrush(sig.type)
|
||||
if (sig) triggerCrush(sig.type, sig.intensity ?? 1)
|
||||
},
|
||||
)
|
||||
|
||||
@@ -270,12 +270,13 @@ function initCanvas() {
|
||||
// 触发捏碎特效
|
||||
// ============================================================
|
||||
|
||||
function triggerCrush(itemType: CrushItemType) {
|
||||
function triggerCrush(itemType: CrushItemType, intensity = 1) {
|
||||
if (!ctx.value || !canvasNode.value) return
|
||||
|
||||
const item = CRUSH_ITEMS[itemType]
|
||||
if (!item) return
|
||||
|
||||
const scale = Math.max(0.45, Math.min(1.15, intensity))
|
||||
const centerX = canvasWidth.value / 2
|
||||
const centerY = canvasHeight.value / 2
|
||||
const maxR = Math.min(canvasWidth.value, canvasHeight.value)
|
||||
@@ -284,39 +285,44 @@ function triggerCrush(itemType: CrushItemType) {
|
||||
flashList.value.push({
|
||||
x: centerX,
|
||||
y: centerY,
|
||||
radius: maxR * 0.12,
|
||||
maxRadius: maxR * 0.62,
|
||||
radius: maxR * 0.12 * scale,
|
||||
maxRadius: maxR * 0.62 * scale,
|
||||
color: item.flashColor,
|
||||
life: 1.0,
|
||||
decay: PHYSICS.FLASH_DECAY,
|
||||
})
|
||||
|
||||
// 2. 双层冲击波
|
||||
// 2. 双层冲击波（低强度时只保留一层）
|
||||
shockwaves.value.push({
|
||||
x: centerX, y: centerY,
|
||||
radius: 12, maxRadius: maxR * 0.62,
|
||||
radius: 12, maxRadius: maxR * 0.62 * scale,
|
||||
color: item.shockwaveColor, width: 5,
|
||||
life: 1.0, decay: PHYSICS.SHOCKWAVE_DECAY,
|
||||
})
|
||||
if (scale >= 0.8) {
|
||||
shockwaves.value.push({
|
||||
x: centerX, y: centerY,
|
||||
radius: 4, maxRadius: maxR * 0.42,
|
||||
radius: 4, maxRadius: maxR * 0.42 * scale,
|
||||
color: item.shockwaveColor, width: 3,
|
||||
life: 1.0, decay: PHYSICS.SHOCKWAVE_DECAY * 1.3,
|
||||
})
|
||||
}
|
||||
|
||||
const debrisCount = Math.max(8, Math.round(item.debrisCount * scale))
|
||||
const sparkCount = Math.max(6, Math.round(item.sparkCount * scale))
|
||||
const sparkleCount = Math.max(2, Math.round(item.sparkleCount * scale))
|
||||
|
||||
// 3. 碎片(主体飞溅)
|
||||
for (let i = 0; i < item.debrisCount; i++) {
|
||||
const angle = (Math.PI * 2 * i) / item.debrisCount + (Math.random() - 0.5) * 0.7
|
||||
const speed = item.speedMin + Math.random() * (item.speedMax - item.speedMin)
|
||||
const size = item.sizeMin + Math.random() * (item.sizeMax - item.sizeMin)
|
||||
for (let i = 0; i < debrisCount; i++) {
|
||||
const angle = (Math.PI * 2 * i) / debrisCount + (Math.random() - 0.5) * 0.7
|
||||
const speed = (item.speedMin + Math.random() * (item.speedMax - item.speedMin)) * (0.85 + scale * 0.15)
|
||||
const size = (item.sizeMin + Math.random() * (item.sizeMax - item.sizeMin)) * scale
|
||||
const color = item.colors[Math.floor(Math.random() * item.colors.length)]
|
||||
const shape = item.shapes[Math.floor(Math.random() * item.shapes.length)]
|
||||
|
||||
// confetti(彩纸): 初速带强烈向上偏移 + 水平摇摆下落
|
||||
const confetti = !!item.confetti
|
||||
const vy0 = confetti
|
||||
? Math.sin(angle) * speed - 2.5 // 先向上窜
|
||||
? Math.sin(angle) * speed - 2.5
|
||||
: Math.sin(angle) * speed
|
||||
|
||||
debrisList.value.push({
|
||||
@@ -330,18 +336,18 @@ function triggerCrush(itemType: CrushItemType) {
|
||||
rotation: Math.random() * Math.PI * 2,
|
||||
rotationSpeed: (Math.random() - 0.5) * (confetti ? 0.5 : 0.3),
|
||||
life: 1.0,
|
||||
decay: PHYSICS.DEBRIS_DECAY * (confetti ? 0.7 : 1),
|
||||
decay: PHYSICS.DEBRIS_DECAY * (confetti ? 0.7 : 1) * (scale < 0.7 ? 1.25 : 1),
|
||||
gravity: item.gravity,
|
||||
glow: !confetti, // 彩纸不发光,实色碎片发光
|
||||
glow: !confetti,
|
||||
swing: confetti ? 0.6 + Math.random() * 1.2 : 0,
|
||||
age: Math.random() * Math.PI * 2,
|
||||
})
|
||||
}
|
||||
|
||||
// 4. 火花(细小高速亮点,带拖尾)
|
||||
for (let i = 0; i < item.sparkCount; i++) {
|
||||
// 4. 火花
|
||||
for (let i = 0; i < sparkCount; i++) {
|
||||
const angle = Math.random() * Math.PI * 2
|
||||
const speed = item.speedMax * (0.9 + Math.random() * 0.8)
|
||||
const speed = item.speedMax * (0.9 + Math.random() * 0.8) * scale
|
||||
const color = item.sparkColors[Math.floor(Math.random() * item.sparkColors.length)]
|
||||
sparkList.value.push({
|
||||
x: centerX,
|
||||
@@ -359,8 +365,8 @@ function triggerCrush(itemType: CrushItemType) {
|
||||
}
|
||||
|
||||
// 5. 闪烁星光
|
||||
for (let i = 0; i < item.sparkleCount; i++) {
|
||||
const r = maxR * (0.08 + Math.random() * 0.32)
|
||||
for (let i = 0; i < sparkleCount; i++) {
|
||||
const r = maxR * (0.08 + Math.random() * 0.32) * scale
|
||||
const a = Math.random() * Math.PI * 2
|
||||
const color = item.sparkColors[Math.floor(Math.random() * item.sparkColors.length)]
|
||||
sparkleList.value.push({
|
||||
@@ -370,7 +376,7 @@ function triggerCrush(itemType: CrushItemType) {
|
||||
rotation: Math.random() * Math.PI,
|
||||
spin: (Math.random() - 0.5) * 0.16,
|
||||
color,
|
||||
life: 1.0 + Math.random() * 0.4, // 错峰出现
|
||||
life: 1.0 + Math.random() * 0.4,
|
||||
decay: PHYSICS.SPARKLE_DECAY * (0.8 + Math.random() * 0.5),
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user