This commit is contained in:
Your Name
2026-08-12 17:18:56 +08:00
parent 28cd110dae
commit f48a66b611
24 changed files with 3095 additions and 233 deletions
+23
View File
@@ -23,6 +23,7 @@ const props = defineProps<{
chatBusy: Readonly<Ref<boolean>>
notice: Readonly<Ref<string>>
hasMoreMessages: Readonly<Ref<boolean>>
transcriptionState: Readonly<Ref<string>>
onSendText: (text: string) => Promise<void>
onSendAttachment: (file: File) => Promise<void>
onLoadMore: () => Promise<void>
@@ -42,6 +43,15 @@ const isChat = computed(() => props.mode.value === 'chat')
const isCalling = computed(() => ['starting', 'dialing', 'connected'].includes(props.phase.value))
const videoVisible = computed(() => !isChat.value || isCalling.value)
const canCapture = computed(() => props.phase.value === 'connected')
const transcriptionActive = computed(() => props.transcriptionState.value === 'recording')
const transcriptionFailed = computed(() => props.transcriptionState.value === 'error')
const transcriptionStatusText = computed(() => {
if (props.transcriptionState.value === 'starting') return '自动录音启动中…'
if (props.transcriptionState.value === 'recording') return '自动录音并转文字中'
if (props.transcriptionState.value === 'stopping') return '正在保存录音文字…'
if (props.transcriptionState.value === 'error') return '自动录音转文字失败'
return '自动录音已结束'
})
watch(
() => props.messages.value.length,
@@ -242,6 +252,19 @@ async function captureScreenshot(): Promise<void> {
</div>
<div v-if="isCalling" class="video-actions">
<div
v-if="canCapture"
class="recording-status"
:class="{
'recording-status--active': transcriptionActive,
'recording-status--error': transcriptionFailed,
}"
role="status"
aria-live="polite"
>
<span class="recording-indicator" aria-hidden="true" />
{{ transcriptionStatusText }}
</div>
<button
class="capture-button"
type="button"
+7
View File
@@ -30,6 +30,13 @@ interface DoctorConsultationApi {
hangup(): Promise<void>
hostCallReady(ok: boolean, message?: string): void
screenshotResult(ok: boolean, message: string): void
transcriptionResult(
operation: 'start' | 'segment' | 'stop',
sessionId: string,
segmentId: string,
ok: boolean,
message: string,
): void
}
interface QtVideoBridge {
+493 -17
View File
@@ -15,6 +15,7 @@ import './style.css'
type CallPhase = 'ready' | 'starting' | 'dialing' | 'connected' | 'ended' | 'error'
type CompanionMode = 'chat' | 'video'
type ChatMessageType = 'text' | 'image' | 'file' | 'audio' | 'video' | 'system'
type TranscriptionState = 'idle' | 'starting' | 'recording' | 'stopping' | 'error'
interface NormalizedCallConfig {
SDKAppID: number
@@ -38,11 +39,74 @@ interface UiChatMessage {
interface BridgeMessage {
source: 'doctor-call'
event: 'ready' | 'call-start-request' | 'status' | 'room' | 'hangup' | 'error'
event:
| 'ready'
| 'call-start-request'
| 'status'
| 'room'
| 'hangup'
| 'error'
| 'transcription-start-request'
| 'transcription-segment'
| 'transcription-stop'
diagnosisId?: number | string
status?: string
roomId?: string
message?: string
sessionId?: string
language?: string
segment?: {
segment_id: string
speaker_user_id: string
speaker_role: 'doctor' | 'patient' | 'unknown'
timestamp: number
text: string
}
}
interface RealtimeTranscriberMessage {
segmentId: string
speakerUserId: string
sourceText: string
timestamp: number
isCompleted: boolean
}
interface RealtimeTranscriberListener {
onReceiveTranscriberMessage: (
roomId: string | number,
message: RealtimeTranscriberMessage,
) => void
onRealtimeTranscriberStarted: (
roomId: string | number,
robotId: string,
sourceLanguage: string,
) => void
onRealtimeTranscriberStopped: (roomId: string | number, robotId: string) => void
onRealtimeTranscriberError: (
roomId: string | number,
robotId: string,
error: number,
errorMessage: string,
) => void
}
interface RealtimeTranscriberManager {
addListener(listener: RealtimeTranscriberListener): void
removeListener(listener: RealtimeTranscriberListener): void
startRealtimeTranscriber(config: { sourceLanguage: string }): Promise<string>
stopRealtimeTranscriber(robotId: string): Promise<void>
}
interface PendingTranscriptionReply {
sessionId: string
resolve: (value: boolean) => void
promise: Promise<boolean>
}
interface PendingSegment {
message: BridgeMessage
attempts: number
}
const phase = ref<CallPhase>('ready')
@@ -54,6 +118,7 @@ const chatReady = ref(false)
const chatBusy = ref(false)
const notice = ref('')
const hasMoreMessages = ref(false)
const transcriptionState = ref<TranscriptionState>('idle')
let activeConfig: NormalizedCallConfig | null = null
let chat: any = null
@@ -65,6 +130,23 @@ let endNotified = true
let starting = false
let emittedRoomId = ''
let resolveHostCallReady: ((value: boolean) => void) | null = null
const pendingTranscriptionStarts = new Map<string, PendingTranscriptionReply>()
const pendingTranscriptionStops = new Map<string, PendingTranscriptionReply>()
let transcriptionSessionId = ''
let transcriptionGeneration = 0
let transcriberRunning = false
let transcriberManager: RealtimeTranscriberManager | null = null
let transcriberListener: RealtimeTranscriberListener | null = null
let transcriberRobotId = ''
let transcriptionStartPromise: Promise<void> | null = null
let transcriptionStopPromise: Promise<void> | null = null
let hangupNotification: Promise<void> | null = null
let callCycleGeneration = 0
let autoTranscriptionAttemptedGeneration = -1
let lastTranscriberMessageAt = 0
let transcriberStoppedAt = 0
const acknowledgedSegmentIds = new Set<string>()
const pendingSegments = new Map<string, PendingSegment>()
function initializeQtWebChannel(): void {
const transport = window.qt?.webChannelTransport
@@ -457,17 +539,372 @@ async function sendAttachment(file: File): Promise<void> {
}
}
function notifyHangup(status = 'ended'): void {
if (endNotified) return
function newTranscriptionSessionId(): string {
const random = window.crypto?.randomUUID?.()
return random ? `call-${random}` : `call-${Date.now()}-${Math.random().toString(16).slice(2)}`
}
function requestTranscriptionStart(sessionId: string): Promise<boolean> {
if (!activeConfig || !window.qtVideoBridge?.notify) return Promise.resolve(false)
const existing = pendingTranscriptionStarts.get(sessionId)
if (existing) return existing.promise
const pending = {} as PendingTranscriptionReply
pending.sessionId = sessionId
pending.promise = new Promise<boolean>((resolve) => {
pending.resolve = resolve
})
pendingTranscriptionStarts.set(sessionId, pending)
emit({
source: 'doctor-call',
event: 'transcription-start-request',
diagnosisId: activeConfig?.diagnosisId,
sessionId,
language: 'zh',
})
window.setTimeout(() => {
if (pendingTranscriptionStarts.get(sessionId) !== pending) return
pendingTranscriptionStarts.delete(sessionId)
pending.resolve(false)
}, 15000)
return pending.promise
}
function requestTranscriptionStop(
sessionId: string,
status: 'completed' | 'partial' | 'failed',
): Promise<boolean> {
if (!activeConfig || !sessionId || !window.qtVideoBridge?.notify) {
return Promise.resolve(false)
}
const existing = pendingTranscriptionStops.get(sessionId)
if (existing) return existing.promise
const pending = {} as PendingTranscriptionReply
pending.sessionId = sessionId
pending.promise = new Promise<boolean>((resolve) => {
pending.resolve = resolve
})
pendingTranscriptionStops.set(sessionId, pending)
emit({
source: 'doctor-call',
event: 'transcription-stop',
diagnosisId: activeConfig?.diagnosisId,
sessionId,
status,
})
window.setTimeout(() => {
if (pendingTranscriptionStops.get(sessionId) !== pending) return
pendingTranscriptionStops.delete(sessionId)
pending.resolve(false)
}, 15000)
return pending.promise
}
function getTranscriberManager(): RealtimeTranscriberManager {
const engine = TUICallKitAPI.getTUICallEngineInstance?.()
const cloud = engine?.getTRTCCloudInstance?.()
const manager = cloud?.getAITranscriberManager?.() as Partial<RealtimeTranscriberManager> | null
if (
!manager
|| typeof manager.addListener !== 'function'
|| typeof manager.removeListener !== 'function'
|| typeof manager.startRealtimeTranscriber !== 'function'
|| typeof manager.stopRealtimeTranscriber !== 'function'
) {
throw new Error('当前视频服务未开通实时语音转写')
}
return manager as RealtimeTranscriberManager
}
function handleTranscriberMessage(
_roomId: string | number,
message: RealtimeTranscriberMessage,
): void {
if (
!activeConfig
|| !['recording', 'stopping'].includes(transcriptionState.value)
|| !transcriptionSessionId
|| message.isCompleted !== true
) return
lastTranscriberMessageAt = Date.now()
const segmentId = String(message.segmentId ?? '').trim()
const text = String(message.sourceText ?? '').trim()
if (
!segmentId
|| acknowledgedSegmentIds.has(segmentId)
|| pendingSegments.has(segmentId)
|| !text
) return
const speakerUserId = String(message.speakerUserId ?? '').trim()
const bridgeMessage: BridgeMessage = {
source: 'doctor-call',
event: 'transcription-segment',
diagnosisId: activeConfig.diagnosisId,
sessionId: transcriptionSessionId,
segment: {
segment_id: segmentId.slice(0, 160),
speaker_user_id: speakerUserId.slice(0, 160),
speaker_role: speakerUserId === activeConfig.userID
? 'doctor'
: speakerUserId === activeConfig.targetUserId
? 'patient'
: 'unknown',
timestamp: Math.max(0, Math.trunc(Number(message.timestamp) || 0)),
text: text.slice(0, 4000),
},
}
pendingSegments.set(segmentId, { message: bridgeMessage, attempts: 1 })
emit(bridgeMessage)
}
function subscribeTranscriber(): {
manager: RealtimeTranscriberManager
listener: RealtimeTranscriberListener
} {
if (transcriberManager && transcriberListener) {
return { manager: transcriberManager, listener: transcriberListener }
}
const manager = getTranscriberManager()
const listener: RealtimeTranscriberListener = {
onReceiveTranscriberMessage: handleTranscriberMessage,
onRealtimeTranscriberStarted: () => undefined,
onRealtimeTranscriberStopped: (_roomId, robotId) => {
if (robotId !== transcriberRobotId) return
transcriberRunning = false
transcriberStoppedAt = Date.now()
if (transcriptionState.value === 'recording') {
void stopTranscription('partial', true)
}
},
onRealtimeTranscriberError: (_roomId, robotId, _error, errorMessage) => {
if (robotId !== transcriberRobotId || transcriptionState.value === 'stopping') return
notice.value = safeErrorMessage(new Error(errorMessage), '实时语音转写发生错误')
void stopTranscription('partial', true)
},
}
manager.addListener(listener)
transcriberManager = manager
transcriberListener = listener
return { manager, listener }
}
function unsubscribeTranscriber(
manager = transcriberManager,
listener = transcriberListener,
): void {
if (manager && listener) {
try {
manager.removeListener(listener)
} catch {
// A destroyed call engine has already released the listener.
}
}
if (transcriberManager === manager && transcriberListener === listener) {
transcriberManager = null
transcriberListener = null
}
}
function transcriptionTokenIsCurrent(token: number, sessionId: string): boolean {
return (
token === transcriptionGeneration
&& sessionId === transcriptionSessionId
&& transcriptionState.value === 'starting'
&& phase.value === 'connected'
&& !endNotified
)
}
async function waitForPendingSegments(timeoutMs = 3000, quietMs = 300): Promise<boolean> {
const startedAt = Date.now()
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
const stoppedOrSettled = transcriberStoppedAt > 0 || Date.now() - startedAt >= 500
const quiet = Date.now() - lastTranscriberMessageAt >= quietMs
if (stoppedOrSettled && quiet && pendingSegments.size === 0) return true
await new Promise((resolve) => window.setTimeout(resolve, 50))
}
return false
}
async function performStartTranscription(): Promise<void> {
if (!activeConfig || phase.value !== 'connected') throw new Error('视频接通后才能开始录音')
if (transcriptionState.value !== 'idle' && transcriptionState.value !== 'error') {
throw new Error('录音任务正在处理中')
}
transcriptionState.value = 'starting'
const sessionId = newTranscriptionSessionId()
const token = ++transcriptionGeneration
transcriptionSessionId = sessionId
acknowledgedSegmentIds.clear()
pendingSegments.clear()
lastTranscriberMessageAt = Date.now()
transcriberStoppedAt = 0
notice.value = '正在准备录音文字存储…'
const storageReady = await requestTranscriptionStart(sessionId)
if (!storageReady) {
if (!transcriptionTokenIsCurrent(token, sessionId)) return
transcriptionState.value = 'error'
transcriptionSessionId = ''
throw new Error(notice.value || '服务端无法保存本次面诊对话文字')
}
if (!transcriptionTokenIsCurrent(token, sessionId)) {
await requestTranscriptionStop(sessionId, 'partial')
return
}
let ownedManager: RealtimeTranscriberManager | null = null
let ownedListener: RealtimeTranscriberListener | null = null
try {
const subscribed = subscribeTranscriber()
const { manager, listener } = subscribed
ownedManager = manager
ownedListener = listener
const robotId = await manager.startRealtimeTranscriber({
sourceLanguage: 'zh',
})
if (!robotId) throw new Error('当前腾讯云项目未开通实时语音转写')
if (!transcriptionTokenIsCurrent(token, sessionId)) {
await manager.stopRealtimeTranscriber(robotId)
unsubscribeTranscriber(manager, listener)
return
}
transcriberRobotId = robotId
transcriberRunning = true
transcriptionState.value = 'recording'
notice.value = '正在录音并实时转换为对话文字'
} catch (error) {
unsubscribeTranscriber(ownedManager, ownedListener)
await requestTranscriptionStop(sessionId, 'failed')
if (sessionId === transcriptionSessionId) transcriptionState.value = 'error'
const message = safeErrorMessage(error, '录音转文字启动失败')
notice.value = message
if (sessionId === transcriptionSessionId) transcriptionSessionId = ''
throw new Error(message)
}
}
function startTranscription(): Promise<void> {
if (transcriptionStartPromise) return transcriptionStartPromise
const operation = performStartTranscription()
const tracked = operation.finally(() => {
if (transcriptionStartPromise === tracked) transcriptionStartPromise = null
})
transcriptionStartPromise = tracked
return transcriptionStartPromise
}
async function stopTranscription(
status: 'completed' | 'partial' | 'failed' = 'completed',
managerAlreadyStopped = false,
): Promise<void> {
if (transcriptionStopPromise) return transcriptionStopPromise
if (transcriptionState.value === 'idle') return
if (!transcriptionSessionId) {
transcriptionState.value = 'idle'
return
}
const sessionId = transcriptionSessionId
const startInFlight = transcriptionStartPromise
transcriptionGeneration += 1
transcriptionState.value = 'stopping'
transcriptionStopPromise = (async () => {
if (startInFlight) {
try {
await startInFlight
} catch {
// Its failure is materialized as a failed/partial transcript below.
}
}
let sdkStopped = managerAlreadyStopped
const manager = transcriberManager
const robotId = transcriberRobotId
if (!managerAlreadyStopped && transcriberRunning && manager && robotId) {
try {
await manager.stopRealtimeTranscriber(robotId)
sdkStopped = true
} catch (error) {
console.warn('[doctor-consultation] 停止实时转写失败', safeErrorMessage(error))
}
}
transcriberRunning = false
transcriberRobotId = ''
const segmentsComplete = await waitForPendingSegments()
unsubscribeTranscriber()
const finalStatus = sdkStopped && segmentsComplete ? status : 'partial'
const persisted = await requestTranscriptionStop(sessionId, finalStatus)
if (sessionId !== transcriptionSessionId) return
transcriptionState.value = persisted ? 'idle' : 'error'
notice.value = persisted
? '本次面诊对话文字已保存到视频记录'
: '对话文字未能完整确认保存,请稍后在面诊记录中检查'
transcriptionSessionId = ''
acknowledgedSegmentIds.clear()
pendingSegments.clear()
})().finally(() => {
transcriptionStopPromise = null
})
return transcriptionStopPromise
}
function transcriptionResult(
operation: 'start' | 'segment' | 'stop',
sessionId: string,
segmentId: string,
ok: boolean,
message: string,
): void {
if (operation === 'start') {
const pending = pendingTranscriptionStarts.get(sessionId)
if (!pending) return
pendingTranscriptionStarts.delete(sessionId)
if (message && sessionId === transcriptionSessionId) notice.value = message
pending.resolve(Boolean(ok))
} else if (operation === 'stop') {
const pending = pendingTranscriptionStops.get(sessionId)
if (!pending) return
pendingTranscriptionStops.delete(sessionId)
if (message && sessionId === transcriptionSessionId) notice.value = message
pending.resolve(Boolean(ok))
} else if (operation === 'segment' && sessionId === transcriptionSessionId) {
const pending = pendingSegments.get(segmentId)
if (!pending) return
if (ok) {
pendingSegments.delete(segmentId)
acknowledgedSegmentIds.add(segmentId)
} else if (pending.attempts < 3) {
pending.attempts += 1
window.setTimeout(() => {
if (pendingSegments.get(segmentId) === pending) emit(pending.message)
}, pending.attempts * 250)
} else {
notice.value = message || '部分对话文字保存失败,本次记录将标记为部分保存'
}
}
}
function notifyHangup(status = 'ended'): Promise<void> {
if (hangupNotification) return hangupNotification
if (endNotified) return Promise.resolve()
endNotified = true
phase.value = 'ended'
statusText.value = mode.value === 'chat' ? '视频通话已结束,IM 保持连接' : '视频问诊已结束'
emit({
source: 'doctor-call',
event: 'hangup',
diagnosisId: activeConfig?.diagnosisId,
status,
})
hangupNotification = (async () => {
try {
if (transcriptionState.value !== 'idle') await stopTranscription('completed')
} catch (error) {
notice.value = safeErrorMessage(error, '录音文字收尾失败,请稍后检查面诊记录')
console.warn('[doctor-consultation] 录音文字收尾失败', notice.value)
} finally {
emit({
source: 'doctor-call',
event: 'hangup',
diagnosisId: activeConfig?.diagnosisId,
status,
})
}
})()
return hangupNotification
}
function readRoomId(): string {
@@ -498,21 +935,31 @@ function handleStatusChanged(payload: unknown): void {
: payload
const status = typeof value === 'string' ? value : 'unknown'
if (status === 'connected' || status.startsWith('calling-')) {
if (endNotified || !activeConfig) return
const cycle = callCycleGeneration
phase.value = 'connected'
statusText.value = '视频问诊进行中'
void pollRoomId()
if (autoTranscriptionAttemptedGeneration !== cycle) {
autoTranscriptionAttemptedGeneration = cycle
void startTranscription().catch((error) => {
if (cycle !== callCycleGeneration || endNotified) return
notice.value = safeErrorMessage(error, '自动录音转文字启动失败')
})
}
} else if (status === 'calling' || status.startsWith('dialing')) {
if (endNotified || !activeConfig) return
phase.value = 'dialing'
statusText.value = '正在等待患者接听'
} else if (status === 'idle' && activeConfig && !starting) {
notifyHangup(status)
void notifyHangup(status)
}
emit({ source: 'doctor-call', event: 'status', diagnosisId: activeConfig?.diagnosisId, status })
}
TUICallKitAPI.setCallback({
statusChanged: handleStatusChanged,
afterCalling: () => notifyHangup('after-calling'),
afterCalling: () => { void notifyHangup('after-calling') },
})
TUICallKitAPI.setLanguage('zh-cn')
TUICallKitAPI.enableFloatWindow(false)
@@ -543,11 +990,17 @@ async function startVideo(): Promise<void> {
if (!activeConfig) throw new Error('问诊配置尚未准备好')
if (starting || !endNotified) throw new Error('已有视频通话正在进行')
starting = true
endNotified = false
phase.value = 'starting'
statusText.value = '正在创建安全视频通话'
notice.value = ''
try {
if (hangupNotification) await hangupNotification
if (!activeConfig || !endNotified) throw new Error('已有视频通话正在进行')
hangupNotification = null
callCycleGeneration += 1
endNotified = false
phase.value = 'starting'
statusText.value = '正在创建安全视频通话'
transcriptionState.value = 'idle'
transcriptionSessionId = ''
notice.value = ''
const allowed = await requestHostCallStart()
if (!allowed) throw new Error(notice.value || '服务器未能创建视频通话记录')
await TUICallKitAPI.init({
@@ -579,10 +1032,14 @@ async function startVideo(): Promise<void> {
}
async function hangup(): Promise<void> {
if (!activeConfig || endNotified) return
if (!activeConfig) return
if (endNotified) {
if (hangupNotification) await hangupNotification
return
}
try {
await TUICallKitAPI.hangup()
notifyHangup('local-hangup')
await notifyHangup('local-hangup')
} catch (error) {
const message = safeErrorMessage(error, '结束视频通话失败')
emit({ source: 'doctor-call', event: 'error', diagnosisId: activeConfig.diagnosisId, message })
@@ -610,7 +1067,17 @@ async function open(config: DoctorCallConfig): Promise<void> {
nextReqMessageID = ''
hasMoreMessages.value = false
endNotified = true
hangupNotification = null
callCycleGeneration += 1
autoTranscriptionAttemptedGeneration = -1
phase.value = 'ready'
transcriptionState.value = 'idle'
transcriptionSessionId = ''
transcriptionGeneration += 1
transcriberRunning = false
transcriberRobotId = ''
acknowledgedSegmentIds.clear()
pendingSegments.clear()
notice.value = ''
if (activeConfig.mode === 'chat') {
try {
@@ -629,6 +1096,13 @@ async function open(config: DoctorCallConfig): Promise<void> {
async function close(): Promise<void> {
if (!endNotified) await hangup()
if (hangupNotification) await hangupNotification
unsubscribeTranscriber()
transcriptionGeneration += 1
for (const pending of pendingTranscriptionStarts.values()) pending.resolve(false)
for (const pending of pendingTranscriptionStops.values()) pending.resolve(false)
pendingTranscriptionStarts.clear()
pendingTranscriptionStops.clear()
await logoutChat()
activeConfig = null
phase.value = 'ended'
@@ -641,6 +1115,7 @@ window.doctorConsultation = {
hangup,
hostCallReady,
screenshotResult,
transcriptionResult,
}
window.doctorCall = { start: open, hangup }
initializeQtWebChannel()
@@ -655,6 +1130,7 @@ createApp(App, {
chatBusy: readonly(chatBusy),
notice: readonly(notice),
hasMoreMessages: readonly(hasMoreMessages),
transcriptionState: readonly(transcriptionState),
onSendText: sendText,
onSendAttachment: sendAttachment,
onLoadMore: () => loadMessages(true),
+25
View File
@@ -294,6 +294,31 @@ button:disabled { cursor: not-allowed; opacity: .55; }
gap: 10px;
}
.video-actions button { padding: 10px 15px; border-radius: 10px; font-weight: 600; }
.recording-status {
display: inline-flex;
align-items: center;
gap: 8px;
padding: 10px 15px;
border: 1px solid rgba(255, 255, 255, .24);
border-radius: 10px;
color: #fff;
background: rgba(22, 29, 39, .88);
font-size: 13px;
font-weight: 600;
}
.recording-status--active { border-color: rgba(244, 99, 115, .64); background: rgba(126, 35, 50, .9); }
.recording-status--error { border-color: rgba(242, 109, 109, .56); color: #ffe4e7; background: rgba(100, 31, 40, .88); }
.recording-indicator {
width: 9px;
height: 9px;
border-radius: 50%;
background: #f46373;
box-shadow: 0 0 0 4px rgba(244, 99, 115, .16);
}
.recording-status--active .recording-indicator { animation: recording-pulse 1.25s ease-in-out infinite; }
@keyframes recording-pulse {
50% { box-shadow: 0 0 0 8px rgba(244, 99, 115, .04); opacity: .72; }
}
.hangup-button { border: 1px solid #b44755; color: #fff; background: rgba(161, 47, 61, .9); }
.hangup-button:hover { background: #be394d; }
.video-notice {