ZYT/zyt
2
0
Fork 0
Code Issues Actions Packages Projects Releases Wiki Activity

更新

Browse Source
This commit is contained in:
Your Name
2026-08-27 14:23:23 +08:00
parent b5b14516a1
commit 2fa8492c56
27 changed files with 3832 additions and 1435 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/video_companion/src/main.ts
+196 -44
View File
@@ -26,6 +26,7 @@ interface NormalizedCallConfig {
targetUserId: string
diagnosisId: number | string
patientName: string
patientCase: UiPatientCase
mode: CompanionMode
}
@@ -44,9 +45,33 @@ interface UiLiveCaption {
id: string
speaker: string
text: string
time: string
completed: boolean
}
interface UiPatientCase {
diagnosisId: string
name: string
gender: string
age: string
height: string
weight: string
diagnosisDate: string
appointmentDate: string
clinicalDiagnosis: string
chiefComplaint: string
presentIllness: string
pastHistory: string
allergyHistory: string
personalHistory: string
familyHistory: string
currentMedication: string
tongue: string
pulse: string
prescriptionOpinion: string
remark: string
}
interface BridgeMessage {
source: 'doctor-call'
event:
@@ -56,6 +81,7 @@ interface BridgeMessage {
| 'room'
| 'hangup'
| 'error'
| 'open-diagnosis-request'
| 'transcription-start-request'
| 'transcription-segment'
| 'transcription-stop'
@@ -127,6 +153,11 @@ interface PendingLocalRecordingReply {
interface TrtcAudioTrackEvent {
userId?: string
track?: MediaStreamTrack
sourceTrack?: MediaStreamTrack
}
interface TrtcRemoteAudioEvent {
userId?: string
}
interface TrtcAudioCloud {
@@ -135,7 +166,9 @@ interface TrtcAudioCloud {
processed?: boolean
} | string): MediaStreamTrack | null
on?(event: 'track', handler: (event: TrtcAudioTrackEvent) => void): void
on?(event: 'remote-audio-available', handler: (event: TrtcRemoteAudioEvent) => void): void
off?(event: 'track', handler: (event: TrtcAudioTrackEvent) => void): void
off?(event: 'remote-audio-available', handler: (event: TrtcRemoteAudioEvent) => void): void
}
const phase = ref<CallPhase>('ready')
@@ -150,6 +183,7 @@ const hasMoreMessages = ref(false)
const transcriptionState = ref<TranscriptionState>('idle')
const localRecordingState = ref<LocalRecordingState>('idle')
const liveCaptions = ref<UiLiveCaption[]>([])
const patientCase = ref<UiPatientCase>(emptyPatientCase())
let activeConfig: NormalizedCallConfig | null = null
let chat: any = null
@@ -192,6 +226,7 @@ let localAudioContext: AudioContext | null = null
let localAudioDestination: MediaStreamAudioDestinationNode | null = null
let localAudioCloud: TrtcAudioCloud | null = null
let localAudioTrackHandler: ((event: TrtcAudioTrackEvent) => void) | null = null
let localRemoteAudioAvailableHandler: ((event: TrtcRemoteAudioEvent) => void) | null = null
let localAudioSources: MediaStreamAudioSourceNode[] = []
let localAudioTrackIds = new Set<string>()
let localAudioOwnedTracks: MediaStreamTrack[] = []
@@ -205,8 +240,6 @@ let localRecordingChunkChain: Promise<void> = Promise.resolve()
let localRecordingStartPromise: Promise<void> | null = null
let localRecordingStopPromise: Promise<void> | null = null
let localRecordingFatalError = ''
let liveCaptionClearTimer: number | null = null
function initializeQtWebChannel(): void {
const transport = window.qt?.webChannelTransport
const QWebChannel = window.QWebChannel
@@ -259,11 +292,80 @@ function emit(message: BridgeMessage): void {
}
}
async function openDiagnosis(): Promise<void> {
if (!activeConfig) throw new Error('诊单上下文尚未就绪')
emit({
source: 'doctor-call',
event: 'open-diagnosis-request',
diagnosisId: activeConfig.diagnosisId,
})
}
function cleanString(value: unknown, field: string): string {
if (typeof value !== 'string' || value.trim() === '') throw new Error(`${field}不能为空`)
return value.trim()
}
function optionalText(value: unknown, maxLength = 2000): string {
if (value === undefined || value === null) return ''
return String(value).trim().slice(0, maxLength)
}
function emptyPatientCase(): UiPatientCase {
return {
diagnosisId: '',
name: '',
gender: '',
age: '',
height: '',
weight: '',
diagnosisDate: '',
appointmentDate: '',
clinicalDiagnosis: '',
chiefComplaint: '',
presentIllness: '',
pastHistory: '',
allergyHistory: '',
personalHistory: '',
familyHistory: '',
currentMedication: '',
tongue: '',
pulse: '',
prescriptionOpinion: '',
remark: '',
}
}
function normalizePatientCase(
value: DoctorPatientCase | undefined,
diagnosisId: number | string,
fallbackName: string,
): UiPatientCase {
const source = value && typeof value === 'object' ? value : {}
return {
diagnosisId: optionalText(source.diagnosisId ?? diagnosisId, 80),
name: optionalText(source.name || fallbackName, 120) || fallbackName,
gender: optionalText(source.gender, 20),
age: optionalText(source.age, 20),
height: optionalText(source.height, 20),
weight: optionalText(source.weight, 20),
diagnosisDate: optionalText(source.diagnosisDate, 80),
appointmentDate: optionalText(source.appointmentDate, 80),
clinicalDiagnosis: optionalText(source.clinicalDiagnosis),
chiefComplaint: optionalText(source.chiefComplaint),
presentIllness: optionalText(source.presentIllness),
pastHistory: optionalText(source.pastHistory),
allergyHistory: optionalText(source.allergyHistory),
personalHistory: optionalText(source.personalHistory),
familyHistory: optionalText(source.familyHistory),
currentMedication: optionalText(source.currentMedication),
tongue: optionalText(source.tongue),
pulse: optionalText(source.pulse),
prescriptionOpinion: optionalText(source.prescriptionOpinion),
remark: optionalText(source.remark),
}
}
function normalizeConfig(config: DoctorCallConfig): NormalizedCallConfig {
if (!config || typeof config !== 'object') throw new Error('问诊配置无效')
const SDKAppID = Number(config.SDKAppID ?? config.sdkAppId)
@@ -274,15 +376,17 @@ function normalizeConfig(config: DoctorCallConfig): NormalizedCallConfig {
throw new Error('诊单ID不能为空')
}
const normalizedPatientName = typeof config.patientName === 'string' && config.patientName.trim()
? config.patientName.trim()
: '患者'
return {
SDKAppID,
userID: cleanString(config.userID ?? config.userId, '医生用户ID'),
userSig: cleanString(config.userSig, '用户签名'),
targetUserId: cleanString(config.targetUserId ?? config.patientUserId, '患者用户ID'),
diagnosisId: typeof diagnosisId === 'string' ? diagnosisId.trim() : diagnosisId,
patientName: typeof config.patientName === 'string' && config.patientName.trim()
? config.patientName.trim()
: '患者',
patientName: normalizedPatientName,
patientCase: normalizePatientCase(config.patientCase, diagnosisId, normalizedPatientName),
mode: config.mode === 'chat' ? 'chat' : 'video',
}
}
@@ -747,30 +851,50 @@ function attachLocalRecordingTrack(
return true
}
function attachDoctorAudioTrack(cloud: TrtcAudioCloud): boolean {
if (typeof cloud.getAudioTrack !== 'function') return false
let attached = false
try {
attached = attachLocalRecordingTrack(cloud.getAudioTrack({ processed: true }), 'doctor')
} catch {
// Some TRTC versions do not support processed tracks.
}
if (!attached) {
try {
attached = attachLocalRecordingTrack(cloud.getAudioTrack(), 'doctor')
} catch {
// The microphone fallback below remains available.
}
}
return attached
}
function attachPatientAudioTrack(cloud: TrtcAudioCloud, userId: string): boolean {
if (typeof cloud.getAudioTrack !== 'function' || !userId) return false
let attached = false
try {
attached = attachLocalRecordingTrack(cloud.getAudioTrack({
userId,
processed: true,
}), 'patient')
} catch {
// Some TRTC versions do not expose a processed remote track.
}
if (!attached) {
try {
attached = attachLocalRecordingTrack(cloud.getAudioTrack(userId), 'patient')
} catch {
// Remote audio can become available a few frames after connected.
}
}
return attached
}
function attachCurrentCallAudioTracks(cloud: TrtcAudioCloud | null): void {
if (!activeConfig) return
if (typeof cloud?.getAudioTrack === 'function') {
try {
attachLocalRecordingTrack(cloud.getAudioTrack({ processed: true }), 'doctor')
} catch {
try {
attachLocalRecordingTrack(cloud.getAudioTrack(), 'doctor')
} catch {
// The rendered media elements below remain a supported fallback.
}
}
try {
attachLocalRecordingTrack(cloud.getAudioTrack({
userId: activeConfig.targetUserId,
processed: true,
}), 'patient')
} catch {
try {
attachLocalRecordingTrack(cloud.getAudioTrack(activeConfig.targetUserId), 'patient')
} catch {
// Remote audio can become available a few frames after connected.
}
}
if (cloud) {
attachDoctorAudioTrack(cloud)
attachPatientAudioTrack(cloud, activeConfig.targetUserId)
}
for (const media of document.querySelectorAll<HTMLMediaElement>('video, audio')) {
@@ -852,8 +976,20 @@ async function cleanupLocalRecordingGraph(): Promise<void> {
// The call engine may already have released its event dispatcher.
}
}
if (
localAudioCloud
&& localRemoteAudioAvailableHandler
&& typeof localAudioCloud.off === 'function'
) {
try {
localAudioCloud.off('remote-audio-available', localRemoteAudioAvailableHandler)
} catch {
// The call engine may already have released its event dispatcher.
}
}
localAudioCloud = null
localAudioTrackHandler = null
localRemoteAudioAvailableHandler = null
for (const source of localAudioSources) {
try {
source.disconnect()
@@ -910,15 +1046,26 @@ async function performStartLocalRecording(): Promise<void> {
localAudioDestination = context.createMediaStreamDestination()
const cloud = getTrtcAudioCloud()
localAudioCloud = cloud
localAudioTrackHandler = (event) => attachLocalRecordingTrack(
event.track,
event.userId === activeConfig?.userID
localAudioTrackHandler = (event) => {
const sourceKind = !event.userId || event.userId === activeConfig?.userID
? 'doctor'
: event.userId === activeConfig?.targetUserId
? 'patient'
: 'unknown',
)
if (typeof cloud?.on === 'function') cloud.on('track', localAudioTrackHandler)
: 'unknown'
if (!attachLocalRecordingTrack(event.track, sourceKind)) {
attachLocalRecordingTrack(event.sourceTrack, sourceKind)
}
}
localRemoteAudioAvailableHandler = (event) => {
const userId = event.userId || activeConfig?.targetUserId || ''
if (cloud && userId === activeConfig?.targetUserId) {
attachPatientAudioTrack(cloud, userId)
}
}
if (typeof cloud?.on === 'function') {
cloud.on('track', localAudioTrackHandler)
cloud.on('remote-audio-available', localRemoteAudioAvailableHandler)
}
if (context.state === 'suspended') await context.resume()
await waitForCallAudioTracks(cloud, sessionId)
startCallAudioDiscovery(cloud)
@@ -1151,10 +1298,6 @@ function getTranscriberManager(): RealtimeTranscriberManager {
}
function clearLiveCaptions(): void {
if (liveCaptionClearTimer !== null) {
window.clearTimeout(liveCaptionClearTimer)
liveCaptionClearTimer = null
}
liveCaptions.value = []
}
@@ -1170,19 +1313,26 @@ function showLiveCaption(message: RealtimeTranscriberMessage): void {
: speakerUserId === activeConfig.targetUserId
? patientName.value || '患者'
: '对话'
const rawTimestamp = Number(message.timestamp)
const captionTimestamp = Number.isFinite(rawTimestamp) && rawTimestamp > 0
? (rawTimestamp < 1_000_000_000_000 ? rawTimestamp * 1000 : rawTimestamp)
: Date.now()
const caption: UiLiveCaption = {
id,
speaker,
text: text.slice(0, 500),
time: new Date(captionTimestamp).toLocaleTimeString('zh-CN', {
hour: '2-digit',
minute: '2-digit',
hour12: false,
}),
completed: message.isCompleted === true,
}
const previous = liveCaptions.value.filter((item) => item.id !== id)
liveCaptions.value = [...previous, caption].slice(-2)
if (liveCaptionClearTimer !== null) window.clearTimeout(liveCaptionClearTimer)
liveCaptionClearTimer = window.setTimeout(() => {
liveCaptions.value = []
liveCaptionClearTimer = null
}, message.isCompleted === true ? 9000 : 5000)
// Keep the current call's transcript visible in the side rail. Repeated
// partial updates replace the same segment and the bounded history prevents
// long calls from growing memory without limit.
liveCaptions.value = [...previous, caption].slice(-120)
}
function handleTranscriberMessage(
@@ -1421,7 +1571,6 @@ async function stopTranscription(
transcriptionSessionId = ''
acknowledgedSegmentIds.clear()
pendingSegments.clear()
clearLiveCaptions()
})().finally(() => {
transcriptionStopPromise = null
})
@@ -1720,6 +1869,7 @@ async function open(config: DoctorCallConfig): Promise<void> {
activeConfig = normalizeConfig(config)
mode.value = activeConfig.mode
patientName.value = activeConfig.patientName
patientCase.value = activeConfig.patientCase
messages.value = []
nextReqMessageID = ''
hasMoreMessages.value = false
@@ -1805,12 +1955,14 @@ createApp(App, {
transcriptionState: readonly(transcriptionState),
localRecordingState: readonly(localRecordingState),
liveCaptions: readonly(liveCaptions),
patientCase: readonly(patientCase),
onSendText: sendText,
onSendAttachment: sendAttachment,
onLoadMore: () => loadMessages(true),
onReconnectChat: reconnectChat,
onStartVideo: startVideo,
onHangup: hangup,
onOpenDiagnosis: openDiagnosis,
onSaveScreenshot: saveScreenshot,
}).mount('#app')