This commit is contained in:
Your Name
2026-08-12 17:18:56 +08:00
parent 28cd110dae
commit f48a66b611
24 changed files with 3095 additions and 233 deletions
@@ -5,7 +5,7 @@ from __future__ import annotations
import hashlib
import struct
import zlib
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from copy import deepcopy
from datetime import date, datetime, timedelta
from os import PathLike
@@ -163,8 +163,27 @@ class DemoDoctorRepository:
"room_id": "demo-room-501",
"status": 2,
"status_text": "已结束",
"recording_status_text": "录制完成",
"recording_urls_list": ["https://media.example.invalid/demo/diagnosis-501.mp4"],
"recording_status_text": "录制完成",
"recording_urls_list": ["https://media.example.invalid/demo/diagnosis-501.mp4"],
"transcription_status": "completed",
"transcription_status_text": "文字已生成",
"transcript_text": "患者:最近睡眠比上周好一些。\n医生:继续记录睡眠和空腹血糖。",
"transcript_segments": [
{
"segment_id": "demo-1",
"speaker_role": "patient",
"speaker_user_id": "patient_301",
"timestamp": 1,
"text": "最近睡眠比上周好一些。",
},
{
"segment_id": "demo-2",
"speaker_role": "doctor",
"speaker_user_id": "doctor_1001",
"timestamp": 2,
"text": "继续记录睡眠和空腹血糖。",
},
],
"start_time_text": f"{self._today.isoformat()} 09:10:00",
"end_time_text": f"{self._today.isoformat()} 09:22:00",
"duration_text": "12分00秒",
@@ -2233,8 +2252,12 @@ class DemoDoctorRepository:
"room_id": "manual-upload",
"status": 2,
"status_text": "已结束",
"recording_status_text": "待上传",
"recording_urls_list": [],
"recording_status_text": "待上传",
"recording_urls_list": [],
"transcription_status": "not_started",
"transcription_status_text": "未生成文字",
"transcript_text": "",
"transcript_segments": [],
"start_time_text": datetime.now().replace(microsecond=0).isoformat(sep=" "),
"end_time_text": datetime.now().replace(microsecond=0).isoformat(sep=" "),
"duration_text": "0秒",
@@ -2531,8 +2554,12 @@ class DemoDoctorRepository:
replay = {
**record,
"status_text": "呼叫中",
"recording_status_text": "未录制",
"recording_urls_list": [],
"recording_status_text": "未录制",
"recording_urls_list": [],
"transcription_status": "not_started",
"transcription_status_text": "未生成文字",
"transcript_text": "",
"transcript_segments": [],
"start_time_text": datetime.now().replace(microsecond=0).isoformat(sep=" "),
"end_time_text": "",
"duration_text": "",
@@ -2567,7 +2594,7 @@ class DemoDoctorRepository:
)
return deepcopy(record)
def bind_call_room(self, diagnosis_id: int, room_id: str) -> dict[str, Any]:
def bind_call_room(self, diagnosis_id: int, room_id: str) -> dict[str, Any]:
"""Persist the room identifier on the active demo call."""
if not room_id.strip():
@@ -2594,11 +2621,133 @@ class DemoDoctorRepository:
"status_text": "通话中",
}
)
return {
"diagnosis_id": diagnosis_id,
"room_id": room_id.strip(),
"cloud_recording": {"started": False, "message": "演示模式不录制"},
}
return {
"diagnosis_id": diagnosis_id,
"room_id": room_id.strip(),
"cloud_recording": {"started": False, "message": "演示模式不录制"},
}
def _demo_call_record(
self, diagnosis_id: int, call_record_id: int | str
) -> dict[str, Any]:
record = next(
(
row
for row in self._call_records.get(diagnosis_id, [])
if str(row.get("id") or row.get("call_record_id") or "")
== str(call_record_id)
),
None,
)
if record is None:
raise RepositoryNotFoundError(f"call record {call_record_id} not found")
return record
def start_call_transcription(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
*,
language: str = "zh-CN",
) -> dict[str, Any]:
"""Start an in-memory transcript attached to one demo call record."""
clean_session = transcription_session_id.strip()
if not clean_session:
raise ValueError("transcription_session_id is required")
with self._lock:
self._find_consultation(diagnosis_id)
record = self._demo_call_record(diagnosis_id, call_record_id)
existing = str(record.get("transcription_session_id") or "")
if existing and existing != clean_session:
raise ValueError("another transcription session already exists")
record.update(
{
"transcription_session_id": clean_session,
"transcription_language": language.strip() or "zh-CN",
"transcription_status": "running",
"transcription_status_text": "录音转写中",
"transcript_text": "",
"transcript_segments": [],
}
)
return deepcopy(record)
def upsert_call_transcript_segments(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
segments: Sequence[Mapping[str, Any]],
) -> dict[str, Any]:
"""Idempotently append/update completed segments in demo mode."""
with self._lock:
record = self._demo_call_record(diagnosis_id, call_record_id)
if record.get("transcription_session_id") != transcription_session_id:
raise ValueError("transcription session does not match the call record")
stored = record.setdefault("transcript_segments", [])
for incoming in segments:
segment_id = str(incoming.get("segment_id") or "").strip()
text = str(incoming.get("text") or "").strip()
if not segment_id or not text:
raise ValueError("transcript segment_id and text are required")
normalized = {
"segment_id": segment_id,
"speaker_role": str(incoming.get("speaker_role") or "unknown"),
"speaker_user_id": str(incoming.get("speaker_user_id") or ""),
"timestamp": int(incoming.get("timestamp") or 0),
"text": text,
}
current = next(
(row for row in stored if row.get("segment_id") == segment_id), None
)
if current is None:
stored.append(normalized)
else:
current.update(normalized)
stored.sort(key=lambda row: (int(row.get("timestamp") or 0), row["segment_id"]))
labels = {"doctor": "医生", "patient": "患者", "unknown": "未知说话人"}
record["transcript_text"] = "\n".join(
f"{labels.get(str(row.get('speaker_role')), '未知说话人')}{row['text']}"
for row in stored
)
return deepcopy(record)
def finish_call_transcription(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
expected_segment_count: int,
*,
status: str = "completed",
) -> dict[str, Any]:
"""Finalize demo transcript fields on the same call record."""
clean_status = status.strip().lower()
if clean_status not in {"completed", "partial", "failed"}:
raise ValueError("transcription status is invalid")
with self._lock:
record = self._demo_call_record(diagnosis_id, call_record_id)
if record.get("transcription_session_id") != transcription_session_id:
raise ValueError("transcription session does not match the call record")
actual_count = len(record.get("transcript_segments") or [])
final_status = clean_status
if clean_status == "completed" and actual_count < expected_segment_count:
final_status = "partial"
record["transcription_status"] = final_status
record["transcription_status_text"] = {
"completed": "文字已生成",
"partial": "文字部分保存",
"failed": "文字生成失败",
}[final_status]
record["transcription_segment_count"] = actual_count
record["transcription_finished_at"] = datetime.now().replace(
microsecond=0
).isoformat(sep=" ")
return deepcopy(record)
def my_self(self) -> Session:
"""Compatibility alias for :meth:`get_session`."""
+175 -20
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import mimetypes
import re
import time
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from contextlib import suppress
from datetime import date
from io import BytesIO
@@ -619,8 +619,38 @@ class DoctorRepository(Protocol):
def end_call(self, diagnosis_id: int) -> Any:
"""End the active diagnosis call."""
def bind_call_room(self, diagnosis_id: int, room_id: str) -> Any:
"""Bind a TRTC room to the active call."""
def bind_call_room(self, diagnosis_id: int, room_id: str) -> Any:
"""Bind a TRTC room to the active call."""
def start_call_transcription(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
*,
language: str = "zh-CN",
) -> Any:
"""Start a transcript stored on one exact call record."""
def upsert_call_transcript_segments(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
segments: Sequence[Mapping[str, Any]],
) -> Any:
"""Idempotently persist completed transcript segments."""
def finish_call_transcription(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
expected_segment_count: int,
*,
status: str = "completed",
) -> Any:
"""Finalize and materialize the transcript text on a call record."""
def my_self(self) -> Session:
"""Compatibility alias for :meth:`get_session`."""
@@ -2171,32 +2201,157 @@ class RemoteDoctorRepository:
ticket.diagnosis_id = diagnosis_id
return ticket
def start_call(self, diagnosis_id: int, patient_id: int, *, call_type: int = 2) -> Any:
"""Create the server-side call record before ringing participants."""
return self.client.post(
"tcm.diagnosis/startCall",
{
"diagnosis_id": diagnosis_id,
"patient_id": patient_id,
"call_type": call_type,
},
)
def start_call(self, diagnosis_id: int, patient_id: int, *, call_type: int = 2) -> Any:
"""Create the server-side call record before ringing participants."""
payload = self.client.post(
"tcm.diagnosis/startCall",
{
"diagnosis_id": diagnosis_id,
"patient_id": patient_id,
"call_type": call_type,
},
)
if not isinstance(payload, Mapping):
raise ApiProtocolError(
"tcm.diagnosis/startCall returned no valid call_record_id object", data=payload
)
raw_id = next(
(
payload[key]
for key in ("call_record_id", "callRecordId", "id")
if key in payload
),
None,
)
try:
if isinstance(raw_id, bool):
raise ValueError
call_record_id = int(raw_id)
except (TypeError, ValueError) as exc:
raise ApiProtocolError(
"tcm.diagnosis/startCall returned no valid call_record_id",
data=dict(payload),
) from exc
if call_record_id <= 0:
raise ApiProtocolError(
"tcm.diagnosis/startCall returned no valid call_record_id",
data=dict(payload),
)
return {"call_record_id": call_record_id}
def end_call(self, diagnosis_id: int) -> Any:
"""End the active call/recording associated with a diagnosis."""
return self.client.post("tcm.diagnosis/endCall", {"diagnosis_id": diagnosis_id})
def bind_call_room(self, diagnosis_id: int, room_id: str) -> Any:
"""Bind the actual TRTC room to the active call record."""
def bind_call_room(self, diagnosis_id: int, room_id: str) -> Any:
"""Bind the actual TRTC room to the active call record."""
if not room_id.strip():
raise ValueError("room_id is required")
return self.client.post(
"tcm.diagnosis/bindCallRoom",
{"diagnosis_id": diagnosis_id, "room_id": room_id.strip()},
)
return self.client.post(
"tcm.diagnosis/bindCallRoom",
{"diagnosis_id": diagnosis_id, "room_id": room_id.strip()},
)
@staticmethod
def _transcription_identity(
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
) -> dict[str, Any]:
if diagnosis_id <= 0:
raise ValueError("diagnosis_id must be positive")
if isinstance(call_record_id, bool) or not str(call_record_id).strip():
raise ValueError("call_record_id is required")
clean_session = transcription_session_id.strip()
if not clean_session or len(clean_session) > 128:
raise ValueError("transcription_session_id must contain 1 to 128 characters")
return {
"diagnosis_id": diagnosis_id,
"call_record_id": call_record_id,
"transcription_session_id": clean_session,
}
def start_call_transcription(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
*,
language: str = "zh-CN",
) -> Any:
"""Create the server transcript session for one call record."""
body = self._transcription_identity(
diagnosis_id, call_record_id, transcription_session_id
)
body["language"] = language.strip()[:32] or "zh-CN"
return self.client.post("tcm.diagnosis/startCallTranscription", body)
def upsert_call_transcript_segments(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
segments: Sequence[Mapping[str, Any]],
) -> Any:
"""Upsert a bounded batch using each segment_id as the idempotency key."""
body = self._transcription_identity(
diagnosis_id, call_record_id, transcription_session_id
)
normalized: list[dict[str, Any]] = []
if len(segments) > 50:
raise ValueError("at most 50 transcript segments may be submitted at once")
for segment in segments:
segment_id = str(segment.get("segment_id") or "").strip()
text = str(segment.get("text") or "").strip()
if not segment_id or len(segment_id) > 160:
raise ValueError("transcript segment_id is invalid")
if not text or len(text) > 4_000:
raise ValueError("transcript text is invalid")
normalized.append(
{
"segment_id": segment_id,
"speaker_user_id": str(segment.get("speaker_user_id") or "")[:160],
"speaker_role": str(segment.get("speaker_role") or "unknown")[:20],
"timestamp": max(int(segment.get("timestamp") or 0), 0),
"text": text,
}
)
if not normalized:
raise ValueError("at least one transcript segment is required")
body["segments"] = normalized
return self.client.post("tcm.diagnosis/upsertCallTranscriptSegments", body)
def finish_call_transcription(
self,
diagnosis_id: int,
call_record_id: int | str,
transcription_session_id: str,
expected_segment_count: int,
*,
status: str = "completed",
) -> Any:
"""Finalize a transcript; repeated requests use the same session identity."""
body = self._transcription_identity(
diagnosis_id, call_record_id, transcription_session_id
)
clean_status = status.strip().lower()
if clean_status not in {"completed", "partial", "failed"}:
raise ValueError("transcription status is invalid")
if expected_segment_count < 0:
raise ValueError("expected_segment_count must not be negative")
body.update(
{
"expected_segment_count": expected_segment_count,
"status": clean_status,
}
)
return self.client.post("tcm.diagnosis/finishCallTranscription", body)
# Compatibility aliases keep UI naming independent from endpoint history.
def my_self(self) -> Session:
@@ -20,6 +20,7 @@ from PySide6.QtWidgets import (
QInputDialog,
QLabel,
QMessageBox,
QPlainTextEdit,
QPushButton,
QScrollArea,
QSizePolicy,
@@ -1411,7 +1412,7 @@ class DiagnosisDialog(QDialog):
layout.setContentsMargins(16, 14, 16, 18)
layout.setSpacing(12)
toolbar = QHBoxLayout()
hint = QLabel("每条通话记录可包含多个回放地址,并可追加指定记录的视频")
hint = QLabel("每条通话记录可包含多个回放地址;录音转写完成后可查看本次面诊对话文字")
hint.setObjectName("DiagnosisDialogGuidance")
hint.setWordWrap(True)
toolbar.addWidget(hint, 1)
@@ -1553,7 +1554,7 @@ class DiagnosisDialog(QDialog):
("房间号", 180),
("时长", 110),
("状态", 90),
("录制", 100),
("录制 / 文字", 120),
("操作", 130),
),
"assign": (
@@ -3365,11 +3366,71 @@ class DiagnosisDialog(QDialog):
cell.stop()
self._inline_recording_cells.clear()
@staticmethod
def _call_transcript_text(row: Any) -> str:
raw = first_value(
row,
"transcript_text",
"call_transcript",
"conversation_text",
"transcript",
default="",
)
if isinstance(raw, Mapping):
raw = first_value(raw, "text", "full_text", "content", default="")
text = str(raw or "").strip()
if text:
return text
segments = first_value(row, "transcript_segments", "segments", default=[]) or []
if isinstance(segments, Sequence) and not isinstance(
segments, (str, bytes, bytearray)
):
labels = {"doctor": "医生", "patient": "患者", "unknown": "未知说话人"}
lines: list[str] = []
for segment in segments:
content = str(first_value(segment, "text", "sourceText", default="") or "").strip()
if not content:
continue
role = str(
first_value(segment, "speaker_role", "speakerRole", default="unknown")
or "unknown"
)
lines.append(f"{labels.get(role, '未知说话人')}{content}")
return "\n".join(lines)
return ""
def _view_call_transcript(self, call_record_id: int, transcript: str) -> None:
dialog = QDialog(self)
dialog.setObjectName("DiagnosisCallTranscriptDialog")
dialog.setWindowTitle(f"面诊对话文字 · 通话记录 #{call_record_id}")
dialog.resize(680, 520)
layout = QVBoxLayout(dialog)
layout.setContentsMargins(20, 18, 20, 18)
layout.setSpacing(12)
title = QLabel("录音转写对话")
title.setObjectName("DiagnosisDailySectionTitle")
layout.addWidget(title)
hint = QLabel("以下内容由语音自动转写,仅作为面诊记录辅助,请由医生核对。")
hint.setObjectName("DiagnosisDialogGuidance")
hint.setWordWrap(True)
layout.addWidget(hint)
content = QPlainTextEdit(dialog)
content.setObjectName("DiagnosisCallTranscriptText")
content.setReadOnly(True)
content.setPlainText(transcript)
layout.addWidget(content, 1)
close_button = QPushButton("关闭")
close_button.setProperty("variant", "primary")
close_button.clicked.connect(dialog.accept)
layout.addWidget(close_button, 0, Qt.AlignmentFlag.AlignRight)
dialog.exec()
def _fill_video(self, rows: Sequence[Any]) -> None:
self._stop_inline_recordings()
matrix: list[tuple[Any, ...]] = []
row_urls: list[list[str]] = []
record_ids: list[int] = []
transcripts: list[str] = []
for row in rows:
raw_urls = first_value(row, "recording_urls_list", "recording_urls", default=[]) or []
if isinstance(raw_urls, str) or not isinstance(raw_urls, Sequence):
@@ -3382,6 +3443,8 @@ class DiagnosisDialog(QDialog):
normalized.append(url)
row_urls.append(normalized)
record_ids.append(_int(first_value(row, "id", "call_record_id"), 0))
transcript = self._call_transcript_text(row)
transcripts.append(transcript)
call_type = _raw_value(row, "call_type_text")
if call_type is _MISSING:
raw_call_type = _raw_value(row, "call_type")
@@ -3398,6 +3461,15 @@ class DiagnosisDialog(QDialog):
3: "未接听",
4: "已取消",
}.get(_int(raw_status, -1), "")
recording_status = first_value(
row, "recording_status_text", "record_status_text", "record_status"
)
transcript_status = first_value(
row,
"transcription_status_text",
"transcript_status_text",
default="文字已生成" if transcript else "未生成文字",
)
matrix.append(
(
"" if normalized else "暂无录制回放",
@@ -3407,9 +3479,7 @@ class DiagnosisDialog(QDialog):
first_value(row, "room_id", "room_no"),
first_value(row, "duration_text", "duration"),
status,
first_value(
row, "recording_status_text", "record_status_text", "record_status"
),
f"{display_text(recording_status)}\n{display_text(transcript_status)}",
"",
)
)
@@ -3434,6 +3504,20 @@ class DiagnosisDialog(QDialog):
if item is not None:
item.setText("")
call_record_id = record_ids[row_index]
actions: list[QPushButton] = []
transcript = transcripts[row_index]
if transcript and call_record_id > 0:
view_transcript = self._action_button(
"查看文字", f"查看通话记录 #{call_record_id} 的录音转写"
)
view_transcript.setObjectName("DiagnosisVideoTranscriptView")
view_transcript.setProperty("callRecordId", call_record_id)
view_transcript.clicked.connect(
lambda _checked=False, selected=call_record_id, text=transcript: self._view_call_transcript(
selected, text
)
)
actions.append(view_transcript)
if self._editable and self._can_video_upload and call_record_id > 0:
upload = self._action_button(
"追加回放", f"上传并绑定通话记录 #{call_record_id}"
@@ -3452,12 +3536,16 @@ class DiagnosisDialog(QDialog):
selected
)
)
upload_host = QWidget()
upload_host.setObjectName("DiagnosisVideoRowUploadCell")
upload_layout = QVBoxLayout(upload_host)
upload_layout.setContentsMargins(0, 0, 0, 0)
upload_layout.addWidget(upload, 0, Qt.AlignmentFlag.AlignCenter)
table.setCellWidget(row_index, 8, upload_host)
actions.append(upload)
if actions:
action_host = QWidget()
action_host.setObjectName("DiagnosisVideoRowUploadCell")
action_layout = QVBoxLayout(action_host)
action_layout.setContentsMargins(0, 0, 0, 0)
action_layout.setSpacing(2)
for action in actions:
action_layout.addWidget(action, 0, Qt.AlignmentFlag.AlignCenter)
table.setCellWidget(row_index, 8, action_host)
upload_item = table.item(row_index, 8)
if upload_item is not None:
upload_item.setText("")
@@ -130,6 +130,7 @@ def _ticket_mapping(ticket: Any) -> Mapping[str, Any]:
"patientUserId": "patient_user_id",
"diagnosisId": "diagnosis_id",
"patientId": "patient_id",
"callRecordId": "call_record_id",
}
adapted = {
json_name: getattr(ticket, attribute_name)
@@ -196,6 +197,7 @@ class VideoCallRequest:
target_user_id: str
diagnosis_id: Identifier
patient_id: Identifier | None = None
call_record_id: Identifier | None = None
backend_mode: BackendMode = BackendMode.EMBEDDED
def __post_init__(self) -> None:
@@ -218,6 +220,12 @@ class VideoCallRequest:
"patient_id",
_identifier(self.patient_id, "patientId"),
)
if self.call_record_id is not None:
object.__setattr__(
self,
"call_record_id",
_identifier(self.call_record_id, "callRecordId"),
)
object.__setattr__(self, "backend_mode", BackendMode.parse(self.backend_mode))
@classmethod
@@ -257,6 +265,7 @@ class VideoCallRequest:
return {
"diagnosis_id": self.diagnosis_id,
"patient_id": self.patient_id,
"call_record_id": self.call_record_id,
"backend_mode": self.backend_mode.value,
}
@@ -285,6 +294,13 @@ def normalize_backend_ticket(
_identifier,
required=False,
)
payload_call_record = _read_aliases(
payload,
("callRecordId", "call_record_id"),
"callRecordId",
_identifier,
required=False,
)
normalized_diagnosis = _merge_identifier(
payload_diagnosis,
@@ -323,6 +339,7 @@ def normalize_backend_ticket(
),
diagnosis_id=normalized_diagnosis,
patient_id=normalized_patient,
call_record_id=payload_call_record,
backend_mode=BackendMode.parse(backend_mode),
)
+237 -1
View File
@@ -74,6 +74,71 @@ def _call_repository_method(method: Callable[..., Any], payload: Mapping[str, An
return _resolve_result(result)
def _mapping_candidate(value: Any) -> Mapping[str, Any] | None:
if isinstance(value, Mapping):
return value
for attribute in ("raw", "data"):
candidate = getattr(value, attribute, None)
if isinstance(candidate, Mapping):
return candidate
return None
def _extract_call_record_id(result: Any) -> int | str | None:
"""Read a positive call-record identity from common backend envelopes."""
pending = [result]
visited: set[int] = set()
while pending:
candidate = pending.pop(0)
mapping = _mapping_candidate(candidate)
if mapping is None or id(mapping) in visited:
continue
visited.add(id(mapping))
for key in ("call_record_id", "callRecordId", "id"):
value = mapping.get(key)
if isinstance(value, bool) or value is None:
continue
if isinstance(value, int) and value > 0:
return value
if isinstance(value, str) and value.strip():
return value.strip()
for key in ("data", "result", "record", "call_record", "callRecord"):
nested = mapping.get(key)
if isinstance(nested, Mapping):
pending.append(nested)
return None
def _clean_transcript_segment(segment: Mapping[str, Any], session_id: str) -> dict[str, Any]:
segment_id = str(segment.get("segment_id", segment.get("segmentId", ""))).strip()
text = str(segment.get("text", segment.get("sourceText", ""))).strip()
speaker_user_id = str(
segment.get("speaker_user_id", segment.get("speakerUserId", ""))
).strip()
speaker_role = str(segment.get("speaker_role", segment.get("speakerRole", "unknown"))).strip()
if not segment_id or len(segment_id) > 160:
raise ValueError("transcript segment_id must contain 1 to 160 characters")
if not text or len(text) > 4_000:
raise ValueError("transcript text must contain 1 to 4000 characters")
if len(speaker_user_id) > 160:
raise ValueError("transcript speaker_user_id is too long")
if speaker_role not in {"doctor", "patient", "unknown"}:
speaker_role = "unknown"
try:
timestamp = int(segment.get("timestamp") or 0)
except (TypeError, ValueError):
timestamp = 0
return {
"segment_id": segment_id,
"transcription_session_id": session_id,
"speaker_user_id": speaker_user_id,
"speaker_role": speaker_role,
"timestamp": max(timestamp, 0),
"text": text,
}
@dataclass(slots=True)
class _WorkItem:
operation: str
@@ -169,11 +234,17 @@ class OrderedCallLifecycle:
self.logger = logger
self.started = False
self.ended = False
self.call_record_id: int | str | None = request.call_record_id
self.bound_room_id: str | None = None
self._claimed_room_id: str | None = None
self._start_future: Future[bool] | None = None
self._bind_future: Future[bool] | None = None
self._end_future: Future[bool] | None = None
self._transcription_start_future: Future[bool] | None = None
self._transcription_finish_future: Future[bool] | None = None
self._transcription_session_id: str | None = None
self._transcription_active = False
self._segment_futures: dict[str, Future[bool]] = {}
self._lock = threading.RLock()
self._worker = _OrderedDaemonWorker(logger, request.safe_log_context())
@@ -181,6 +252,10 @@ class OrderedCallLifecycle:
def worker_is_daemon(self) -> bool:
return self._worker.is_daemon
@property
def transcription_session_id(self) -> str | None:
return self._transcription_session_id
def start(self) -> Future[bool]:
with self._lock:
if self._start_future is not None:
@@ -196,9 +271,13 @@ class OrderedCallLifecycle:
payload["patient_id"] = self.request.patient_id
def operation() -> bool:
_call_repository_method(method, payload)
result = _call_repository_method(method, payload)
record_id = _extract_call_record_id(result)
if record_id is None:
raise ValueError("startCall response did not include the current call_record_id")
with self._lock:
self.started = True
self.call_record_id = record_id
self.logger.info(
"video call record started",
extra={"video_call": self.request.safe_log_context()},
@@ -309,10 +388,167 @@ class OrderedCallLifecycle:
return self._worker.submit("screenshot", operation)
def start_transcription(self, session_id: str, *, language: str = "zh-CN") -> Future[bool]:
"""Start persisted realtime transcription for this exact call record."""
clean_session = str(session_id or "").strip()
clean_language = str(language or "zh-CN").strip()[:32] or "zh-CN"
if not clean_session or len(clean_session) > 128:
raise ValueError("transcription session id must contain 1 to 128 characters")
with self._lock:
if self._end_future is not None:
raise RuntimeError("video call has already ended")
if self._transcription_start_future is not None:
if self._transcription_session_id != clean_session:
raise RuntimeError("another transcription session already exists")
return self._transcription_start_future
if self._start_future is None:
self.start()
method = getattr(self.repository, "start_call_transcription", None)
if not callable(method):
raise ValueError("video repository does not implement call transcription storage")
self._transcription_session_id = clean_session
def operation() -> bool:
with self._lock:
started = self.started
record_id = self.call_record_id
if not started:
return False
if record_id is None:
raise ValueError("server did not return the current call_record_id")
_call_repository_method(
method,
{
"diagnosis_id": self.request.diagnosis_id,
"call_record_id": record_id,
"transcription_session_id": clean_session,
"language": clean_language,
},
)
with self._lock:
self._transcription_active = True
self.logger.info(
"video call transcription started",
extra={"video_call": self.request.safe_log_context()},
)
return True
self._transcription_start_future = self._worker.submit(
"transcription-start", operation
)
return self._transcription_start_future
def save_transcript_segment(self, segment: Mapping[str, Any]) -> Future[bool]:
"""Upsert one completed, bounded transcript segment without logging its text."""
with self._lock:
session_id = self._transcription_session_id
if not session_id or self._transcription_start_future is None:
raise RuntimeError("call transcription has not started")
if self._transcription_finish_future is not None:
raise RuntimeError("call transcription has already stopped")
cleaned = _clean_transcript_segment(segment, session_id)
segment_id = cleaned["segment_id"]
existing = self._segment_futures.get(segment_id)
if existing is not None:
return existing
method = getattr(self.repository, "upsert_call_transcript_segments", None)
if not callable(method):
raise ValueError("video repository does not implement transcript segment storage")
def operation() -> bool:
with self._lock:
active = self._transcription_active
record_id = self.call_record_id
if not active:
return False
if record_id is None:
raise ValueError("server did not return the current call_record_id")
_call_repository_method(
method,
{
"diagnosis_id": self.request.diagnosis_id,
"call_record_id": record_id,
"transcription_session_id": session_id,
"segments": [cleaned],
},
)
return True
future = self._worker.submit("transcription-segment", operation)
self._segment_futures[segment_id] = future
def release_failed(completed: Future[bool]) -> None:
if completed.cancelled() or completed.exception() is not None:
with self._lock:
if self._segment_futures.get(segment_id) is completed:
self._segment_futures.pop(segment_id, None)
future.add_done_callback(release_failed)
return future
def finish_transcription(self, *, status: str = "completed") -> Future[bool]:
"""Flush and finalize the current transcript exactly once."""
clean_status = str(status or "completed").strip().lower()
if clean_status not in {"completed", "partial", "failed"}:
raise ValueError("transcription status is invalid")
with self._lock:
if self._transcription_finish_future is not None:
return self._transcription_finish_future
if self._transcription_start_future is None or not self._transcription_session_id:
return _settled_future(False)
method = getattr(self.repository, "finish_call_transcription", None)
if not callable(method):
raise ValueError("video repository does not implement transcription finalization")
session_id = self._transcription_session_id
def operation() -> bool:
with self._lock:
active = self._transcription_active
record_id = self.call_record_id
expected_count = len(self._segment_futures)
if not active:
return False
if record_id is None:
raise ValueError("server did not return the current call_record_id")
_call_repository_method(
method,
{
"diagnosis_id": self.request.diagnosis_id,
"call_record_id": record_id,
"transcription_session_id": session_id,
"expected_segment_count": expected_count,
"status": clean_status,
},
)
with self._lock:
self._transcription_active = False
self.logger.info(
"video call transcription finalized",
extra={
"video_call": self.request.safe_log_context(),
"transcription_status": clean_status,
"segment_count": expected_count,
},
)
return True
self._transcription_finish_future = self._worker.submit(
"transcription-finish", operation
)
return self._transcription_finish_future
def end(self, reason: str) -> Future[bool]:
with self._lock:
if self._end_future is not None:
return self._end_future
if (
self._transcription_start_future is not None
and self._transcription_finish_future is None
):
self.finish_transcription(status="partial")
method = getattr(self.repository, "end_call", None)
def operation() -> bool:
+120 -2
View File
@@ -210,6 +210,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
call_error = Signal(str) # type: ignore[misc]
_start_completed = Signal(bool) # type: ignore[misc]
_screenshot_completed = Signal(bool, str) # type: ignore[misc]
_transcription_completed = Signal(str, str, str, bool, str) # type: ignore[misc]
def __init__(
self,
@@ -284,6 +285,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
self._start_completed.connect(self._on_lifecycle_started)
self._screenshot_completed.connect(self._on_screenshot_completed)
self._transcription_completed.connect(self._on_transcription_completed)
self.web_view.loadFinished.connect(self._on_load_finished)
self.web_view.setUrl(QUrl(self.location.url))
@@ -426,6 +428,21 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
str(message.get("message") or "截屏图片无效。")[:200],
)
return
if event == "transcription-start-request":
self._start_transcription(
str(message.get("sessionId") or ""),
str(message.get("language") or "zh-CN"),
)
return
if event == "transcription-segment":
self._save_transcript_segment(message)
return
if event == "transcription-stop":
self._finish_transcription(
str(message.get("sessionId") or ""),
str(message.get("status") or "completed"),
)
return
room_id = message.get("roomId", message.get("room_id"))
if room_id not in (None, ""):
self.lifecycle.bind_room(room_id)
@@ -434,8 +451,6 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
if event == "status":
status = str(message.get("status", "unknown"))[:80]
self.status_changed.emit(status)
if status == "idle" and not self.open_im:
self._close_from_companion("remote-idle")
elif event == "hangup":
status = str(message.get("status", "ended"))[:80]
self.call_ended.emit(status)
@@ -509,6 +524,109 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
f"window.doctorConsultation?.screenshotResult?.({state}, {payload});"
)
def _notify_transcription_completed(
self,
operation: str,
session_id: str,
segment_id: str,
future: Future[bool],
) -> None:
try:
succeeded = bool(future.result())
except Exception as error:
succeeded = False
message = str(error)[:200] or "录音文字保存失败。"
else:
message = {
"start": "录音文字存储已准备。",
"segment": "",
"stop": "本次面诊对话文字已保存。",
}.get(operation, "")
with suppress(RuntimeError):
self._transcription_completed.emit(
operation, session_id, segment_id, succeeded, message
)
def _on_transcription_completed(
self,
operation: str,
session_id: str,
segment_id: str,
succeeded: bool,
message: str,
) -> None:
payload = json.dumps(str(message)[:200], ensure_ascii=True)
state = "true" if succeeded else "false"
self._page.runJavaScript(
"window.doctorConsultation?.transcriptionResult?.("
f"{json.dumps(operation)}, {json.dumps(session_id)}, "
f"{json.dumps(segment_id)}, {state}, {payload});"
)
def _start_transcription(self, session_id: str, language: str) -> None:
try:
future = self.lifecycle.start_transcription(session_id, language=language)
except Exception as error:
self._on_transcription_completed(
"start", session_id, "", False, str(error)[:200]
)
return
future.add_done_callback(
lambda completed: self._notify_transcription_completed(
"start", session_id, "", completed
)
)
def _save_transcript_segment(self, message: Mapping[str, Any]) -> None:
session_id = str(message.get("sessionId") or "").strip()
segment = message.get("segment")
segment_id = (
str(segment.get("segment_id") or segment.get("segmentId") or "").strip()
if isinstance(segment, Mapping)
else ""
)
if not isinstance(segment, Mapping):
self._on_transcription_completed(
"segment", session_id, segment_id, False, "录音文字片段无效。"
)
return
if session_id != str(self.lifecycle.transcription_session_id or ""):
self._on_transcription_completed(
"segment", session_id, segment_id, False, "录音会话标识不匹配。"
)
return
try:
future = self.lifecycle.save_transcript_segment(segment)
except Exception as error:
self._on_transcription_completed(
"segment", session_id, segment_id, False, str(error)[:200]
)
return
future.add_done_callback(
lambda completed: self._notify_transcription_completed(
"segment", session_id, segment_id, completed
)
)
def _finish_transcription(self, session_id: str, status: str) -> None:
if session_id.strip() != str(self.lifecycle.transcription_session_id or ""):
self._on_transcription_completed(
"stop", session_id, "", False, "录音会话标识不匹配。"
)
return
try:
future = self.lifecycle.finish_transcription(status=status)
except Exception as error:
self._on_transcription_completed(
"stop", session_id, "", False, str(error)[:200]
)
return
future.add_done_callback(
lambda completed: self._notify_transcription_completed(
"stop", session_id, "", completed
)
)
def _close_from_companion(self, reason: str) -> None:
self._companion_ended = True
self._close_reason = reason
+65
View File
@@ -232,6 +232,69 @@ def test_demo_call_lifecycle_mutates_record(repository: DemoDoctorRepository) ->
assert ended["room_id"] == "room-501"
def test_demo_transcript_upsert_and_finish_round_trip_in_call_records(
repository: DemoDoctorRepository,
) -> None:
"""Demo replay reads expose one finalized segment for a repeated segment ID."""
started = repository.start_call(501, 301)
call_record_id = started["id"]
repository.start_call_transcription(501, call_record_id, "session-1")
repository.upsert_call_transcript_segments(
501,
call_record_id,
"session-1",
[
{
"segment_id": "seg-1",
"speaker_user_id": "patient_301",
"speaker_role": "patient",
"timestamp": 1200,
"text": "draft words",
}
],
)
repository.upsert_call_transcript_segments(
501,
call_record_id,
"session-1",
[
{
"segment_id": "seg-1",
"speaker_user_id": "patient_301",
"speaker_role": "patient",
"timestamp": 1200,
"text": "final words",
}
],
)
repository.finish_call_transcription(
501,
call_record_id,
"session-1",
expected_segment_count=1,
status="completed",
)
repository.end_call(501)
record = next(
row for row in repository.list_call_records(501) if row["id"] == call_record_id
)
assert record["status"] == 2
assert record["transcription_status"] == "completed"
assert record["transcription_segment_count"] == 1
assert record["transcript_segments"] == [
{
"segment_id": "seg-1",
"speaker_role": "patient",
"speaker_user_id": "patient_301",
"timestamp": 1200,
"text": "final words",
}
]
assert "final words" in record["transcript_text"]
def test_tolerant_page_parsing_accepts_aliases_and_bad_rows() -> None:
"""List parsing handles nullable fields, aliases and non-object rows safely."""
@@ -315,6 +378,8 @@ class _StubApiClient:
"userSig": "short-lived",
"patientUserId": "patient_2",
}
if endpoint == "tcm.diagnosis/startCall":
return {"call_record_id": 901}
return {"ok": True}
+103
View File
@@ -7,6 +7,7 @@ from typing import Any
import pytest
from doctor_workstation.core.errors import ApiProtocolError
from doctor_workstation.core.models import Appointment, Consultation, PageResult, Prescription
from doctor_workstation.services.mock_repository import DemoDoctorRepository
from doctor_workstation.services.repository import (
@@ -76,6 +77,8 @@ class RecordingClient:
self.post_calls.append((endpoint, body))
if endpoint in {"tcm.prescription/add", "tcm.prescriptionOrder/create"}:
return {"id": 88}
if endpoint == "tcm.diagnosis/startCall":
return {"call_record_id": 901}
return {"ok": True}
@@ -253,6 +256,106 @@ def test_remote_new_contracts_use_exact_admin_endpoints_and_dtos() -> None:
} <= get_endpoints
def test_remote_transcription_endpoints_use_exact_normalized_dtos() -> None:
"""Realtime transcript persistence stays within the three audited POST DTOs."""
client = RecordingClient()
repository = RemoteDoctorRepository(client) # type: ignore[arg-type]
repository.start_call_transcription(501, 901, " session-1 ", language=" zh-CN ")
repository.upsert_call_transcript_segments(
501,
901,
"session-1",
[
{
"segment_id": "seg-1",
"speaker_user_id": "patient_301",
"speaker_role": "patient",
"timestamp": "1200",
"text": " patient words ",
}
],
)
repository.finish_call_transcription(
501,
901,
"session-1",
expected_segment_count=1,
status="completed",
)
assert client.post_calls == [
(
"tcm.diagnosis/startCallTranscription",
{
"diagnosis_id": 501,
"call_record_id": 901,
"transcription_session_id": "session-1",
"language": "zh-CN",
},
),
(
"tcm.diagnosis/upsertCallTranscriptSegments",
{
"diagnosis_id": 501,
"call_record_id": 901,
"transcription_session_id": "session-1",
"segments": [
{
"segment_id": "seg-1",
"speaker_user_id": "patient_301",
"speaker_role": "patient",
"timestamp": 1200,
"text": "patient words",
}
],
},
),
(
"tcm.diagnosis/finishCallTranscription",
{
"diagnosis_id": 501,
"call_record_id": 901,
"transcription_session_id": "session-1",
"expected_segment_count": 1,
"status": "completed",
},
),
]
def test_remote_start_call_requires_and_normalizes_current_record_id() -> None:
client = RecordingClient()
repository = RemoteDoctorRepository(client) # type: ignore[arg-type]
assert repository.start_call(501, 301) == {"call_record_id": 901}
assert client.post_calls == [
(
"tcm.diagnosis/startCall",
{"diagnosis_id": 501, "patient_id": 301, "call_type": 2},
)
]
@pytest.mark.parametrize(
"response",
[None, {}, {"ok": True}, {"call_record_id": 0}, {"callRecordId": -1}, {"id": True}],
)
def test_remote_start_call_rejects_missing_or_invalid_record_id(response: Any) -> None:
class StartCallClient(RecordingClient):
def post(self, endpoint: str, payload: dict[str, Any] | None = None) -> Any:
if endpoint == "tcm.diagnosis/startCall":
self.post_calls.append((endpoint, dict(payload or {})))
return response
return super().post(endpoint, payload)
repository = RemoteDoctorRepository(StartCallClient()) # type: ignore[arg-type]
with pytest.raises(ApiProtocolError, match="call_record"):
repository.start_call(501, 301)
@pytest.mark.parametrize(
"unsafe_reference",
[r"C:\records\tongue.jpg", r"\\server\share\report.pdf", "file:///tmp/a.jpg"],
+230 -2
View File
@@ -188,10 +188,13 @@ def test_call_lifecycle_is_fifo_daemon_and_never_blocks_caller() -> None:
release_start = threading.Event()
class Repository:
def start_call(self, diagnosis_id: int, patient_id: int, *, call_type: int) -> None:
def start_call(
self, diagnosis_id: int, patient_id: int, *, call_type: int
) -> dict[str, int]:
start_entered.set()
assert release_start.wait(2)
events.append(("start", diagnosis_id, patient_id, call_type))
return {"call_record_id": 900}
def bind_call_room(self, diagnosis_id: int, room_id: str) -> None:
events.append(("bind", diagnosis_id, room_id))
@@ -237,6 +240,208 @@ def test_call_lifecycle_is_fifo_daemon_and_never_blocks_caller() -> None:
]
def test_transcription_lifecycle_uses_start_record_id_and_remains_fifo() -> None:
events: list[tuple[object, ...]] = []
class Repository:
def start_call(
self, diagnosis_id: int, patient_id: int, *, call_type: int
) -> dict[str, object]:
events.append(("start", diagnosis_id, patient_id, call_type))
return {"data": {"callRecordId": 901}}
def start_call_transcription(
self,
diagnosis_id: int,
call_record_id: int,
transcription_session_id: str,
*,
language: str,
) -> None:
events.append(
(
"transcription-start",
diagnosis_id,
call_record_id,
transcription_session_id,
language,
)
)
def upsert_call_transcript_segments(
self,
diagnosis_id: int,
call_record_id: int,
transcription_session_id: str,
segments: list[dict[str, object]],
) -> None:
events.append(
(
"segment",
diagnosis_id,
call_record_id,
transcription_session_id,
segments,
)
)
def finish_call_transcription(
self,
diagnosis_id: int,
call_record_id: int,
transcription_session_id: str,
expected_segment_count: int,
*,
status: str,
) -> None:
events.append(
(
"finish",
diagnosis_id,
call_record_id,
transcription_session_id,
expected_segment_count,
status,
)
)
def end_call(self, diagnosis_id: int) -> None:
events.append(("end", diagnosis_id))
request = VideoCallRequest(
sdk_app_id=1400123456,
user_id="doctor_42",
user_sig="short-lived-ticket",
target_user_id="patient_8",
diagnosis_id=123,
patient_id=8,
)
lifecycle = OrderedCallLifecycle(request, Repository(), logging.getLogger(__name__))
start = lifecycle.start()
transcription_start = lifecycle.start_transcription("session-1")
segment = lifecycle.save_transcript_segment(
{
"segment_id": "seg-1",
"speaker_user_id": "patient_8",
"speaker_role": "patient",
"timestamp": 1200,
"text": "patient words",
}
)
duplicate = lifecycle.save_transcript_segment(
{"segment_id": "seg-1", "text": "must not produce another write"}
)
finish = lifecycle.finish_transcription(status="completed")
end = lifecycle.end("test")
assert duplicate is segment
assert start.result(timeout=2) is True
assert transcription_start.result(timeout=2) is True
assert segment.result(timeout=2) is True
assert finish.result(timeout=2) is True
assert end.result(timeout=2) is True
assert lifecycle.wait(1) is True
assert lifecycle.call_record_id == 901
assert events == [
("start", 123, 8, 2),
("transcription-start", 123, 901, "session-1", "zh-CN"),
(
"segment",
123,
901,
"session-1",
[
{
"segment_id": "seg-1",
"transcription_session_id": "session-1",
"speaker_user_id": "patient_8",
"speaker_role": "patient",
"timestamp": 1200,
"text": "patient words",
}
],
),
("finish", 123, 901, "session-1", 1, "completed"),
("end", 123),
]
def test_end_auto_finishes_active_transcription_as_partial() -> None:
events: list[tuple[object, ...]] = []
class Repository:
def start_call(
self, diagnosis_id: int, patient_id: int, *, call_type: int
) -> dict[str, int]:
events.append(("start", diagnosis_id, patient_id, call_type))
return {"call_record_id": 902}
def start_call_transcription(
self,
diagnosis_id: int,
call_record_id: int,
transcription_session_id: str,
*,
language: str,
) -> None:
events.append(
(
"transcription-start",
diagnosis_id,
call_record_id,
transcription_session_id,
language,
)
)
def finish_call_transcription(
self,
diagnosis_id: int,
call_record_id: int,
transcription_session_id: str,
expected_segment_count: int,
*,
status: str,
) -> None:
events.append(
(
"finish",
diagnosis_id,
call_record_id,
transcription_session_id,
expected_segment_count,
status,
)
)
def end_call(self, diagnosis_id: int) -> None:
events.append(("end", diagnosis_id))
request = VideoCallRequest(
sdk_app_id=1400123456,
user_id="doctor_42",
user_sig="short-lived-ticket",
target_user_id="patient_8",
diagnosis_id=123,
patient_id=8,
)
lifecycle = OrderedCallLifecycle(request, Repository(), logging.getLogger(__name__))
lifecycle.start()
lifecycle.start_transcription("session-auto-partial")
ended = lifecycle.end("window-closed")
assert ended.result(timeout=2) is True
assert lifecycle.wait(1) is True
assert events == [
("start", 123, 8, 2),
("transcription-start", 123, 902, "session-auto-partial", "zh-CN"),
("finish", 123, 902, "session-auto-partial", 0, "partial"),
("end", 123),
]
def test_failed_start_prevents_bind_and_end_writes() -> None:
events: list[str] = []
@@ -278,8 +483,9 @@ def test_video_screenshot_is_uploaded_and_appended_to_patient_tongue_images() ->
events: list[tuple[object, ...]] = []
class Repository:
def start_call(self, diagnosis_id: int, *, call_type: int) -> None:
def start_call(self, diagnosis_id: int, *, call_type: int) -> dict[str, int]:
events.append(("start", diagnosis_id, call_type))
return {"call_record_id": 903}
def upload_material_bytes(
self,
@@ -325,6 +531,28 @@ def test_video_screenshot_is_uploaded_and_appended_to_patient_tongue_images() ->
]
def test_start_rejects_missing_record_id_without_using_ticket_fallback() -> None:
class Repository:
def start_call(self, diagnosis_id: int, *, call_type: int) -> dict[str, object]:
del diagnosis_id, call_type
return {}
request = VideoCallRequest(
sdk_app_id=1400123456,
user_id="doctor_42",
user_sig="short-lived-ticket",
target_user_id="patient_8",
diagnosis_id=123,
call_record_id=77,
)
lifecycle = OrderedCallLifecycle(request, Repository(), logging.getLogger(__name__))
with pytest.raises(ValueError, match="startCall response did not include"):
lifecycle.start().result(timeout=2)
assert lifecycle.started is False
def test_https_document_policy_is_exact_and_origin_scoped() -> None:
policy = TrustedDocumentPolicy.from_url(
"https://RTC.Example.com/doctor-call/index.html?tenant=a#boot",
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+2 -2
View File
@@ -5,8 +5,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="color-scheme" content="light" />
<title>视频面诊</title>
<script type="module" crossorigin src="./assets/index-DwSVWep6.js"></script>
<link rel="stylesheet" crossorigin href="./assets/index-qOBmgxQV.css">
<script type="module" crossorigin src="./assets/index-R5GzqA8s.js"></script>
<link rel="stylesheet" crossorigin href="./assets/index-CED5X2W4.css">
</head>
<body>
<div id="app"></div>
+23
View File
@@ -23,6 +23,7 @@ const props = defineProps<{
chatBusy: Readonly<Ref<boolean>>
notice: Readonly<Ref<string>>
hasMoreMessages: Readonly<Ref<boolean>>
transcriptionState: Readonly<Ref<string>>
onSendText: (text: string) => Promise<void>
onSendAttachment: (file: File) => Promise<void>
onLoadMore: () => Promise<void>
@@ -42,6 +43,15 @@ const isChat = computed(() => props.mode.value === 'chat')
const isCalling = computed(() => ['starting', 'dialing', 'connected'].includes(props.phase.value))
const videoVisible = computed(() => !isChat.value || isCalling.value)
const canCapture = computed(() => props.phase.value === 'connected')
const transcriptionActive = computed(() => props.transcriptionState.value === 'recording')
const transcriptionFailed = computed(() => props.transcriptionState.value === 'error')
const transcriptionStatusText = computed(() => {
if (props.transcriptionState.value === 'starting') return '自动录音启动中…'
if (props.transcriptionState.value === 'recording') return '自动录音并转文字中'
if (props.transcriptionState.value === 'stopping') return '正在保存录音文字…'
if (props.transcriptionState.value === 'error') return '自动录音转文字失败'
return '自动录音已结束'
})
watch(
() => props.messages.value.length,
@@ -242,6 +252,19 @@ async function captureScreenshot(): Promise<void> {
</div>
<div v-if="isCalling" class="video-actions">
<div
v-if="canCapture"
class="recording-status"
:class="{
'recording-status--active': transcriptionActive,
'recording-status--error': transcriptionFailed,
}"
role="status"
aria-live="polite"
>
<span class="recording-indicator" aria-hidden="true" />
{{ transcriptionStatusText }}
</div>
<button
class="capture-button"
type="button"
+7
View File
@@ -30,6 +30,13 @@ interface DoctorConsultationApi {
hangup(): Promise<void>
hostCallReady(ok: boolean, message?: string): void
screenshotResult(ok: boolean, message: string): void
transcriptionResult(
operation: 'start' | 'segment' | 'stop',
sessionId: string,
segmentId: string,
ok: boolean,
message: string,
): void
}
interface QtVideoBridge {
+493 -17
View File
@@ -15,6 +15,7 @@ import './style.css'
type CallPhase = 'ready' | 'starting' | 'dialing' | 'connected' | 'ended' | 'error'
type CompanionMode = 'chat' | 'video'
type ChatMessageType = 'text' | 'image' | 'file' | 'audio' | 'video' | 'system'
type TranscriptionState = 'idle' | 'starting' | 'recording' | 'stopping' | 'error'
interface NormalizedCallConfig {
SDKAppID: number
@@ -38,11 +39,74 @@ interface UiChatMessage {
interface BridgeMessage {
source: 'doctor-call'
event: 'ready' | 'call-start-request' | 'status' | 'room' | 'hangup' | 'error'
event:
| 'ready'
| 'call-start-request'
| 'status'
| 'room'
| 'hangup'
| 'error'
| 'transcription-start-request'
| 'transcription-segment'
| 'transcription-stop'
diagnosisId?: number | string
status?: string
roomId?: string
message?: string
sessionId?: string
language?: string
segment?: {
segment_id: string
speaker_user_id: string
speaker_role: 'doctor' | 'patient' | 'unknown'
timestamp: number
text: string
}
}
interface RealtimeTranscriberMessage {
segmentId: string
speakerUserId: string
sourceText: string
timestamp: number
isCompleted: boolean
}
interface RealtimeTranscriberListener {
onReceiveTranscriberMessage: (
roomId: string | number,
message: RealtimeTranscriberMessage,
) => void
onRealtimeTranscriberStarted: (
roomId: string | number,
robotId: string,
sourceLanguage: string,
) => void
onRealtimeTranscriberStopped: (roomId: string | number, robotId: string) => void
onRealtimeTranscriberError: (
roomId: string | number,
robotId: string,
error: number,
errorMessage: string,
) => void
}
interface RealtimeTranscriberManager {
addListener(listener: RealtimeTranscriberListener): void
removeListener(listener: RealtimeTranscriberListener): void
startRealtimeTranscriber(config: { sourceLanguage: string }): Promise<string>
stopRealtimeTranscriber(robotId: string): Promise<void>
}
interface PendingTranscriptionReply {
sessionId: string
resolve: (value: boolean) => void
promise: Promise<boolean>
}
interface PendingSegment {
message: BridgeMessage
attempts: number
}
const phase = ref<CallPhase>('ready')
@@ -54,6 +118,7 @@ const chatReady = ref(false)
const chatBusy = ref(false)
const notice = ref('')
const hasMoreMessages = ref(false)
const transcriptionState = ref<TranscriptionState>('idle')
let activeConfig: NormalizedCallConfig | null = null
let chat: any = null
@@ -65,6 +130,23 @@ let endNotified = true
let starting = false
let emittedRoomId = ''
let resolveHostCallReady: ((value: boolean) => void) | null = null
const pendingTranscriptionStarts = new Map<string, PendingTranscriptionReply>()
const pendingTranscriptionStops = new Map<string, PendingTranscriptionReply>()
let transcriptionSessionId = ''
let transcriptionGeneration = 0
let transcriberRunning = false
let transcriberManager: RealtimeTranscriberManager | null = null
let transcriberListener: RealtimeTranscriberListener | null = null
let transcriberRobotId = ''
let transcriptionStartPromise: Promise<void> | null = null
let transcriptionStopPromise: Promise<void> | null = null
let hangupNotification: Promise<void> | null = null
let callCycleGeneration = 0
let autoTranscriptionAttemptedGeneration = -1
let lastTranscriberMessageAt = 0
let transcriberStoppedAt = 0
const acknowledgedSegmentIds = new Set<string>()
const pendingSegments = new Map<string, PendingSegment>()
function initializeQtWebChannel(): void {
const transport = window.qt?.webChannelTransport
@@ -457,17 +539,372 @@ async function sendAttachment(file: File): Promise<void> {
}
}
function notifyHangup(status = 'ended'): void {
if (endNotified) return
function newTranscriptionSessionId(): string {
const random = window.crypto?.randomUUID?.()
return random ? `call-${random}` : `call-${Date.now()}-${Math.random().toString(16).slice(2)}`
}
function requestTranscriptionStart(sessionId: string): Promise<boolean> {
if (!activeConfig || !window.qtVideoBridge?.notify) return Promise.resolve(false)
const existing = pendingTranscriptionStarts.get(sessionId)
if (existing) return existing.promise
const pending = {} as PendingTranscriptionReply
pending.sessionId = sessionId
pending.promise = new Promise<boolean>((resolve) => {
pending.resolve = resolve
})
pendingTranscriptionStarts.set(sessionId, pending)
emit({
source: 'doctor-call',
event: 'transcription-start-request',
diagnosisId: activeConfig?.diagnosisId,
sessionId,
language: 'zh',
})
window.setTimeout(() => {
if (pendingTranscriptionStarts.get(sessionId) !== pending) return
pendingTranscriptionStarts.delete(sessionId)
pending.resolve(false)
}, 15000)
return pending.promise
}
function requestTranscriptionStop(
sessionId: string,
status: 'completed' | 'partial' | 'failed',
): Promise<boolean> {
if (!activeConfig || !sessionId || !window.qtVideoBridge?.notify) {
return Promise.resolve(false)
}
const existing = pendingTranscriptionStops.get(sessionId)
if (existing) return existing.promise
const pending = {} as PendingTranscriptionReply
pending.sessionId = sessionId
pending.promise = new Promise<boolean>((resolve) => {
pending.resolve = resolve
})
pendingTranscriptionStops.set(sessionId, pending)
emit({
source: 'doctor-call',
event: 'transcription-stop',
diagnosisId: activeConfig?.diagnosisId,
sessionId,
status,
})
window.setTimeout(() => {
if (pendingTranscriptionStops.get(sessionId) !== pending) return
pendingTranscriptionStops.delete(sessionId)
pending.resolve(false)
}, 15000)
return pending.promise
}
function getTranscriberManager(): RealtimeTranscriberManager {
const engine = TUICallKitAPI.getTUICallEngineInstance?.()
const cloud = engine?.getTRTCCloudInstance?.()
const manager = cloud?.getAITranscriberManager?.() as Partial<RealtimeTranscriberManager> | null
if (
!manager
|| typeof manager.addListener !== 'function'
|| typeof manager.removeListener !== 'function'
|| typeof manager.startRealtimeTranscriber !== 'function'
|| typeof manager.stopRealtimeTranscriber !== 'function'
) {
throw new Error('当前视频服务未开通实时语音转写')
}
return manager as RealtimeTranscriberManager
}
function handleTranscriberMessage(
_roomId: string | number,
message: RealtimeTranscriberMessage,
): void {
if (
!activeConfig
|| !['recording', 'stopping'].includes(transcriptionState.value)
|| !transcriptionSessionId
|| message.isCompleted !== true
) return
lastTranscriberMessageAt = Date.now()
const segmentId = String(message.segmentId ?? '').trim()
const text = String(message.sourceText ?? '').trim()
if (
!segmentId
|| acknowledgedSegmentIds.has(segmentId)
|| pendingSegments.has(segmentId)
|| !text
) return
const speakerUserId = String(message.speakerUserId ?? '').trim()
const bridgeMessage: BridgeMessage = {
source: 'doctor-call',
event: 'transcription-segment',
diagnosisId: activeConfig.diagnosisId,
sessionId: transcriptionSessionId,
segment: {
segment_id: segmentId.slice(0, 160),
speaker_user_id: speakerUserId.slice(0, 160),
speaker_role: speakerUserId === activeConfig.userID
? 'doctor'
: speakerUserId === activeConfig.targetUserId
? 'patient'
: 'unknown',
timestamp: Math.max(0, Math.trunc(Number(message.timestamp) || 0)),
text: text.slice(0, 4000),
},
}
pendingSegments.set(segmentId, { message: bridgeMessage, attempts: 1 })
emit(bridgeMessage)
}
function subscribeTranscriber(): {
manager: RealtimeTranscriberManager
listener: RealtimeTranscriberListener
} {
if (transcriberManager && transcriberListener) {
return { manager: transcriberManager, listener: transcriberListener }
}
const manager = getTranscriberManager()
const listener: RealtimeTranscriberListener = {
onReceiveTranscriberMessage: handleTranscriberMessage,
onRealtimeTranscriberStarted: () => undefined,
onRealtimeTranscriberStopped: (_roomId, robotId) => {
if (robotId !== transcriberRobotId) return
transcriberRunning = false
transcriberStoppedAt = Date.now()
if (transcriptionState.value === 'recording') {
void stopTranscription('partial', true)
}
},
onRealtimeTranscriberError: (_roomId, robotId, _error, errorMessage) => {
if (robotId !== transcriberRobotId || transcriptionState.value === 'stopping') return
notice.value = safeErrorMessage(new Error(errorMessage), '实时语音转写发生错误')
void stopTranscription('partial', true)
},
}
manager.addListener(listener)
transcriberManager = manager
transcriberListener = listener
return { manager, listener }
}
function unsubscribeTranscriber(
manager = transcriberManager,
listener = transcriberListener,
): void {
if (manager && listener) {
try {
manager.removeListener(listener)
} catch {
// A destroyed call engine has already released the listener.
}
}
if (transcriberManager === manager && transcriberListener === listener) {
transcriberManager = null
transcriberListener = null
}
}
function transcriptionTokenIsCurrent(token: number, sessionId: string): boolean {
return (
token === transcriptionGeneration
&& sessionId === transcriptionSessionId
&& transcriptionState.value === 'starting'
&& phase.value === 'connected'
&& !endNotified
)
}
async function waitForPendingSegments(timeoutMs = 3000, quietMs = 300): Promise<boolean> {
const startedAt = Date.now()
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
const stoppedOrSettled = transcriberStoppedAt > 0 || Date.now() - startedAt >= 500
const quiet = Date.now() - lastTranscriberMessageAt >= quietMs
if (stoppedOrSettled && quiet && pendingSegments.size === 0) return true
await new Promise((resolve) => window.setTimeout(resolve, 50))
}
return false
}
async function performStartTranscription(): Promise<void> {
if (!activeConfig || phase.value !== 'connected') throw new Error('视频接通后才能开始录音')
if (transcriptionState.value !== 'idle' && transcriptionState.value !== 'error') {
throw new Error('录音任务正在处理中')
}
transcriptionState.value = 'starting'
const sessionId = newTranscriptionSessionId()
const token = ++transcriptionGeneration
transcriptionSessionId = sessionId
acknowledgedSegmentIds.clear()
pendingSegments.clear()
lastTranscriberMessageAt = Date.now()
transcriberStoppedAt = 0
notice.value = '正在准备录音文字存储…'
const storageReady = await requestTranscriptionStart(sessionId)
if (!storageReady) {
if (!transcriptionTokenIsCurrent(token, sessionId)) return
transcriptionState.value = 'error'
transcriptionSessionId = ''
throw new Error(notice.value || '服务端无法保存本次面诊对话文字')
}
if (!transcriptionTokenIsCurrent(token, sessionId)) {
await requestTranscriptionStop(sessionId, 'partial')
return
}
let ownedManager: RealtimeTranscriberManager | null = null
let ownedListener: RealtimeTranscriberListener | null = null
try {
const subscribed = subscribeTranscriber()
const { manager, listener } = subscribed
ownedManager = manager
ownedListener = listener
const robotId = await manager.startRealtimeTranscriber({
sourceLanguage: 'zh',
})
if (!robotId) throw new Error('当前腾讯云项目未开通实时语音转写')
if (!transcriptionTokenIsCurrent(token, sessionId)) {
await manager.stopRealtimeTranscriber(robotId)
unsubscribeTranscriber(manager, listener)
return
}
transcriberRobotId = robotId
transcriberRunning = true
transcriptionState.value = 'recording'
notice.value = '正在录音并实时转换为对话文字'
} catch (error) {
unsubscribeTranscriber(ownedManager, ownedListener)
await requestTranscriptionStop(sessionId, 'failed')
if (sessionId === transcriptionSessionId) transcriptionState.value = 'error'
const message = safeErrorMessage(error, '录音转文字启动失败')
notice.value = message
if (sessionId === transcriptionSessionId) transcriptionSessionId = ''
throw new Error(message)
}
}
function startTranscription(): Promise<void> {
if (transcriptionStartPromise) return transcriptionStartPromise
const operation = performStartTranscription()
const tracked = operation.finally(() => {
if (transcriptionStartPromise === tracked) transcriptionStartPromise = null
})
transcriptionStartPromise = tracked
return transcriptionStartPromise
}
async function stopTranscription(
status: 'completed' | 'partial' | 'failed' = 'completed',
managerAlreadyStopped = false,
): Promise<void> {
if (transcriptionStopPromise) return transcriptionStopPromise
if (transcriptionState.value === 'idle') return
if (!transcriptionSessionId) {
transcriptionState.value = 'idle'
return
}
const sessionId = transcriptionSessionId
const startInFlight = transcriptionStartPromise
transcriptionGeneration += 1
transcriptionState.value = 'stopping'
transcriptionStopPromise = (async () => {
if (startInFlight) {
try {
await startInFlight
} catch {
// Its failure is materialized as a failed/partial transcript below.
}
}
let sdkStopped = managerAlreadyStopped
const manager = transcriberManager
const robotId = transcriberRobotId
if (!managerAlreadyStopped && transcriberRunning && manager && robotId) {
try {
await manager.stopRealtimeTranscriber(robotId)
sdkStopped = true
} catch (error) {
console.warn('[doctor-consultation] 停止实时转写失败', safeErrorMessage(error))
}
}
transcriberRunning = false
transcriberRobotId = ''
const segmentsComplete = await waitForPendingSegments()
unsubscribeTranscriber()
const finalStatus = sdkStopped && segmentsComplete ? status : 'partial'
const persisted = await requestTranscriptionStop(sessionId, finalStatus)
if (sessionId !== transcriptionSessionId) return
transcriptionState.value = persisted ? 'idle' : 'error'
notice.value = persisted
? '本次面诊对话文字已保存到视频记录'
: '对话文字未能完整确认保存,请稍后在面诊记录中检查'
transcriptionSessionId = ''
acknowledgedSegmentIds.clear()
pendingSegments.clear()
})().finally(() => {
transcriptionStopPromise = null
})
return transcriptionStopPromise
}
function transcriptionResult(
operation: 'start' | 'segment' | 'stop',
sessionId: string,
segmentId: string,
ok: boolean,
message: string,
): void {
if (operation === 'start') {
const pending = pendingTranscriptionStarts.get(sessionId)
if (!pending) return
pendingTranscriptionStarts.delete(sessionId)
if (message && sessionId === transcriptionSessionId) notice.value = message
pending.resolve(Boolean(ok))
} else if (operation === 'stop') {
const pending = pendingTranscriptionStops.get(sessionId)
if (!pending) return
pendingTranscriptionStops.delete(sessionId)
if (message && sessionId === transcriptionSessionId) notice.value = message
pending.resolve(Boolean(ok))
} else if (operation === 'segment' && sessionId === transcriptionSessionId) {
const pending = pendingSegments.get(segmentId)
if (!pending) return
if (ok) {
pendingSegments.delete(segmentId)
acknowledgedSegmentIds.add(segmentId)
} else if (pending.attempts < 3) {
pending.attempts += 1
window.setTimeout(() => {
if (pendingSegments.get(segmentId) === pending) emit(pending.message)
}, pending.attempts * 250)
} else {
notice.value = message || '部分对话文字保存失败,本次记录将标记为部分保存'
}
}
}
function notifyHangup(status = 'ended'): Promise<void> {
if (hangupNotification) return hangupNotification
if (endNotified) return Promise.resolve()
endNotified = true
phase.value = 'ended'
statusText.value = mode.value === 'chat' ? '视频通话已结束,IM 保持连接' : '视频问诊已结束'
emit({
source: 'doctor-call',
event: 'hangup',
diagnosisId: activeConfig?.diagnosisId,
status,
})
hangupNotification = (async () => {
try {
if (transcriptionState.value !== 'idle') await stopTranscription('completed')
} catch (error) {
notice.value = safeErrorMessage(error, '录音文字收尾失败,请稍后检查面诊记录')
console.warn('[doctor-consultation] 录音文字收尾失败', notice.value)
} finally {
emit({
source: 'doctor-call',
event: 'hangup',
diagnosisId: activeConfig?.diagnosisId,
status,
})
}
})()
return hangupNotification
}
function readRoomId(): string {
@@ -498,21 +935,31 @@ function handleStatusChanged(payload: unknown): void {
: payload
const status = typeof value === 'string' ? value : 'unknown'
if (status === 'connected' || status.startsWith('calling-')) {
if (endNotified || !activeConfig) return
const cycle = callCycleGeneration
phase.value = 'connected'
statusText.value = '视频问诊进行中'
void pollRoomId()
if (autoTranscriptionAttemptedGeneration !== cycle) {
autoTranscriptionAttemptedGeneration = cycle
void startTranscription().catch((error) => {
if (cycle !== callCycleGeneration || endNotified) return
notice.value = safeErrorMessage(error, '自动录音转文字启动失败')
})
}
} else if (status === 'calling' || status.startsWith('dialing')) {
if (endNotified || !activeConfig) return
phase.value = 'dialing'
statusText.value = '正在等待患者接听'
} else if (status === 'idle' && activeConfig && !starting) {
notifyHangup(status)
void notifyHangup(status)
}
emit({ source: 'doctor-call', event: 'status', diagnosisId: activeConfig?.diagnosisId, status })
}
TUICallKitAPI.setCallback({
statusChanged: handleStatusChanged,
afterCalling: () => notifyHangup('after-calling'),
afterCalling: () => { void notifyHangup('after-calling') },
})
TUICallKitAPI.setLanguage('zh-cn')
TUICallKitAPI.enableFloatWindow(false)
@@ -543,11 +990,17 @@ async function startVideo(): Promise<void> {
if (!activeConfig) throw new Error('问诊配置尚未准备好')
if (starting || !endNotified) throw new Error('已有视频通话正在进行')
starting = true
endNotified = false
phase.value = 'starting'
statusText.value = '正在创建安全视频通话'
notice.value = ''
try {
if (hangupNotification) await hangupNotification
if (!activeConfig || !endNotified) throw new Error('已有视频通话正在进行')
hangupNotification = null
callCycleGeneration += 1
endNotified = false
phase.value = 'starting'
statusText.value = '正在创建安全视频通话'
transcriptionState.value = 'idle'
transcriptionSessionId = ''
notice.value = ''
const allowed = await requestHostCallStart()
if (!allowed) throw new Error(notice.value || '服务器未能创建视频通话记录')
await TUICallKitAPI.init({
@@ -579,10 +1032,14 @@ async function startVideo(): Promise<void> {
}
async function hangup(): Promise<void> {
if (!activeConfig || endNotified) return
if (!activeConfig) return
if (endNotified) {
if (hangupNotification) await hangupNotification
return
}
try {
await TUICallKitAPI.hangup()
notifyHangup('local-hangup')
await notifyHangup('local-hangup')
} catch (error) {
const message = safeErrorMessage(error, '结束视频通话失败')
emit({ source: 'doctor-call', event: 'error', diagnosisId: activeConfig.diagnosisId, message })
@@ -610,7 +1067,17 @@ async function open(config: DoctorCallConfig): Promise<void> {
nextReqMessageID = ''
hasMoreMessages.value = false
endNotified = true
hangupNotification = null
callCycleGeneration += 1
autoTranscriptionAttemptedGeneration = -1
phase.value = 'ready'
transcriptionState.value = 'idle'
transcriptionSessionId = ''
transcriptionGeneration += 1
transcriberRunning = false
transcriberRobotId = ''
acknowledgedSegmentIds.clear()
pendingSegments.clear()
notice.value = ''
if (activeConfig.mode === 'chat') {
try {
@@ -629,6 +1096,13 @@ async function open(config: DoctorCallConfig): Promise<void> {
async function close(): Promise<void> {
if (!endNotified) await hangup()
if (hangupNotification) await hangupNotification
unsubscribeTranscriber()
transcriptionGeneration += 1
for (const pending of pendingTranscriptionStarts.values()) pending.resolve(false)
for (const pending of pendingTranscriptionStops.values()) pending.resolve(false)
pendingTranscriptionStarts.clear()
pendingTranscriptionStops.clear()
await logoutChat()
activeConfig = null
phase.value = 'ended'
@@ -641,6 +1115,7 @@ window.doctorConsultation = {
hangup,
hostCallReady,
screenshotResult,
transcriptionResult,
}
window.doctorCall = { start: open, hangup }
initializeQtWebChannel()
@@ -655,6 +1130,7 @@ createApp(App, {
chatBusy: readonly(chatBusy),
notice: readonly(notice),
hasMoreMessages: readonly(hasMoreMessages),
transcriptionState: readonly(transcriptionState),
onSendText: sendText,
onSendAttachment: sendAttachment,
onLoadMore: () => loadMessages(true),
+25
View File
@@ -294,6 +294,31 @@ button:disabled { cursor: not-allowed; opacity: .55; }
gap: 10px;
}
.video-actions button { padding: 10px 15px; border-radius: 10px; font-weight: 600; }
.recording-status {
display: inline-flex;
align-items: center;
gap: 8px;
padding: 10px 15px;
border: 1px solid rgba(255, 255, 255, .24);
border-radius: 10px;
color: #fff;
background: rgba(22, 29, 39, .88);
font-size: 13px;
font-weight: 600;
}
.recording-status--active { border-color: rgba(244, 99, 115, .64); background: rgba(126, 35, 50, .9); }
.recording-status--error { border-color: rgba(242, 109, 109, .56); color: #ffe4e7; background: rgba(100, 31, 40, .88); }
.recording-indicator {
width: 9px;
height: 9px;
border-radius: 50%;
background: #f46373;
box-shadow: 0 0 0 4px rgba(244, 99, 115, .16);
}
.recording-status--active .recording-indicator { animation: recording-pulse 1.25s ease-in-out infinite; }
@keyframes recording-pulse {
50% { box-shadow: 0 0 0 8px rgba(244, 99, 115, .04); opacity: .72; }
}
.hangup-button { border: 1px solid #b44755; color: #fff; background: rgba(161, 47, 61, .9); }
.hangup-button:hover { background: #be394d; }
.video-notice {