This commit is contained in:
Your Name
2026-10-30 13:41:35 +08:00
parent 28ba9b97f2
commit e2fc30e1f7
69 changed files with 8702 additions and 252 deletions
+2 -2
View File
@@ -3,9 +3,9 @@
__all__ = ["DEBUG_MODE", "ONLINE_API_BASE_URL", "__version__"]
# Single source of truth for runtime, package, installer, and executable versions.
__version__ = "1.5.0"
__version__ = "1.5.1"
# 调试模式开启时,登录页显示“演示模式”和“服务器设置”。
# 正式发布请保持 False;此时程序只使用下面配置的线上域名。
DEBUG_MODE = True
DEBUG_MODE = False
ONLINE_API_BASE_URL = "https://admin.zhenyangtang.com.cn"
@@ -0,0 +1,23 @@
"""Offline script conversion for speech transcripts, without paraphrasing."""
from __future__ import annotations
from threading import local
_converters = local()
def simplified_transcript(text: str) -> str:
"""Convert character forms only; retain dialect words, numbers and negations.
Each thread owns its converter. Cache dictionaries, never patient text.
The t2s configuration deliberately excludes regional vocabulary rewriting.
Missing package data is an error, rather than silently mixing scripts.
"""
converter = getattr(_converters, "simplified", None)
if converter is None:
from opencc import OpenCC
converter = OpenCC("t2s")
_converters.simplified = converter
return converter.convert(text)
@@ -214,6 +214,13 @@ class DoctorRepository(Protocol):
def cancel_diagnosis_ai(self, diagnosis_id: int, request_id: str) -> dict[str, Any]:
"""Request server/upstream cancellation of one correlated stream."""
def stream_video_ai(
self, diagnosis_id: int, *, call_record_id: int,
transcription_session_id: str, task: str = "live_consultation",
request_id: str, cancelled: Callable[[], bool] | None = None,
) -> Iterator[dict[str, Any]]:
"""Analyze the server-owned snapshot of this exact video call."""
def generate_ai_prescription(self, diagnosis_id: int) -> dict[str, Any]:
"""Generate a server-validated prescription draft for doctor review."""
@@ -1478,6 +1485,42 @@ class RemoteDoctorRepository:
yield {"event": "delta", "text": answer, "fallback": True}
yield {**result, "event": "done", "fallback": True}
def stream_video_ai(
self, diagnosis_id: int, *, call_record_id: int,
transcription_session_id: str, task: str = "live_consultation",
request_id: str, cancelled: Callable[[], bool] | None = None,
) -> Iterator[dict[str, Any]]:
"""No legacy fallback: an older route cannot bind a live call safely."""
if (isinstance(diagnosis_id, bool) or not isinstance(diagnosis_id, int)
or diagnosis_id <= 0 or isinstance(call_record_id, bool)
or not isinstance(call_record_id, int) or call_record_id <= 0):
raise ValueError("diagnosis and call_record_id must be positive integers")
if (not isinstance(transcription_session_id, str)
or re.fullmatch(r"[A-Za-z0-9_-]{1,128}", transcription_session_id) is None):
raise ValueError("invalid transcription session")
if task not in {"live_consultation", "live_review"}:
raise ValueError("unsupported video AI task")
body = {
"id": diagnosis_id, "task": task,
"prompt": "请依据本次通话已保存的原文整理症状、追问和待核实信息,供医生复核。",
"call_record_id": call_record_id,
"transcription_session_id": transcription_session_id,
"request_id": request_id,
}
for raw in self.client.post_event_stream(
"tcm.diagnosis/aiAssistantStream", body, timeout=105.0, cancelled=cancelled,
):
if cancelled is not None and cancelled():
return
event = _normalise_diagnosis_ai_event(raw)
if event is not None:
yield event
if event["event"] == "done":
return
if cancelled is None or not cancelled():
raise ApiProtocolError("video AI stream ended before done")
def cancel_diagnosis_ai(self, diagnosis_id: int, request_id: str) -> dict[str, Any]:
"""Use an independent short request, even while the stream is idle."""
@@ -3060,10 +3103,10 @@ def _normalise_diagnosis_ai_event(raw: Mapping[str, Any]) -> dict[str, Any] | No
if not isinstance(text, str):
raise ApiProtocolError("AI assistant delta content must be text", data=data)
return {"event": "delta", "text": text}
if kind == "done":
if kind in {"live_update", "done"}:
payload.pop("event", None)
payload.pop("type", None)
return {**payload, "event": "done"}
return {**payload, "event": kind}
if kind == "error":
message = (
data
@@ -48,6 +48,7 @@ from PySide6.QtWidgets import (
)
from ...core.appointment_modes import appointment_type_description, appointment_type_value
from ...core.transcript_text import simplified_transcript
from ..design_system import clinical_stylesheet
from ..diagnosis_drawer import (
DIAGNOSIS_QSS,
@@ -4033,7 +4034,8 @@ class DiagnosisDialog(QDialog):
content = QPlainTextEdit(dialog)
content.setObjectName("DiagnosisCallTranscriptText")
content.setReadOnly(True)
content.setPlainText(transcript)
# Historical records keep their stored source; only the view is converted.
content.setPlainText(simplified_transcript(transcript))
layout.addWidget(content, 1)
close_button = QPushButton("关闭")
close_button.setProperty("variant", "primary")
+25 -8
View File
@@ -18,6 +18,7 @@ from dataclasses import dataclass
from pathlib import Path
from typing import Any, TypeVar
from ..core.transcript_text import simplified_transcript
from .launcher import VideoCallRequest, normalize_backend_ticket
ResultT = TypeVar("ResultT")
@@ -134,7 +135,7 @@ def _extract_cloud_recording_outcome(result: Any) -> tuple[bool, str] | None:
return None
def _clean_transcript_segment(segment: Mapping[str, Any], session_id: str) -> dict[str, Any]:
def normalize_transcript_segment(segment: Mapping[str, Any], session_id: str) -> dict[str, Any]:
segment_id = str(segment.get("segment_id", segment.get("segmentId", ""))).strip()
text = str(segment.get("text", segment.get("sourceText", ""))).strip()
speaker_user_id = str(
@@ -145,13 +146,14 @@ def _clean_transcript_segment(segment: Mapping[str, Any], session_id: str) -> di
raise ValueError("transcript segment_id must contain 1 to 160 characters")
if not text or len(text) > 4_000:
raise ValueError("transcript text must contain 1 to 4000 characters")
text = simplified_transcript(text)
if len(speaker_user_id) > 160:
raise ValueError("transcript speaker_user_id is too long")
if speaker_role not in {"doctor", "patient", "unknown"}:
speaker_role = "unknown"
try:
timestamp = int(segment.get("timestamp") or 0)
except (TypeError, ValueError):
except (TypeError, ValueError, OverflowError):
timestamp = 0
return {
"segment_id": segment_id,
@@ -543,10 +545,25 @@ class OrderedCallLifecycle:
with self._lock:
if self._end_future is not None:
raise RuntimeError("video call has already ended")
if self._transcription_start_future is not None:
if self._transcription_session_id != clean_session:
raise RuntimeError("another transcription session already exists")
return self._transcription_start_future
if self._transcription_finish_future is not None:
raise RuntimeError("call transcription has already stopped")
if (
self._transcription_session_id is not None
and self._transcription_session_id != clean_session
):
raise RuntimeError("another transcription session already exists")
previous = self._transcription_start_future
if previous is not None:
if not previous.done() or (
not previous.cancelled()
and previous.exception() is None
and previous.result() is True
):
return previous
# A completed failure must not poison this call's retry button.
# Keep the exact session identity: the server may have accepted
# an earlier request whose response never reached this client.
self._transcription_active = False
if self._start_future is None:
self.start()
method = getattr(self.repository, "start_call_transcription", None)
@@ -593,7 +610,7 @@ class OrderedCallLifecycle:
raise RuntimeError("call transcription has not started")
if self._transcription_finish_future is not None:
raise RuntimeError("call transcription has already stopped")
cleaned = _clean_transcript_segment(segment, session_id)
cleaned = normalize_transcript_segment(segment, session_id)
segment_id = cleaned["segment_id"]
existing = self._segment_futures.get(segment_id)
if existing is not None:
@@ -738,4 +755,4 @@ class OrderedCallLifecycle:
return self._worker.wait(timeout)
__all__ = ["OrderedCallLifecycle"]
__all__ = ["OrderedCallLifecycle", "normalize_transcript_segment"]
+414
View File
@@ -0,0 +1,414 @@
"""Incremental video assistance; never runs on the call persistence FIFO or Qt.
Only persisted segment notifications trigger work. One in-flight request and
one coalesced latest revision are retained, with explicit patient/call/session
correlation. Clinical content is never written or sent to the patient here.
"""
from __future__ import annotations
import copy
import logging
import math
import re
from collections.abc import Callable, Mapping
from contextlib import suppress
from dataclasses import dataclass, field
from threading import BoundedSemaphore, Event, RLock, Thread
from time import monotonic
from typing import Any
from uuid import uuid4
from ..core.errors import (
ApiBusinessError,
ApiHttpError,
ApiProtocolError,
ApiTimeoutError,
ApiTransportError,
AuthenticationExpiredError,
)
_LOGGER = logging.getLogger(__name__)
_SLOTS = BoundedSemaphore(2)
_CANCEL_SLOTS = BoundedSemaphore(4)
def _failure_details(error: Exception) -> tuple[str, str]:
"""Translate stable error codes, never echo an upstream exception/response."""
if isinstance(error, ApiTimeoutError):
code = "UPSTREAM_TIMEOUT"
elif isinstance(error, AuthenticationExpiredError):
code = "AUTH_REQUIRED"
elif isinstance(error, ApiHttpError):
code = ({401: "AUTH_REQUIRED", 403: "PERMISSION_DENIED", 404: "SERVICE_UPDATE_REQUIRED",
405: "SERVICE_UPDATE_REQUIRED", 429: "RATE_LIMITED"}.get(error.status_code)
or "UPSTREAM_UNAVAILABLE")
elif isinstance(error, ApiBusinessError) and isinstance(error.data, Mapping):
code = error.data.get("error_code") or error.data.get("code")
elif isinstance(error, ApiProtocolError):
code = "STREAM_INCOMPLETE"
elif isinstance(error, ApiTransportError):
code = "NETWORK_ERROR"
elif isinstance(error, ValueError):
code = "LIVE_RESULT_INVALID"
else:
code = "UPSTREAM_UNAVAILABLE"
messages = {
"UPSTREAM_TIMEOUT": "分析服务响应超时,可点击重新生成。",
"AUTH_REQUIRED": "登录状态已失效,请重新登录后使用 AI 辅助。",
"PERMISSION_DENIED": "当前账号没有 AI 辅助权限,请联系管理员开通。",
"SERVICE_UPDATE_REQUIRED": "服务端尚不支持实时 AI,请更新服务端后重试。",
"RATE_LIMITED": "分析请求较多,请稍后重新生成。",
"LIVE_RESULT_INVALID": "本次建议未通过格式或引用校验,可点击重新生成。",
"LIVE_CONTEXT_UNAVAILABLE": "本次通话的分析资料尚未就绪,请等待转写保存后重试。",
"STREAM_INCOMPLETE": "分析连接提前结束,可点击重新生成。",
"NETWORK_ERROR": "分析服务连接中断,请检查网络后重试。",
"UPSTREAM_UNAVAILABLE": "分析服务暂时不可用,请稍后重新生成。",
}
if not isinstance(code, str) or code not in messages:
code = "UPSTREAM_UNAVAILABLE"
if code == "LIVE_RESULT_INVALID" and isinstance(getattr(error, "data", None), Mapping):
reason = error.data.get("live_validation_reason")
details = {
"JSON_INVALID": "分析服务返回的格式不完整,可点击重新生成。",
"SCHEMA_INVALID": "分析服务返回的格式不符合要求,可点击重新生成。",
"CITATION_INVALID": "部分建议的原文引用无法核对,可点击重新生成。",
"SOURCE_QUOTE_MISMATCH": "部分建议与引用原文不一致,可点击重新生成。",
"SAFETY_REJECTED": "本次建议包含超出辅助范围的内容,未通过校验,可重新生成。",
}
if isinstance(reason, str) and reason in details:
return code, details[reason]
return code, messages[code]
def empty_video_ai_state(available: bool) -> dict[str, Any]:
return {
"enabled": available, "available": available, "busy": False,
"phase": "waiting", "message": ("接通并产生转写文字后,自动整理本次对话。" if available
else "当前账号或服务暂不能使用 AI 辅助;语音转写可独立运行。"),
"error": "", "revision": "", "throughTimestampMs": 0, "segmentCount": 0,
"transcriptionStatus": "idle", "summary": [], "questions": [],
"checks": [], "review": [], "sources": [],
"elapsedMs": 0, "lastDurationMs": 0, "firstResultMs": None,
"pendingSegments": 0, "errorCode": "", "analysisScope": "",
}
@dataclass
class _Job:
generation: int
call_id: int
session_id: str
dirty: int
task: str
started_at: float = 0.0
first_result_ms: int | None = None
revision: str = ""
analysis_scope: str | None = None
request_id: str = field(default_factory=lambda: str(uuid4()))
cancelled: Event = field(default_factory=Event)
cancel_sent: bool = False
class LiveAiController:
def __init__(self, repository: Any, *, diagnosis_id: int, enabled: bool,
clock: Callable[[], float] = monotonic, interval: float = 2.0) -> None:
self.repository = repository
self.diagnosis_id = diagnosis_id
self.available = enabled is True and callable(getattr(repository, "stream_video_ai", None))
self._clock = clock
self._interval = max(0, interval)
self._lock = RLock()
self._state = empty_video_ai_state(self.available)
self._mailbox: dict[str, Any] | None = copy.deepcopy(self._state)
self._closed = False
self._generation = 0
self._call_id = 0
self._session_id = ""
self._dirty = 0
self._attempted = 0
self._displayed_dirty = 0
self._due = 0.0
self._forced_task = ""
self._running: _Job | None = None
self._worker: Thread | None = None
def _publish(self, **changes: Any) -> None:
self._state.update(changes)
self._state["pendingSegments"] = max(0, self._dirty - self._displayed_dirty)
self._mailbox = copy.deepcopy(self._state)
def take_state(self) -> dict[str, Any] | None:
with self._lock:
result, self._mailbox = self._mailbox, None
return None if self._closed else result
def bind(self, call_id: int, session_id: str) -> None:
with self._lock:
if self._closed or (self._call_id == call_id and self._session_id == session_id):
return
self._cancel()
enabled = self._state["enabled"]
self._generation += 1
self._call_id, self._session_id = call_id, session_id
self._dirty = self._attempted = 0
self._displayed_dirty = 0
self._forced_task = ""
self._state = empty_video_ai_state(self.available)
self._publish(enabled=enabled, transcriptionStatus="running" if session_id else "idle")
def reset(self) -> None:
"""Clear the previous call before a new call's model/storage is ready."""
self.bind(0, "")
def changed(self, session_id: str) -> None:
with self._lock:
if self._closed or session_id != self._session_id:
return
was_clean = self._dirty == self._attempted
self._dirty += 1
if was_clean:
# Coalesce a short speech burst without repeatedly moving the deadline.
self._due = self._clock() + (0.8 if not self._state["revision"] else self._interval)
self._publish()
def transcription_error(self, message: str, *, status: str = "partial") -> None:
with self._lock:
if not self._closed:
self._publish(transcriptionStatus=status, error=message[:240])
def transcription_starting(self) -> None:
with self._lock:
if not self._closed:
self._publish(transcriptionStatus="starting", error="")
def transcription_running(self, session_id: str) -> None:
"""A retried cloud recognizer is ready without resetting saved evidence."""
with self._lock:
if not self._closed and session_id == self._session_id:
self._publish(transcriptionStatus="running", error="")
def finish(self, session_id: str, status: str) -> None:
with self._lock:
if not self._closed and session_id == self._session_id:
self._publish(transcriptionStatus=status)
def handle(self, action: str, enabled: Any = None) -> None:
with self._lock:
if self._closed or not self.available:
return
if action == "toggle" and isinstance(enabled, bool):
if not enabled:
self._cancel()
self._publish(enabled=enabled, busy=False if not enabled else self._state["busy"],
message="已暂停自动分析。" if not enabled else "有新增对话后自动更新。")
return
if action not in {"refresh", "review"} or not self._dirty or self._running is not None:
return
if action == "review" and self._state["transcriptionStatus"] not in {"completed", "partial"}:
return
self._forced_task = "live_review" if action == "review" else "live_consultation"
self._due = self._clock()
def tick(self) -> None:
with self._lock:
if self._running is not None and self._current(self._running):
elapsed = self._elapsed(self._running)
if elapsed // 1000 != self._state["elapsedMs"] // 1000:
self._publish(elapsedMs=elapsed)
if (self._closed or not self.available or not self._call_id or not self._session_id
or self._running is not None or not self._dirty
or (not self._forced_task and (not self._state["enabled"] or self._dirty <= self._attempted))
or self._clock() < self._due):
return
if not _SLOTS.acquire(blocking=False):
return
job = _Job(self._generation, self._call_id, self._session_id, self._dirty,
self._forced_task or "live_consultation", started_at=self._clock())
self._forced_task = ""
self._attempted = self._dirty
self._running = job
self._publish(busy=True, phase="analyzing", error="", errorCode="", elapsedMs=0, firstResultMs=None,
message="正在分析最新对话,优先整理下一步追问…")
self._worker = Thread(target=self._run, args=(job,), name="video-ai", daemon=True)
try:
self._worker.start()
except RuntimeError:
self._running = None
_SLOTS.release()
self._publish(busy=False, phase="error", error="分析任务无法启动,请重试。")
def _elapsed(self, job: _Job) -> int:
return max(0, int((self._clock() - job.started_at) * 1000))
def _current(self, job: _Job) -> bool:
return not self._closed and job.generation == self._generation and not job.cancelled.is_set()
def _validated_result(self, job: _Job, event: Mapping[str, Any]) -> dict[str, Any]:
context = event.get("live_context")
result = event.get("live_assistance")
# Only label the scope of validated displayed content. Older servers do
# not report it; never infer dialogue-only behavior from the task name.
scope = event.get("analysis_scope", "")
if not isinstance(scope, str) or scope not in {"", "current_call_only", "authorized_case"}:
raise ValueError("invalid analysis scope")
if (not isinstance(context, Mapping) or not isinstance(result, Mapping)
or event.get("diagnosis_id") != self.diagnosis_id
or context.get("call_record_id") != job.call_id
or context.get("transcription_session_id") != job.session_id
or not isinstance(context.get("revision"), str)
or not re.fullmatch(r"[a-f0-9]{64}", context["revision"])):
raise ValueError("context mismatch")
sources = context.get("sources")
if not isinstance(sources, list) or len(sources) > 1000:
raise ValueError("invalid evidence")
known: set[str] = set()
clean_sources = []
for source in sources:
if (not isinstance(source, Mapping) or not isinstance(source.get("id"), str)
or not source["id"] or source["id"] in known
or not isinstance(source.get("text"), str) or not source["text"].strip()
or len(source["text"]) > 4000
or source.get("role") not in {"doctor", "patient", "unknown"}
or isinstance(source.get("timestamp_ms"), bool)
or not isinstance(source.get("timestamp_ms"), int) or source["timestamp_ms"] < 0):
raise ValueError("invalid source")
known.add(source["id"])
clean_sources.append({key: source.get(key) for key in ("id", "text", "role", "timestamp_ms")})
cleaned: dict[str, Any] = {}
for key in ("summary", "questions", "checks", "review"):
items = result.get(key, [])
if not isinstance(items, list) or len(items) > 6:
raise ValueError("invalid items")
cleaned[key] = []
for item in items:
if (not isinstance(item, Mapping) or not isinstance(item.get("text"), str)
or not item["text"].strip() or len(item["text"]) > (4020 if key == "summary" else 600)
or not isinstance(item.get("evidence_ids"), list)
or not 1 <= len(item["evidence_ids"]) <= 8
or any(not isinstance(s, str) or s not in known for s in item["evidence_ids"])
or len(set(item["evidence_ids"])) != len(item["evidence_ids"])):
raise ValueError("invalid cited item")
reason = item.get("reason", "")
if not isinstance(reason, str) or len(reason) > 600:
raise ValueError("invalid reason")
cleaned[key].append({"text": item["text"], "reason": reason,
"evidence_ids": list(item["evidence_ids"])})
count = context.get("segment_count")
through = context.get("through_timestamp_ms")
if (isinstance(count, bool) or not isinstance(count, int) or count < 1
or isinstance(through, bool) or not isinstance(through, (int, float))
or not math.isfinite(through) or through < 0
or context.get("transcription_status") not in {"running", "completed", "partial"}):
raise ValueError("invalid revision metadata")
return {**cleaned, "sources": clean_sources, "revision": context["revision"], "analysisScope": scope,
"segmentCount": count, "throughTimestampMs": through,
"transcriptionStatus": context.get("transcription_status", "partial")}
def _run(self, job: _Job) -> None:
events = None
success = False
try:
if job.cancelled.is_set():
return
events = self.repository.stream_video_ai(
self.diagnosis_id, call_record_id=job.call_id, transcription_session_id=job.session_id,
task=job.task, request_id=job.request_id, cancelled=job.cancelled.is_set,
)
for event in events:
with self._lock:
if not self._current(job):
return
if event.get("event") in {"start", "progress"}:
self._publish(message=str(event.get("message") or "正在分析本次对话…")[:200])
elif event.get("event") in {"live_update", "done"}:
result = self._validated_result(job, event)
if job.revision and job.revision != result["revision"]:
raise ValueError("revision changed inside a single snapshot")
if job.analysis_scope is not None and job.analysis_scope != result["analysisScope"]:
raise ValueError("analysis scope changed inside a single snapshot")
job.revision = result["revision"]
job.analysis_scope = result["analysisScope"]
# Preserve a newer native completion status while an older
# in-flight snapshot still reports running.
if self._state["transcriptionStatus"] in {"completed", "partial", "failed"}:
result["transcriptionStatus"] = self._state["transcriptionStatus"]
complete = event.get("event") == "done"
duration = self._elapsed(job)
if job.first_result_ms is None and any(
result[key] for key in ("summary", "questions", "checks", "review")
):
job.first_result_ms = duration
self._displayed_dirty = job.dirty
self._publish(**result, busy=not complete,
phase="ready" if complete else "analyzing", error="", errorCode="",
elapsedMs=duration,
firstResultMs=job.first_result_ms,
lastDurationMs=duration if complete else self._state["lastDurationMs"],
message="已更新,请对照原话复核。" if complete
else "已显示可核对的建议,其余内容仍在整理。")
if complete:
success = True
return
if not job.cancelled.is_set():
raise ApiProtocolError("incomplete stream")
except Exception as error:
with self._lock:
if self._current(job):
code, message = _failure_details(error)
self._publish(busy=False, phase="error", error=message, errorCode=code,
elapsedMs=self._elapsed(job), lastDurationMs=self._elapsed(job),
message=("已显示的建议仍可核对,本次分析尚未全部完成。"
if job.first_result_ms is not None else
"保留上次结果;新内容尚未分析完成。" if any(
self._state[key] for key in ("summary", "questions", "checks", "review")
) else
"本次暂未生成可用建议,可点击重新生成。"))
# Log categories/timing only, never response text or patient details.
_LOGGER.warning("video_ai_failed code=%s type=%s duration_ms=%d",
code, type(error).__name__, self._elapsed(job))
finally:
if events is not None and callable(getattr(events, "close", None)):
with suppress(Exception):
events.close()
with self._lock:
if not success:
self._send_cancel(job)
if self._running is job:
self._running = None
_SLOTS.release()
if success:
_LOGGER.info("video_ai_completed duration_ms=%d first_result_ms=%s",
self._elapsed(job), job.first_result_ms)
def _cancel(self) -> None:
if self._running is not None:
self._running.cancelled.set()
self._send_cancel(self._running)
def _send_cancel(self, job: _Job) -> None:
if job.cancel_sent or not _CANCEL_SLOTS.acquire(blocking=False):
return
job.cancel_sent = True
def cancel() -> None:
try:
self.repository.cancel_diagnosis_ai(self.diagnosis_id, job.request_id)
except Exception:
pass
finally:
_CANCEL_SLOTS.release()
try:
Thread(target=cancel, name="video-ai-cancel", daemon=True).start()
except RuntimeError:
_CANCEL_SLOTS.release()
def close(self) -> None:
with self._lock:
self._closed = True
self._cancel()
self._mailbox = None
def wait(self, timeout: float = 1.0) -> bool:
if self._worker is not None:
self._worker.join(max(0, min(timeout, 5)))
return self._worker is None or not self._worker.is_alive()
@@ -0,0 +1,446 @@
"""Bounded, in-memory speech recognition using an explicitly installed local model.
This module has no dependency on Qt, the recording queue, or the call lifecycle.
Callbacks run on one daemon worker and must only enqueue work (for example, emit
a Qt signal). Runtime code never downloads a model or sends audio to a service.
"""
from __future__ import annotations
import base64
import binascii
import importlib.util
import os
import re
import sys
import threading
from collections import deque
from collections.abc import Callable
from contextlib import suppress
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Protocol
from platformdirs import user_data_path
from ..core.transcript_text import simplified_transcript
SAMPLE_RATE = 16_000
MAX_CHUNK_SECONDS = 12
MAX_PCM_BYTES = SAMPLE_RATE * 2 * MAX_CHUNK_SECONDS
MAX_BASE64_CHARS = ((MAX_PCM_BYTES + 2) // 3) * 4
MAX_SEQUENCE = 2_147_483_647
MAX_TIMESTAMP_MS = 253_402_300_799_999
MAX_TRANSCRIPT_CHARS = 4_000
_SESSION_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z")
_REQUIRED_FILES = ("model.bin", "config.json", "tokenizer.json")
_DEPENDENCIES = ("faster_whisper", "numpy", "ctranslate2", "onnxruntime", "opencc")
_MESSAGES = {
"model_missing": "本地语音模型未安装或文件不完整,请先运行本地转写安装工具。",
"dependency_missing": "本地转写运行库未安装,请先安装本地转写依赖。",
"dependency_unavailable": "本地转写运行库无法加载,请检查安装包及系统运行库。",
"model_load_failed": "本地语音模型加载失败,请检查模型文件和本机运行环境。",
"conversion_unavailable": "简体转换字典无法加载,请修复本地转写安装。",
"invalid_audio": "本地转写音频格式无效,需要最长 12 秒的单声道 16 kHz PCM16 音频。",
"invalid_metadata": "本地转写音频的角色、序号或时间戳无效。",
"sequence_out_of_order": "本地转写音频序号重复或倒序,该片段未被接受。",
"sequence_gap": "本地转写检测到音频片段缺失,转写结果可能不完整。",
"overloaded": "本地转写处理速度不足,部分音频片段未转写;通话和录音继续。",
"transcription_failed": "本地音频片段转写失败,结果可能不完整;通话和录音继续。",
}
def default_model_path() -> Path:
"""Return the model directory without creating it or accessing the network."""
override = os.getenv("DOCTOR_ASR_MODEL_PATH", "").strip()
if override:
return Path(override).expanduser().resolve()
bundle = Path(getattr(sys, "_MEIPASS", Path(__file__).resolve().parents[3]))
bundled_model = bundle / "resources" / "local-asr" / "model"
if bundled_model.is_dir():
return bundled_model
return (
user_data_path("ZhenyangDoctorWorkstation", appauthor="ZhenYangTang")
/ "local-asr"
/ "model"
)
def _missing_model_files(path: Path) -> list[str]:
def exists(name: str) -> bool:
try:
file = path / name
return file.is_file() and file.stat().st_size > 0
except OSError:
return False
missing = [name for name in _REQUIRED_FILES if not exists(name)]
if not any(exists(name) for name in ("vocabulary.txt", "vocabulary.json")):
missing.append("vocabulary.txt or vocabulary.json")
return missing
def local_transcription_readiness(model_path: Path | str | None = None) -> dict[str, Any]:
"""Cheap installation preflight; ``available`` does not claim a model has loaded."""
path = Path(model_path).expanduser().resolve() if model_path else default_model_path()
missing_dependencies = []
for name in _DEPENDENCIES:
try:
available = importlib.util.find_spec(name) is not None
except (ImportError, ValueError):
available = False
if not available:
missing_dependencies.append(name)
missing_files = _missing_model_files(path)
code = (
"model_missing" if missing_files else "dependency_missing" if missing_dependencies else ""
)
return {
"available": not (missing_files or missing_dependencies),
"modelPath": str(path),
"missingDependencies": missing_dependencies,
"missingFiles": missing_files,
"code": code,
"message": _MESSAGES.get(code, "本地转写文件检查通过,启动后将加载模型。"),
}
class LocalTranscriptionError(RuntimeError):
"""A fixed, patient-data-free error suitable for displaying in the UI."""
def __init__(self, code: str) -> None:
self.code = code if code in _MESSAGES else "transcription_failed"
super().__init__(_MESSAGES[self.code])
class SpeechEngine(Protocol):
def transcribe(self, pcm: bytes) -> str:
"""Return verbatim recognized text for mono, little-endian PCM16 at 16 kHz."""
class FasterWhisperEngine:
"""CPU/int8 recognition. Construct only on a background worker or setup CLI."""
def __init__(self, model_path: Path | str | None = None) -> None:
path = Path(model_path).expanduser().resolve() if model_path else default_model_path()
# In particular, require tokenizer.json: upstream may otherwise fetch a
# tokenizer even when WhisperModel(local_files_only=True) is requested.
if _missing_model_files(path):
raise LocalTranscriptionError("model_missing")
try:
import numpy as np
from faster_whisper import WhisperModel
from faster_whisper.vad import get_vad_model
except ModuleNotFoundError:
raise LocalTranscriptionError("dependency_missing") from None
except (ImportError, OSError):
raise LocalTranscriptionError("dependency_unavailable") from None
self._np = np
try:
# Load local dictionaries before ready, so a broken/frozen install
# cannot claim successful ASR and then fail on its first sentence.
simplified_transcript("")
except ImportError:
raise LocalTranscriptionError("dependency_missing") from None
except Exception:
raise LocalTranscriptionError("conversion_unavailable") from None
try:
self._model = WhisperModel(
str(path),
device="cpu",
compute_type="int8",
cpu_threads=min(4, max(1, (os.cpu_count() or 2) // 2)),
num_workers=1,
local_files_only=True,
)
# Silero's ONNX asset ships inside faster-whisper; preflight it so
# a packaged application cannot report ready with a missing VAD.
get_vad_model()
except Exception:
raise LocalTranscriptionError("model_load_failed") from None
def transcribe(self, pcm: bytes) -> str:
if not pcm or len(pcm) % 2 or len(pcm) > MAX_PCM_BYTES:
raise LocalTranscriptionError("invalid_audio")
if not any(pcm):
return ""
audio = self._np.frombuffer(pcm, dtype="<i2").astype(self._np.float32) / 32768.0
segments, _ = self._model.transcribe(
audio,
language="zh",
task="transcribe",
beam_size=5,
temperature=0.0,
condition_on_previous_text=False,
vad_filter=True,
vad_parameters={
"min_speech_duration_ms": 0,
"min_silence_duration_ms": 500,
"speech_pad_ms": 400,
},
)
# Iterating is essential: faster-whisper returns a lazy generator. Do
# not rewrite numbers, units, negations, repetitions, or short answers.
return "".join(segment.text for segment in segments).strip()
@dataclass(frozen=True, slots=True)
class _Chunk:
role: str
sequence: int
timestamp_ms: int
pcm: bytes
@dataclass(slots=True)
class _Session:
id: str
initialized: bool = False
stopping: bool = False
stopped: bool = False
partial: bool = False
last_sequence: dict[str, int] = field(default_factory=dict)
chunks: deque[_Chunk] = field(default_factory=deque)
errors: deque[str] = field(default_factory=deque)
reported_errors: set[str] = field(default_factory=set)
class LocalTranscriptionController:
"""One bounded daemon worker, isolated by session object identity.
``start`` accepts a new session, replacing queued old work. ``submit`` only
acknowledges enqueueing, while ``ready`` confirms successful model loading.
``stop`` rejects further submissions and drains accepted chunks in order.
``close`` cancels immediately without waiting for native inference. No
callbacks from an old generation occur after start/close returns.
The callback must be quick, thread-safe, and must not join/wait for another
thread that is calling this controller. It is invoked under a reentrant
lock, which makes close/session replacement atomic with callback delivery.
"""
def __init__(
self,
on_event: Callable[[dict[str, Any]], None],
engine_factory: Callable[[], SpeechEngine] | None = None,
*,
max_pending_chunks: int = 12,
) -> None:
if type(max_pending_chunks) is not int or not 1 <= max_pending_chunks <= 64:
raise ValueError("max_pending_chunks must be an integer between 1 and 64")
self._on_event = on_event
self._engine_factory = engine_factory or FasterWhisperEngine
self._max_pending_chunks = max_pending_chunks
self._condition = threading.Condition(threading.RLock())
self._session: _Session | None = None
self._engine: SpeechEngine | None = None
self._closed = False
self._worker: threading.Thread | None = None
def start(self, session_id: str) -> bool:
if not isinstance(session_id, str) or not _SESSION_ID.fullmatch(session_id):
return False
with self._condition:
if self._closed:
return False
if self._session and self._session.id == session_id:
return not (self._session.stopping or self._session.stopped)
if self._session:
self._session.chunks.clear()
self._session.errors.clear()
self._session = _Session(session_id)
if self._worker is None:
self._worker = threading.Thread(
target=self._run, name="doctor-local-asr", daemon=True
)
self._worker.start()
self._condition.notify_all()
return True
def submit(
self,
session_id: str,
role: str,
sequence: int,
timestamp_ms: int,
pcm_base64: str,
) -> bool:
with self._condition:
session = self._session
if (
self._closed
or session is None
or session.id != session_id
or session.stopping
or session.stopped
):
return False
if (
role not in ("doctor", "patient")
or type(sequence) is not int
or not 0 <= sequence <= MAX_SEQUENCE
or type(timestamp_ms) is not int
or not 0 < timestamp_ms <= MAX_TIMESTAMP_MS
):
self._error_locked(session, "invalid_metadata")
return False
if (
not isinstance(pcm_base64, str)
or not pcm_base64
or len(pcm_base64) > MAX_BASE64_CHARS
):
self._error_locked(session, "invalid_audio")
return False
try:
pcm = base64.b64decode(pcm_base64, validate=True)
except (ValueError, binascii.Error):
self._error_locked(session, "invalid_audio")
return False
if not pcm or len(pcm) % 2 or len(pcm) > MAX_PCM_BYTES:
self._error_locked(session, "invalid_audio")
return False
previous = session.last_sequence.get(role)
if previous is not None and sequence <= previous:
self._error_locked(session, "sequence_out_of_order")
return False
if previous is not None and sequence != previous + 1:
self._error_locked(session, "sequence_gap")
session.last_sequence[role] = sequence
if len(session.chunks) >= self._max_pending_chunks:
self._error_locked(session, "overloaded")
return False
session.chunks.append(_Chunk(role, sequence, timestamp_ms, pcm))
self._condition.notify_all()
return True
def stop(self, session_id: str) -> bool:
with self._condition:
session = self._session
if self._closed or session is None or session.id != session_id:
return False
session.stopping = True
self._condition.notify_all()
return True
def close(self) -> None:
with self._condition:
self._closed = True
if self._session:
self._session.chunks.clear()
self._session.errors.clear()
self._session = None
self._condition.notify_all()
def _error_locked(self, session: _Session, code: str) -> None:
session.partial = True
# At most one pending/report per fixed error code, even when the browser
# continues submitting into an overloaded queue. No unbounded UI queue.
if code not in session.reported_errors:
session.reported_errors.add(code)
session.errors.append(code)
self._condition.notify_all()
def _emit(self, session: _Session, event: dict[str, Any]) -> None:
with self._condition:
if self._closed or self._session is not session:
return
# A detached UI must not kill the worker or expose transcript
# contents through exception logging.
with suppress(Exception):
self._on_event({"sessionId": session.id, **event})
def _initialize(self, session: _Session) -> None:
try:
if self._engine is None:
self._engine = self._engine_factory()
except Exception as error:
code = error.code if isinstance(error, LocalTranscriptionError) else "model_load_failed"
with self._condition:
if self._closed or self._session is not session:
return
session.initialized = True
session.stopping = True
session.chunks.clear()
self._error_locked(session, code)
return
with self._condition:
if self._closed or self._session is not session:
return
session.initialized = True
if not session.stopping:
self._emit(session, {"type": "ready"})
def _transcribe(self, session: _Session, chunk: _Chunk) -> None:
try:
# Reject only exact digital silence here. VAD in the real engine
# handles room noise without an amplitude cutoff that loses quiet
# numerals or one-word negations.
if not any(chunk.pcm):
return
assert self._engine is not None
text = self._engine.transcribe(chunk.pcm)
if not isinstance(text, str) or len(text) > MAX_TRANSCRIPT_CHARS:
raise LocalTranscriptionError("transcription_failed")
# Normalize before persistence and captions, so AI evidence and the
# doctor's visible transcript refer to the exact same text.
text = simplified_transcript(text.strip())
if text:
self._emit(
session,
{
"type": "segment",
"segmentId": f"{session.id}:{chunk.role}:{chunk.sequence}",
"role": chunk.role,
"text": text,
"timestamp": chunk.timestamp_ms,
"isCompleted": True,
},
)
except Exception:
with self._condition:
if not self._closed and self._session is session:
self._error_locked(session, "transcription_failed")
def _run(self) -> None:
try:
while True:
with self._condition:
while True:
if self._closed:
return
session = self._session
if session is not None and not session.stopped:
if session.errors:
action, payload = "error", session.errors.popleft()
break
if not session.initialized:
action, payload = "initialize", None
break
if session.chunks:
action, payload = "chunk", session.chunks.popleft()
break
if session.stopping:
session.stopped = True
action, payload = "stopped", None
break
self._condition.wait()
if action == "initialize":
self._initialize(session)
elif action == "chunk":
self._transcribe(session, payload)
payload = None # Release raw PCM before waiting for work.
elif action == "error":
self._emit(
session,
{"type": "error", "code": payload, "message": _MESSAGES[payload]},
)
else:
self._emit(
session,
{
"type": "stopped",
"status": "partial" if session.partial else "completed",
},
)
finally:
self._engine = None
+228 -6
View File
@@ -11,6 +11,7 @@ import base64
import binascii
import json
import logging
import math
import os
import re
import sqlite3
@@ -34,7 +35,8 @@ from .launcher import (
VideoTicketError,
require_supported_backend,
)
from .lifecycle import OrderedCallLifecycle
from .lifecycle import OrderedCallLifecycle, normalize_transcript_segment
from .live_ai import LiveAiController
from .screenshot import ScreenshotPreviewSize, capture_view_region
from .security import TrustedDocumentError, TrustedDocumentPolicy
@@ -296,6 +298,16 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
}
)
@Slot(str, str, int, float, str) # type: ignore[misc]
def appendLocalTranscriptionChunk( # noqa: N802
self, session_id: str, role: str, sequence: int, timestamp: float, encoded: str
) -> None:
if len(encoded) > 512_000:
return
self._callback({"source": "doctor-call", "event": "local-transcription-chunk",
"sessionId": session_id, "role": role, "sequence": sequence,
"timestamp": timestamp, "data": encoded})
class _EmbeddedVideoWindow(QMainWindow): # type: ignore[misc, valid-type]
status_changed = Signal(str) # type: ignore[misc]
call_ended = Signal(str) # type: ignore[misc]
@@ -304,7 +316,8 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
_policy_completed = Signal(str, object) # type: ignore[misc]
_room_completed = Signal(str, bool, str) # type: ignore[misc]
_screenshot_completed = Signal(bool, str) # type: ignore[misc]
_transcription_completed = Signal(str, str, str, bool, str) # type: ignore[misc]
_transcription_completed = Signal(str, str, str, bool, str, str) # type: ignore[misc]
_local_transcription_event = Signal(object) # type: ignore[misc]
_local_recording_completed = Signal( # type: ignore[misc]
str, str, int, bool, str
)
@@ -364,6 +377,20 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
self._im_ai_timer.timeout.connect(self._flush_im_ai)
if self._im_ai.enabled:
self._im_ai_timer.start()
self._live_ai = LiveAiController(
repository, diagnosis_id=request.diagnosis_id, enabled=can_ai_assistant is True,
)
# The default recognizer is Tencent's existing call transcriber.
# Keep local audio recording independent; no local model is loaded.
self._local_transcription = None
self._asr_session_id = ""
self._asr_ready = False
self._asr_partial = False
self._transcription_final_status = "idle"
self._video_ai_timer = QTimer(self)
self._video_ai_timer.setInterval(100)
self._video_ai_timer.timeout.connect(self._flush_video_ai)
self._video_ai_timer.start()
self._screenshot_preview_size = ScreenshotPreviewSize()
self._local_audio_capture: _LocalAudioCapture | None = None
self._local_audio_store: LocalAudioQueueStore | None = None
@@ -414,6 +441,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
self._room_completed.connect(self._on_room_completed)
self._screenshot_completed.connect(self._on_screenshot_completed)
self._transcription_completed.connect(self._on_transcription_completed)
self._local_transcription_event.connect(self._on_local_transcription)
self._local_recording_completed.connect(self._on_local_recording_completed)
self.web_view.loadFinished.connect(self._on_load_finished)
self.web_view.setUrl(QUrl(self.location.url))
@@ -554,6 +582,14 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
if self._closing:
return
event = str(message.get("event", ""))
if event.startswith("local-transcription-"):
# An old companion must not start a second recognizer.
return
if event == "video-ai-action":
self._handle_video_assistance_message(message)
return
if event.startswith("transcription-") and not self._local_transcription_context_is_trusted():
return
if event in {"im-ai-request", "im-ai-cancel"}:
self._handle_im_ai_message(message)
return
@@ -604,6 +640,9 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
self._abort_local_audio_recording(str(message.get("sessionId") or ""))
return
if event == "transcription-start-request":
if (self._shutdown_requested or not self._start_requested
or self._call_cycle_closed or not self.lifecycle.call_record_id):
return
self._start_transcription(
str(message.get("sessionId") or ""),
str(message.get("language") or "zh-CN"),
@@ -612,6 +651,22 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
if event == "transcription-segment":
self._save_transcript_segment(message)
return
if event == "transcription-ready":
session_id = str(message.get("sessionId") or "")
if (not self._shutdown_requested and session_id
and session_id == str(self.lifecycle.transcription_session_id or "")):
self._live_ai.transcription_running(session_id)
return
if event == "transcription-error":
if (str(message.get("sessionId") or "")
== str(self.lifecycle.transcription_session_id or "")
and self.lifecycle.transcription_session_id):
self._asr_partial = True
self._live_ai.transcription_error(
str(message.get("message") or "腾讯云实时转写未完成,请重试。")[:240],
status="failed",
)
return
if event == "transcription-stop":
self._finish_transcription(
str(message.get("sessionId") or ""),
@@ -679,6 +734,110 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
if timer is not None:
timer.stop()
def _local_transcription_context_is_trusted(self) -> bool:
return (not self._closing and not self._released and self._injected
and self._policy.allows_main_document(self._page.url().toString()))
def _handle_video_assistance_message(self, message: Mapping[str, Any]) -> None:
if not self._local_transcription_context_is_trusted():
return
event = message.get("event")
if event == "video-ai-action":
if not self._shutdown_requested:
self._live_ai.handle(str(message.get("action") or ""), message.get("enabled"))
return
session_id = str(message.get("sessionId") or "")
if event == "local-transcription-start":
if (self._shutdown_requested or not self._start_requested or self._call_cycle_closed
or not self.lifecycle.call_record_id
or re.fullmatch(r"[A-Za-z0-9_-]{1,128}", session_id) is None):
return
existing = str(self.lifecycle.transcription_session_id or "")
if existing and existing != session_id:
return
was_ready = self._asr_session_id == session_id and self._asr_ready
if self._asr_session_id != session_id:
self._asr_partial = False
self._asr_ready = False
self._asr_session_id = session_id
self._live_ai.transcription_starting()
accepted = self._local_transcription.start(session_id)
if accepted and was_ready:
self._on_local_transcription({"type": "ready", "sessionId": session_id})
elif not accepted:
self._on_local_transcription({"type": "error", "sessionId": session_id,
"message": "本次转写已结束,请重新发起通话。"})
elif session_id and session_id == self._asr_session_id:
if event == "local-transcription-chunk":
if session_id != str(self.lifecycle.transcription_session_id or ""):
return
timestamp = message.get("timestamp")
if (isinstance(timestamp, bool) or not isinstance(timestamp, (int, float))
or not math.isfinite(timestamp) or not 0 < timestamp < 2**53):
return
self._local_transcription.submit(
session_id, str(message.get("role") or ""), message.get("sequence"),
int(timestamp), str(message.get("data") or ""),
)
elif event == "local-transcription-stop":
self._local_transcription.stop(session_id)
def _emit_local_transcription(self, event: dict[str, Any]) -> None:
with suppress(RuntimeError):
self._local_transcription_event.emit(event)
def _on_local_transcription(self, event: Mapping[str, Any]) -> None:
if (not self._local_transcription_context_is_trusted()
or event.get("sessionId") != self._asr_session_id):
return
kind = event.get("type")
if kind == "ready":
self._asr_ready = True
elif kind == "stopped":
self._asr_ready = False
elif kind == "segment":
role = event.get("role")
if role not in {"doctor", "patient"}:
return
self._save_transcript_segment({
"sessionId": self._asr_session_id,
"segment": {
"segment_id": event.get("segmentId"),
"speaker_role": role,
"speaker_user_id": (self.request.user_id if role == "doctor"
else self.request.target_user_id),
"timestamp": event.get("timestamp"), "text": event.get("text"),
},
})
elif kind == "error":
self._asr_partial = True
self._live_ai.transcription_error(
str(event.get("message") or "本地转写未完成"),
status="partial" if self._asr_ready else "failed",
)
payload = json.dumps(dict(event), ensure_ascii=True, separators=(",", ":"))
self._page.runJavaScript(f"window.doctorConsultation?.localTranscriptionEvent?.({payload});")
def _flush_video_ai(self) -> None:
if not self._local_transcription_context_is_trusted() or self._shutdown_requested:
return
self._live_ai.tick()
state = self._live_ai.take_state()
if state is not None:
payload = json.dumps(state, ensure_ascii=True, separators=(",", ":"))
self._page.runJavaScript(f"window.doctorConsultation?.videoAiState?.({payload});")
def _close_video_assistance(self, *, close_asr: bool = True) -> None:
controller = getattr(self, "_live_ai", None)
if controller is not None:
controller.close()
timer = getattr(self, "_video_ai_timer", None)
if timer is not None:
timer.stop()
controller = getattr(self, "_local_transcription", None)
if close_asr and controller is not None:
controller.close()
def _open_diagnosis_safely(self) -> None:
if self._closing or self._on_open_diagnosis is None:
return
@@ -1063,6 +1222,12 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
if self._closing or self._shutdown_requested or self._start_requested:
return
self._prepare_call_cycle()
live_ai = getattr(self, "_live_ai", None)
if live_ai is not None:
live_ai.reset()
self._asr_session_id = ""
self._asr_ready = False
self._asr_partial = False
self._start_requested = True
try:
future = self.lifecycle.start(call_type=call_type)
@@ -1141,6 +1306,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
session_id: str,
segment_id: str,
future: Future[bool],
normalized_segment: Mapping[str, Any] | None = None,
) -> None:
try:
succeeded = bool(future.result())
@@ -1155,7 +1321,8 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
}.get(operation, "")
with suppress(RuntimeError):
self._transcription_completed.emit(
operation, session_id, segment_id, succeeded, message
operation, session_id, segment_id, succeeded, message,
json.dumps(normalized_segment if succeeded else None, ensure_ascii=True),
)
def _on_transcription_completed(
@@ -1165,16 +1332,47 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
segment_id: str,
succeeded: bool,
message: str,
normalized_segment: str | Mapping[str, Any] | None = None,
) -> None:
if not self._local_transcription_context_is_trusted():
return
live_ai = getattr(self, "_live_ai", None)
if (live_ai is not None
and session_id == str(self.lifecycle.transcription_session_id or "")):
if operation == "start" and succeeded:
live_ai.bind(int(self.lifecycle.call_record_id), session_id)
elif operation == "start":
live_ai.transcription_error(message or "转写文字存储未就绪,请重试。", status="failed")
elif operation == "segment":
if succeeded:
live_ai.changed(session_id)
else:
self._asr_partial = True
live_ai.transcription_error("部分对话文字保存失败,分析资料可能不完整。")
elif operation == "stop":
live_ai.finish(session_id, self._transcription_final_status if succeeded else "partial")
payload = json.dumps(str(message)[:200], ensure_ascii=True)
state = "true" if succeeded else "false"
if isinstance(normalized_segment, str):
try:
normalized_segment = json.loads(normalized_segment)
except (ValueError, TypeError):
normalized_segment = None
caption = json.dumps(
dict(normalized_segment)
if succeeded and operation == "segment" and isinstance(normalized_segment, Mapping)
else None,
ensure_ascii=True,
)
self._page.runJavaScript(
"window.doctorConsultation?.transcriptionResult?.("
f"{json.dumps(operation)}, {json.dumps(session_id)}, "
f"{json.dumps(segment_id)}, {state}, {payload});"
f"{json.dumps(segment_id)}, {state}, {payload}, {caption});"
)
def _start_transcription(self, session_id: str, language: str) -> None:
if not self._local_transcription_context_is_trusted() or self._shutdown_requested:
return
try:
future = self.lifecycle.start_transcription(session_id, language=language)
except Exception as error:
@@ -1182,6 +1380,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
"start", session_id, "", False, str(error)[:200]
)
return
self._live_ai.transcription_starting()
future.add_done_callback(
lambda completed: self._notify_transcription_completed(
"start", session_id, "", completed
@@ -1189,6 +1388,8 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
)
def _save_transcript_segment(self, message: Mapping[str, Any]) -> None:
if not self._local_transcription_context_is_trusted():
return
session_id = str(message.get("sessionId") or "").strip()
segment = message.get("segment")
segment_id = (
@@ -1207,25 +1408,44 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
)
return
try:
future = self.lifecycle.save_transcript_segment(segment)
normalized = normalize_transcript_segment(segment, session_id)
speaker = normalized["speaker_user_id"]
normalized["speaker_role"] = (
"doctor" if speaker == self.request.user_id
else "patient" if speaker == self.request.target_user_id
else "unknown"
)
future = self.lifecycle.save_transcript_segment(normalized)
except Exception as error:
self._on_transcription_completed(
"segment", session_id, segment_id, False, str(error)[:200]
)
return
caption = {
"segmentId": normalized["segment_id"],
"speakerUserId": normalized["speaker_user_id"],
"sourceText": normalized["text"],
"timestamp": normalized["timestamp"],
"isCompleted": True,
}
future.add_done_callback(
lambda completed: self._notify_transcription_completed(
"segment", session_id, segment_id, completed
"segment", session_id, segment_id, completed, caption
)
)
def _finish_transcription(self, session_id: str, status: str) -> None:
if not self._local_transcription_context_is_trusted():
return
if session_id.strip() != str(self.lifecycle.transcription_session_id or ""):
self._on_transcription_completed(
"stop", session_id, "", False, "录音会话标识不匹配。"
)
return
try:
if getattr(self, "_asr_partial", False):
status = "partial"
self._transcription_final_status = status
future = self.lifecycle.finish_transcription(status=status)
except Exception as error:
self._on_transcription_completed(
@@ -1255,6 +1475,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
"""Let MediaRecorder finish and upload before WebEngine is destroyed."""
_EmbeddedVideoWindow._close_im_ai(self)
_EmbeddedVideoWindow._close_video_assistance(self, close_asr=False)
self._close_reason = reason
if self._closing or self._released:
return
@@ -1300,6 +1521,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
if self._closing:
return
_EmbeddedVideoWindow._close_im_ai(self)
_EmbeddedVideoWindow._close_video_assistance(self)
self._on_compact_close = None
self._closing = True
self._media_active = False