更新
This commit is contained in:
@@ -3,9 +3,9 @@
|
||||
__all__ = ["DEBUG_MODE", "ONLINE_API_BASE_URL", "__version__"]
|
||||
|
||||
# Single source of truth for runtime, package, installer, and executable versions.
|
||||
__version__ = "1.5.0"
|
||||
__version__ = "1.5.1"
|
||||
|
||||
# 调试模式开启时,登录页显示“演示模式”和“服务器设置”。
|
||||
# 正式发布请保持 False;此时程序只使用下面配置的线上域名。
|
||||
DEBUG_MODE = True
|
||||
DEBUG_MODE = False
|
||||
ONLINE_API_BASE_URL = "https://admin.zhenyangtang.com.cn"
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Offline script conversion for speech transcripts, without paraphrasing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from threading import local
|
||||
|
||||
_converters = local()
|
||||
|
||||
|
||||
def simplified_transcript(text: str) -> str:
|
||||
"""Convert character forms only; retain dialect words, numbers and negations.
|
||||
|
||||
Each thread owns its converter. Cache dictionaries, never patient text.
|
||||
The t2s configuration deliberately excludes regional vocabulary rewriting.
|
||||
Missing package data is an error, rather than silently mixing scripts.
|
||||
"""
|
||||
converter = getattr(_converters, "simplified", None)
|
||||
if converter is None:
|
||||
from opencc import OpenCC
|
||||
|
||||
converter = OpenCC("t2s")
|
||||
_converters.simplified = converter
|
||||
return converter.convert(text)
|
||||
@@ -214,6 +214,13 @@ class DoctorRepository(Protocol):
|
||||
def cancel_diagnosis_ai(self, diagnosis_id: int, request_id: str) -> dict[str, Any]:
|
||||
"""Request server/upstream cancellation of one correlated stream."""
|
||||
|
||||
def stream_video_ai(
|
||||
self, diagnosis_id: int, *, call_record_id: int,
|
||||
transcription_session_id: str, task: str = "live_consultation",
|
||||
request_id: str, cancelled: Callable[[], bool] | None = None,
|
||||
) -> Iterator[dict[str, Any]]:
|
||||
"""Analyze the server-owned snapshot of this exact video call."""
|
||||
|
||||
def generate_ai_prescription(self, diagnosis_id: int) -> dict[str, Any]:
|
||||
"""Generate a server-validated prescription draft for doctor review."""
|
||||
|
||||
@@ -1478,6 +1485,42 @@ class RemoteDoctorRepository:
|
||||
yield {"event": "delta", "text": answer, "fallback": True}
|
||||
yield {**result, "event": "done", "fallback": True}
|
||||
|
||||
def stream_video_ai(
|
||||
self, diagnosis_id: int, *, call_record_id: int,
|
||||
transcription_session_id: str, task: str = "live_consultation",
|
||||
request_id: str, cancelled: Callable[[], bool] | None = None,
|
||||
) -> Iterator[dict[str, Any]]:
|
||||
"""No legacy fallback: an older route cannot bind a live call safely."""
|
||||
|
||||
if (isinstance(diagnosis_id, bool) or not isinstance(diagnosis_id, int)
|
||||
or diagnosis_id <= 0 or isinstance(call_record_id, bool)
|
||||
or not isinstance(call_record_id, int) or call_record_id <= 0):
|
||||
raise ValueError("diagnosis and call_record_id must be positive integers")
|
||||
if (not isinstance(transcription_session_id, str)
|
||||
or re.fullmatch(r"[A-Za-z0-9_-]{1,128}", transcription_session_id) is None):
|
||||
raise ValueError("invalid transcription session")
|
||||
if task not in {"live_consultation", "live_review"}:
|
||||
raise ValueError("unsupported video AI task")
|
||||
body = {
|
||||
"id": diagnosis_id, "task": task,
|
||||
"prompt": "请依据本次通话已保存的原文整理症状、追问和待核实信息,供医生复核。",
|
||||
"call_record_id": call_record_id,
|
||||
"transcription_session_id": transcription_session_id,
|
||||
"request_id": request_id,
|
||||
}
|
||||
for raw in self.client.post_event_stream(
|
||||
"tcm.diagnosis/aiAssistantStream", body, timeout=105.0, cancelled=cancelled,
|
||||
):
|
||||
if cancelled is not None and cancelled():
|
||||
return
|
||||
event = _normalise_diagnosis_ai_event(raw)
|
||||
if event is not None:
|
||||
yield event
|
||||
if event["event"] == "done":
|
||||
return
|
||||
if cancelled is None or not cancelled():
|
||||
raise ApiProtocolError("video AI stream ended before done")
|
||||
|
||||
def cancel_diagnosis_ai(self, diagnosis_id: int, request_id: str) -> dict[str, Any]:
|
||||
"""Use an independent short request, even while the stream is idle."""
|
||||
|
||||
@@ -3060,10 +3103,10 @@ def _normalise_diagnosis_ai_event(raw: Mapping[str, Any]) -> dict[str, Any] | No
|
||||
if not isinstance(text, str):
|
||||
raise ApiProtocolError("AI assistant delta content must be text", data=data)
|
||||
return {"event": "delta", "text": text}
|
||||
if kind == "done":
|
||||
if kind in {"live_update", "done"}:
|
||||
payload.pop("event", None)
|
||||
payload.pop("type", None)
|
||||
return {**payload, "event": "done"}
|
||||
return {**payload, "event": kind}
|
||||
if kind == "error":
|
||||
message = (
|
||||
data
|
||||
|
||||
@@ -48,6 +48,7 @@ from PySide6.QtWidgets import (
|
||||
)
|
||||
|
||||
from ...core.appointment_modes import appointment_type_description, appointment_type_value
|
||||
from ...core.transcript_text import simplified_transcript
|
||||
from ..design_system import clinical_stylesheet
|
||||
from ..diagnosis_drawer import (
|
||||
DIAGNOSIS_QSS,
|
||||
@@ -4033,7 +4034,8 @@ class DiagnosisDialog(QDialog):
|
||||
content = QPlainTextEdit(dialog)
|
||||
content.setObjectName("DiagnosisCallTranscriptText")
|
||||
content.setReadOnly(True)
|
||||
content.setPlainText(transcript)
|
||||
# Historical records keep their stored source; only the view is converted.
|
||||
content.setPlainText(simplified_transcript(transcript))
|
||||
layout.addWidget(content, 1)
|
||||
close_button = QPushButton("关闭")
|
||||
close_button.setProperty("variant", "primary")
|
||||
|
||||
@@ -18,6 +18,7 @@ from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from ..core.transcript_text import simplified_transcript
|
||||
from .launcher import VideoCallRequest, normalize_backend_ticket
|
||||
|
||||
ResultT = TypeVar("ResultT")
|
||||
@@ -134,7 +135,7 @@ def _extract_cloud_recording_outcome(result: Any) -> tuple[bool, str] | None:
|
||||
return None
|
||||
|
||||
|
||||
def _clean_transcript_segment(segment: Mapping[str, Any], session_id: str) -> dict[str, Any]:
|
||||
def normalize_transcript_segment(segment: Mapping[str, Any], session_id: str) -> dict[str, Any]:
|
||||
segment_id = str(segment.get("segment_id", segment.get("segmentId", ""))).strip()
|
||||
text = str(segment.get("text", segment.get("sourceText", ""))).strip()
|
||||
speaker_user_id = str(
|
||||
@@ -145,13 +146,14 @@ def _clean_transcript_segment(segment: Mapping[str, Any], session_id: str) -> di
|
||||
raise ValueError("transcript segment_id must contain 1 to 160 characters")
|
||||
if not text or len(text) > 4_000:
|
||||
raise ValueError("transcript text must contain 1 to 4000 characters")
|
||||
text = simplified_transcript(text)
|
||||
if len(speaker_user_id) > 160:
|
||||
raise ValueError("transcript speaker_user_id is too long")
|
||||
if speaker_role not in {"doctor", "patient", "unknown"}:
|
||||
speaker_role = "unknown"
|
||||
try:
|
||||
timestamp = int(segment.get("timestamp") or 0)
|
||||
except (TypeError, ValueError):
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
timestamp = 0
|
||||
return {
|
||||
"segment_id": segment_id,
|
||||
@@ -543,10 +545,25 @@ class OrderedCallLifecycle:
|
||||
with self._lock:
|
||||
if self._end_future is not None:
|
||||
raise RuntimeError("video call has already ended")
|
||||
if self._transcription_start_future is not None:
|
||||
if self._transcription_session_id != clean_session:
|
||||
raise RuntimeError("another transcription session already exists")
|
||||
return self._transcription_start_future
|
||||
if self._transcription_finish_future is not None:
|
||||
raise RuntimeError("call transcription has already stopped")
|
||||
if (
|
||||
self._transcription_session_id is not None
|
||||
and self._transcription_session_id != clean_session
|
||||
):
|
||||
raise RuntimeError("another transcription session already exists")
|
||||
previous = self._transcription_start_future
|
||||
if previous is not None:
|
||||
if not previous.done() or (
|
||||
not previous.cancelled()
|
||||
and previous.exception() is None
|
||||
and previous.result() is True
|
||||
):
|
||||
return previous
|
||||
# A completed failure must not poison this call's retry button.
|
||||
# Keep the exact session identity: the server may have accepted
|
||||
# an earlier request whose response never reached this client.
|
||||
self._transcription_active = False
|
||||
if self._start_future is None:
|
||||
self.start()
|
||||
method = getattr(self.repository, "start_call_transcription", None)
|
||||
@@ -593,7 +610,7 @@ class OrderedCallLifecycle:
|
||||
raise RuntimeError("call transcription has not started")
|
||||
if self._transcription_finish_future is not None:
|
||||
raise RuntimeError("call transcription has already stopped")
|
||||
cleaned = _clean_transcript_segment(segment, session_id)
|
||||
cleaned = normalize_transcript_segment(segment, session_id)
|
||||
segment_id = cleaned["segment_id"]
|
||||
existing = self._segment_futures.get(segment_id)
|
||||
if existing is not None:
|
||||
@@ -738,4 +755,4 @@ class OrderedCallLifecycle:
|
||||
return self._worker.wait(timeout)
|
||||
|
||||
|
||||
__all__ = ["OrderedCallLifecycle"]
|
||||
__all__ = ["OrderedCallLifecycle", "normalize_transcript_segment"]
|
||||
|
||||
@@ -0,0 +1,414 @@
|
||||
"""Incremental video assistance; never runs on the call persistence FIFO or Qt.
|
||||
|
||||
Only persisted segment notifications trigger work. One in-flight request and
|
||||
one coalesced latest revision are retained, with explicit patient/call/session
|
||||
correlation. Clinical content is never written or sent to the patient here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Callable, Mapping
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from threading import BoundedSemaphore, Event, RLock, Thread
|
||||
from time import monotonic
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from ..core.errors import (
|
||||
ApiBusinessError,
|
||||
ApiHttpError,
|
||||
ApiProtocolError,
|
||||
ApiTimeoutError,
|
||||
ApiTransportError,
|
||||
AuthenticationExpiredError,
|
||||
)
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
_SLOTS = BoundedSemaphore(2)
|
||||
_CANCEL_SLOTS = BoundedSemaphore(4)
|
||||
|
||||
|
||||
def _failure_details(error: Exception) -> tuple[str, str]:
|
||||
"""Translate stable error codes, never echo an upstream exception/response."""
|
||||
if isinstance(error, ApiTimeoutError):
|
||||
code = "UPSTREAM_TIMEOUT"
|
||||
elif isinstance(error, AuthenticationExpiredError):
|
||||
code = "AUTH_REQUIRED"
|
||||
elif isinstance(error, ApiHttpError):
|
||||
code = ({401: "AUTH_REQUIRED", 403: "PERMISSION_DENIED", 404: "SERVICE_UPDATE_REQUIRED",
|
||||
405: "SERVICE_UPDATE_REQUIRED", 429: "RATE_LIMITED"}.get(error.status_code)
|
||||
or "UPSTREAM_UNAVAILABLE")
|
||||
elif isinstance(error, ApiBusinessError) and isinstance(error.data, Mapping):
|
||||
code = error.data.get("error_code") or error.data.get("code")
|
||||
elif isinstance(error, ApiProtocolError):
|
||||
code = "STREAM_INCOMPLETE"
|
||||
elif isinstance(error, ApiTransportError):
|
||||
code = "NETWORK_ERROR"
|
||||
elif isinstance(error, ValueError):
|
||||
code = "LIVE_RESULT_INVALID"
|
||||
else:
|
||||
code = "UPSTREAM_UNAVAILABLE"
|
||||
messages = {
|
||||
"UPSTREAM_TIMEOUT": "分析服务响应超时,可点击重新生成。",
|
||||
"AUTH_REQUIRED": "登录状态已失效,请重新登录后使用 AI 辅助。",
|
||||
"PERMISSION_DENIED": "当前账号没有 AI 辅助权限,请联系管理员开通。",
|
||||
"SERVICE_UPDATE_REQUIRED": "服务端尚不支持实时 AI,请更新服务端后重试。",
|
||||
"RATE_LIMITED": "分析请求较多,请稍后重新生成。",
|
||||
"LIVE_RESULT_INVALID": "本次建议未通过格式或引用校验,可点击重新生成。",
|
||||
"LIVE_CONTEXT_UNAVAILABLE": "本次通话的分析资料尚未就绪,请等待转写保存后重试。",
|
||||
"STREAM_INCOMPLETE": "分析连接提前结束,可点击重新生成。",
|
||||
"NETWORK_ERROR": "分析服务连接中断,请检查网络后重试。",
|
||||
"UPSTREAM_UNAVAILABLE": "分析服务暂时不可用,请稍后重新生成。",
|
||||
}
|
||||
if not isinstance(code, str) or code not in messages:
|
||||
code = "UPSTREAM_UNAVAILABLE"
|
||||
if code == "LIVE_RESULT_INVALID" and isinstance(getattr(error, "data", None), Mapping):
|
||||
reason = error.data.get("live_validation_reason")
|
||||
details = {
|
||||
"JSON_INVALID": "分析服务返回的格式不完整,可点击重新生成。",
|
||||
"SCHEMA_INVALID": "分析服务返回的格式不符合要求,可点击重新生成。",
|
||||
"CITATION_INVALID": "部分建议的原文引用无法核对,可点击重新生成。",
|
||||
"SOURCE_QUOTE_MISMATCH": "部分建议与引用原文不一致,可点击重新生成。",
|
||||
"SAFETY_REJECTED": "本次建议包含超出辅助范围的内容,未通过校验,可重新生成。",
|
||||
}
|
||||
if isinstance(reason, str) and reason in details:
|
||||
return code, details[reason]
|
||||
return code, messages[code]
|
||||
|
||||
|
||||
def empty_video_ai_state(available: bool) -> dict[str, Any]:
|
||||
return {
|
||||
"enabled": available, "available": available, "busy": False,
|
||||
"phase": "waiting", "message": ("接通并产生转写文字后,自动整理本次对话。" if available
|
||||
else "当前账号或服务暂不能使用 AI 辅助;语音转写可独立运行。"),
|
||||
"error": "", "revision": "", "throughTimestampMs": 0, "segmentCount": 0,
|
||||
"transcriptionStatus": "idle", "summary": [], "questions": [],
|
||||
"checks": [], "review": [], "sources": [],
|
||||
"elapsedMs": 0, "lastDurationMs": 0, "firstResultMs": None,
|
||||
"pendingSegments": 0, "errorCode": "", "analysisScope": "",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Job:
|
||||
generation: int
|
||||
call_id: int
|
||||
session_id: str
|
||||
dirty: int
|
||||
task: str
|
||||
started_at: float = 0.0
|
||||
first_result_ms: int | None = None
|
||||
revision: str = ""
|
||||
analysis_scope: str | None = None
|
||||
request_id: str = field(default_factory=lambda: str(uuid4()))
|
||||
cancelled: Event = field(default_factory=Event)
|
||||
cancel_sent: bool = False
|
||||
|
||||
|
||||
class LiveAiController:
|
||||
def __init__(self, repository: Any, *, diagnosis_id: int, enabled: bool,
|
||||
clock: Callable[[], float] = monotonic, interval: float = 2.0) -> None:
|
||||
self.repository = repository
|
||||
self.diagnosis_id = diagnosis_id
|
||||
self.available = enabled is True and callable(getattr(repository, "stream_video_ai", None))
|
||||
self._clock = clock
|
||||
self._interval = max(0, interval)
|
||||
self._lock = RLock()
|
||||
self._state = empty_video_ai_state(self.available)
|
||||
self._mailbox: dict[str, Any] | None = copy.deepcopy(self._state)
|
||||
self._closed = False
|
||||
self._generation = 0
|
||||
self._call_id = 0
|
||||
self._session_id = ""
|
||||
self._dirty = 0
|
||||
self._attempted = 0
|
||||
self._displayed_dirty = 0
|
||||
self._due = 0.0
|
||||
self._forced_task = ""
|
||||
self._running: _Job | None = None
|
||||
self._worker: Thread | None = None
|
||||
|
||||
def _publish(self, **changes: Any) -> None:
|
||||
self._state.update(changes)
|
||||
self._state["pendingSegments"] = max(0, self._dirty - self._displayed_dirty)
|
||||
self._mailbox = copy.deepcopy(self._state)
|
||||
|
||||
def take_state(self) -> dict[str, Any] | None:
|
||||
with self._lock:
|
||||
result, self._mailbox = self._mailbox, None
|
||||
return None if self._closed else result
|
||||
|
||||
def bind(self, call_id: int, session_id: str) -> None:
|
||||
with self._lock:
|
||||
if self._closed or (self._call_id == call_id and self._session_id == session_id):
|
||||
return
|
||||
self._cancel()
|
||||
enabled = self._state["enabled"]
|
||||
self._generation += 1
|
||||
self._call_id, self._session_id = call_id, session_id
|
||||
self._dirty = self._attempted = 0
|
||||
self._displayed_dirty = 0
|
||||
self._forced_task = ""
|
||||
self._state = empty_video_ai_state(self.available)
|
||||
self._publish(enabled=enabled, transcriptionStatus="running" if session_id else "idle")
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear the previous call before a new call's model/storage is ready."""
|
||||
self.bind(0, "")
|
||||
|
||||
def changed(self, session_id: str) -> None:
|
||||
with self._lock:
|
||||
if self._closed or session_id != self._session_id:
|
||||
return
|
||||
was_clean = self._dirty == self._attempted
|
||||
self._dirty += 1
|
||||
if was_clean:
|
||||
# Coalesce a short speech burst without repeatedly moving the deadline.
|
||||
self._due = self._clock() + (0.8 if not self._state["revision"] else self._interval)
|
||||
self._publish()
|
||||
|
||||
def transcription_error(self, message: str, *, status: str = "partial") -> None:
|
||||
with self._lock:
|
||||
if not self._closed:
|
||||
self._publish(transcriptionStatus=status, error=message[:240])
|
||||
|
||||
def transcription_starting(self) -> None:
|
||||
with self._lock:
|
||||
if not self._closed:
|
||||
self._publish(transcriptionStatus="starting", error="")
|
||||
|
||||
def transcription_running(self, session_id: str) -> None:
|
||||
"""A retried cloud recognizer is ready without resetting saved evidence."""
|
||||
with self._lock:
|
||||
if not self._closed and session_id == self._session_id:
|
||||
self._publish(transcriptionStatus="running", error="")
|
||||
|
||||
def finish(self, session_id: str, status: str) -> None:
|
||||
with self._lock:
|
||||
if not self._closed and session_id == self._session_id:
|
||||
self._publish(transcriptionStatus=status)
|
||||
|
||||
def handle(self, action: str, enabled: Any = None) -> None:
|
||||
with self._lock:
|
||||
if self._closed or not self.available:
|
||||
return
|
||||
if action == "toggle" and isinstance(enabled, bool):
|
||||
if not enabled:
|
||||
self._cancel()
|
||||
self._publish(enabled=enabled, busy=False if not enabled else self._state["busy"],
|
||||
message="已暂停自动分析。" if not enabled else "有新增对话后自动更新。")
|
||||
return
|
||||
if action not in {"refresh", "review"} or not self._dirty or self._running is not None:
|
||||
return
|
||||
if action == "review" and self._state["transcriptionStatus"] not in {"completed", "partial"}:
|
||||
return
|
||||
self._forced_task = "live_review" if action == "review" else "live_consultation"
|
||||
self._due = self._clock()
|
||||
|
||||
def tick(self) -> None:
|
||||
with self._lock:
|
||||
if self._running is not None and self._current(self._running):
|
||||
elapsed = self._elapsed(self._running)
|
||||
if elapsed // 1000 != self._state["elapsedMs"] // 1000:
|
||||
self._publish(elapsedMs=elapsed)
|
||||
if (self._closed or not self.available or not self._call_id or not self._session_id
|
||||
or self._running is not None or not self._dirty
|
||||
or (not self._forced_task and (not self._state["enabled"] or self._dirty <= self._attempted))
|
||||
or self._clock() < self._due):
|
||||
return
|
||||
if not _SLOTS.acquire(blocking=False):
|
||||
return
|
||||
job = _Job(self._generation, self._call_id, self._session_id, self._dirty,
|
||||
self._forced_task or "live_consultation", started_at=self._clock())
|
||||
self._forced_task = ""
|
||||
self._attempted = self._dirty
|
||||
self._running = job
|
||||
self._publish(busy=True, phase="analyzing", error="", errorCode="", elapsedMs=0, firstResultMs=None,
|
||||
message="正在分析最新对话,优先整理下一步追问…")
|
||||
self._worker = Thread(target=self._run, args=(job,), name="video-ai", daemon=True)
|
||||
try:
|
||||
self._worker.start()
|
||||
except RuntimeError:
|
||||
self._running = None
|
||||
_SLOTS.release()
|
||||
self._publish(busy=False, phase="error", error="分析任务无法启动,请重试。")
|
||||
|
||||
def _elapsed(self, job: _Job) -> int:
|
||||
return max(0, int((self._clock() - job.started_at) * 1000))
|
||||
|
||||
def _current(self, job: _Job) -> bool:
|
||||
return not self._closed and job.generation == self._generation and not job.cancelled.is_set()
|
||||
|
||||
def _validated_result(self, job: _Job, event: Mapping[str, Any]) -> dict[str, Any]:
|
||||
context = event.get("live_context")
|
||||
result = event.get("live_assistance")
|
||||
# Only label the scope of validated displayed content. Older servers do
|
||||
# not report it; never infer dialogue-only behavior from the task name.
|
||||
scope = event.get("analysis_scope", "")
|
||||
if not isinstance(scope, str) or scope not in {"", "current_call_only", "authorized_case"}:
|
||||
raise ValueError("invalid analysis scope")
|
||||
if (not isinstance(context, Mapping) or not isinstance(result, Mapping)
|
||||
or event.get("diagnosis_id") != self.diagnosis_id
|
||||
or context.get("call_record_id") != job.call_id
|
||||
or context.get("transcription_session_id") != job.session_id
|
||||
or not isinstance(context.get("revision"), str)
|
||||
or not re.fullmatch(r"[a-f0-9]{64}", context["revision"])):
|
||||
raise ValueError("context mismatch")
|
||||
sources = context.get("sources")
|
||||
if not isinstance(sources, list) or len(sources) > 1000:
|
||||
raise ValueError("invalid evidence")
|
||||
known: set[str] = set()
|
||||
clean_sources = []
|
||||
for source in sources:
|
||||
if (not isinstance(source, Mapping) or not isinstance(source.get("id"), str)
|
||||
or not source["id"] or source["id"] in known
|
||||
or not isinstance(source.get("text"), str) or not source["text"].strip()
|
||||
or len(source["text"]) > 4000
|
||||
or source.get("role") not in {"doctor", "patient", "unknown"}
|
||||
or isinstance(source.get("timestamp_ms"), bool)
|
||||
or not isinstance(source.get("timestamp_ms"), int) or source["timestamp_ms"] < 0):
|
||||
raise ValueError("invalid source")
|
||||
known.add(source["id"])
|
||||
clean_sources.append({key: source.get(key) for key in ("id", "text", "role", "timestamp_ms")})
|
||||
cleaned: dict[str, Any] = {}
|
||||
for key in ("summary", "questions", "checks", "review"):
|
||||
items = result.get(key, [])
|
||||
if not isinstance(items, list) or len(items) > 6:
|
||||
raise ValueError("invalid items")
|
||||
cleaned[key] = []
|
||||
for item in items:
|
||||
if (not isinstance(item, Mapping) or not isinstance(item.get("text"), str)
|
||||
or not item["text"].strip() or len(item["text"]) > (4020 if key == "summary" else 600)
|
||||
or not isinstance(item.get("evidence_ids"), list)
|
||||
or not 1 <= len(item["evidence_ids"]) <= 8
|
||||
or any(not isinstance(s, str) or s not in known for s in item["evidence_ids"])
|
||||
or len(set(item["evidence_ids"])) != len(item["evidence_ids"])):
|
||||
raise ValueError("invalid cited item")
|
||||
reason = item.get("reason", "")
|
||||
if not isinstance(reason, str) or len(reason) > 600:
|
||||
raise ValueError("invalid reason")
|
||||
cleaned[key].append({"text": item["text"], "reason": reason,
|
||||
"evidence_ids": list(item["evidence_ids"])})
|
||||
count = context.get("segment_count")
|
||||
through = context.get("through_timestamp_ms")
|
||||
if (isinstance(count, bool) or not isinstance(count, int) or count < 1
|
||||
or isinstance(through, bool) or not isinstance(through, (int, float))
|
||||
or not math.isfinite(through) or through < 0
|
||||
or context.get("transcription_status") not in {"running", "completed", "partial"}):
|
||||
raise ValueError("invalid revision metadata")
|
||||
return {**cleaned, "sources": clean_sources, "revision": context["revision"], "analysisScope": scope,
|
||||
"segmentCount": count, "throughTimestampMs": through,
|
||||
"transcriptionStatus": context.get("transcription_status", "partial")}
|
||||
|
||||
def _run(self, job: _Job) -> None:
|
||||
events = None
|
||||
success = False
|
||||
try:
|
||||
if job.cancelled.is_set():
|
||||
return
|
||||
events = self.repository.stream_video_ai(
|
||||
self.diagnosis_id, call_record_id=job.call_id, transcription_session_id=job.session_id,
|
||||
task=job.task, request_id=job.request_id, cancelled=job.cancelled.is_set,
|
||||
)
|
||||
for event in events:
|
||||
with self._lock:
|
||||
if not self._current(job):
|
||||
return
|
||||
if event.get("event") in {"start", "progress"}:
|
||||
self._publish(message=str(event.get("message") or "正在分析本次对话…")[:200])
|
||||
elif event.get("event") in {"live_update", "done"}:
|
||||
result = self._validated_result(job, event)
|
||||
if job.revision and job.revision != result["revision"]:
|
||||
raise ValueError("revision changed inside a single snapshot")
|
||||
if job.analysis_scope is not None and job.analysis_scope != result["analysisScope"]:
|
||||
raise ValueError("analysis scope changed inside a single snapshot")
|
||||
job.revision = result["revision"]
|
||||
job.analysis_scope = result["analysisScope"]
|
||||
# Preserve a newer native completion status while an older
|
||||
# in-flight snapshot still reports running.
|
||||
if self._state["transcriptionStatus"] in {"completed", "partial", "failed"}:
|
||||
result["transcriptionStatus"] = self._state["transcriptionStatus"]
|
||||
complete = event.get("event") == "done"
|
||||
duration = self._elapsed(job)
|
||||
if job.first_result_ms is None and any(
|
||||
result[key] for key in ("summary", "questions", "checks", "review")
|
||||
):
|
||||
job.first_result_ms = duration
|
||||
self._displayed_dirty = job.dirty
|
||||
self._publish(**result, busy=not complete,
|
||||
phase="ready" if complete else "analyzing", error="", errorCode="",
|
||||
elapsedMs=duration,
|
||||
firstResultMs=job.first_result_ms,
|
||||
lastDurationMs=duration if complete else self._state["lastDurationMs"],
|
||||
message="已更新,请对照原话复核。" if complete
|
||||
else "已显示可核对的建议,其余内容仍在整理。")
|
||||
if complete:
|
||||
success = True
|
||||
return
|
||||
if not job.cancelled.is_set():
|
||||
raise ApiProtocolError("incomplete stream")
|
||||
except Exception as error:
|
||||
with self._lock:
|
||||
if self._current(job):
|
||||
code, message = _failure_details(error)
|
||||
self._publish(busy=False, phase="error", error=message, errorCode=code,
|
||||
elapsedMs=self._elapsed(job), lastDurationMs=self._elapsed(job),
|
||||
message=("已显示的建议仍可核对,本次分析尚未全部完成。"
|
||||
if job.first_result_ms is not None else
|
||||
"保留上次结果;新内容尚未分析完成。" if any(
|
||||
self._state[key] for key in ("summary", "questions", "checks", "review")
|
||||
) else
|
||||
"本次暂未生成可用建议,可点击重新生成。"))
|
||||
# Log categories/timing only, never response text or patient details.
|
||||
_LOGGER.warning("video_ai_failed code=%s type=%s duration_ms=%d",
|
||||
code, type(error).__name__, self._elapsed(job))
|
||||
finally:
|
||||
if events is not None and callable(getattr(events, "close", None)):
|
||||
with suppress(Exception):
|
||||
events.close()
|
||||
with self._lock:
|
||||
if not success:
|
||||
self._send_cancel(job)
|
||||
if self._running is job:
|
||||
self._running = None
|
||||
_SLOTS.release()
|
||||
if success:
|
||||
_LOGGER.info("video_ai_completed duration_ms=%d first_result_ms=%s",
|
||||
self._elapsed(job), job.first_result_ms)
|
||||
|
||||
def _cancel(self) -> None:
|
||||
if self._running is not None:
|
||||
self._running.cancelled.set()
|
||||
self._send_cancel(self._running)
|
||||
|
||||
def _send_cancel(self, job: _Job) -> None:
|
||||
if job.cancel_sent or not _CANCEL_SLOTS.acquire(blocking=False):
|
||||
return
|
||||
job.cancel_sent = True
|
||||
|
||||
def cancel() -> None:
|
||||
try:
|
||||
self.repository.cancel_diagnosis_ai(self.diagnosis_id, job.request_id)
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
_CANCEL_SLOTS.release()
|
||||
try:
|
||||
Thread(target=cancel, name="video-ai-cancel", daemon=True).start()
|
||||
except RuntimeError:
|
||||
_CANCEL_SLOTS.release()
|
||||
|
||||
def close(self) -> None:
|
||||
with self._lock:
|
||||
self._closed = True
|
||||
self._cancel()
|
||||
self._mailbox = None
|
||||
|
||||
def wait(self, timeout: float = 1.0) -> bool:
|
||||
if self._worker is not None:
|
||||
self._worker.join(max(0, min(timeout, 5)))
|
||||
return self._worker is None or not self._worker.is_alive()
|
||||
@@ -0,0 +1,446 @@
|
||||
"""Bounded, in-memory speech recognition using an explicitly installed local model.
|
||||
|
||||
This module has no dependency on Qt, the recording queue, or the call lifecycle.
|
||||
Callbacks run on one daemon worker and must only enqueue work (for example, emit
|
||||
a Qt signal). Runtime code never downloads a model or sends audio to a service.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
from collections import deque
|
||||
from collections.abc import Callable
|
||||
from contextlib import suppress
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from platformdirs import user_data_path
|
||||
|
||||
from ..core.transcript_text import simplified_transcript
|
||||
|
||||
SAMPLE_RATE = 16_000
|
||||
MAX_CHUNK_SECONDS = 12
|
||||
MAX_PCM_BYTES = SAMPLE_RATE * 2 * MAX_CHUNK_SECONDS
|
||||
MAX_BASE64_CHARS = ((MAX_PCM_BYTES + 2) // 3) * 4
|
||||
MAX_SEQUENCE = 2_147_483_647
|
||||
MAX_TIMESTAMP_MS = 253_402_300_799_999
|
||||
MAX_TRANSCRIPT_CHARS = 4_000
|
||||
_SESSION_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}\Z")
|
||||
_REQUIRED_FILES = ("model.bin", "config.json", "tokenizer.json")
|
||||
_DEPENDENCIES = ("faster_whisper", "numpy", "ctranslate2", "onnxruntime", "opencc")
|
||||
_MESSAGES = {
|
||||
"model_missing": "本地语音模型未安装或文件不完整,请先运行本地转写安装工具。",
|
||||
"dependency_missing": "本地转写运行库未安装,请先安装本地转写依赖。",
|
||||
"dependency_unavailable": "本地转写运行库无法加载,请检查安装包及系统运行库。",
|
||||
"model_load_failed": "本地语音模型加载失败,请检查模型文件和本机运行环境。",
|
||||
"conversion_unavailable": "简体转换字典无法加载,请修复本地转写安装。",
|
||||
"invalid_audio": "本地转写音频格式无效,需要最长 12 秒的单声道 16 kHz PCM16 音频。",
|
||||
"invalid_metadata": "本地转写音频的角色、序号或时间戳无效。",
|
||||
"sequence_out_of_order": "本地转写音频序号重复或倒序,该片段未被接受。",
|
||||
"sequence_gap": "本地转写检测到音频片段缺失,转写结果可能不完整。",
|
||||
"overloaded": "本地转写处理速度不足,部分音频片段未转写;通话和录音继续。",
|
||||
"transcription_failed": "本地音频片段转写失败,结果可能不完整;通话和录音继续。",
|
||||
}
|
||||
|
||||
|
||||
def default_model_path() -> Path:
|
||||
"""Return the model directory without creating it or accessing the network."""
|
||||
override = os.getenv("DOCTOR_ASR_MODEL_PATH", "").strip()
|
||||
if override:
|
||||
return Path(override).expanduser().resolve()
|
||||
bundle = Path(getattr(sys, "_MEIPASS", Path(__file__).resolve().parents[3]))
|
||||
bundled_model = bundle / "resources" / "local-asr" / "model"
|
||||
if bundled_model.is_dir():
|
||||
return bundled_model
|
||||
return (
|
||||
user_data_path("ZhenyangDoctorWorkstation", appauthor="ZhenYangTang")
|
||||
/ "local-asr"
|
||||
/ "model"
|
||||
)
|
||||
|
||||
|
||||
def _missing_model_files(path: Path) -> list[str]:
|
||||
def exists(name: str) -> bool:
|
||||
try:
|
||||
file = path / name
|
||||
return file.is_file() and file.stat().st_size > 0
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
missing = [name for name in _REQUIRED_FILES if not exists(name)]
|
||||
if not any(exists(name) for name in ("vocabulary.txt", "vocabulary.json")):
|
||||
missing.append("vocabulary.txt or vocabulary.json")
|
||||
return missing
|
||||
|
||||
|
||||
def local_transcription_readiness(model_path: Path | str | None = None) -> dict[str, Any]:
|
||||
"""Cheap installation preflight; ``available`` does not claim a model has loaded."""
|
||||
path = Path(model_path).expanduser().resolve() if model_path else default_model_path()
|
||||
missing_dependencies = []
|
||||
for name in _DEPENDENCIES:
|
||||
try:
|
||||
available = importlib.util.find_spec(name) is not None
|
||||
except (ImportError, ValueError):
|
||||
available = False
|
||||
if not available:
|
||||
missing_dependencies.append(name)
|
||||
missing_files = _missing_model_files(path)
|
||||
code = (
|
||||
"model_missing" if missing_files else "dependency_missing" if missing_dependencies else ""
|
||||
)
|
||||
return {
|
||||
"available": not (missing_files or missing_dependencies),
|
||||
"modelPath": str(path),
|
||||
"missingDependencies": missing_dependencies,
|
||||
"missingFiles": missing_files,
|
||||
"code": code,
|
||||
"message": _MESSAGES.get(code, "本地转写文件检查通过,启动后将加载模型。"),
|
||||
}
|
||||
|
||||
|
||||
class LocalTranscriptionError(RuntimeError):
|
||||
"""A fixed, patient-data-free error suitable for displaying in the UI."""
|
||||
|
||||
def __init__(self, code: str) -> None:
|
||||
self.code = code if code in _MESSAGES else "transcription_failed"
|
||||
super().__init__(_MESSAGES[self.code])
|
||||
|
||||
|
||||
class SpeechEngine(Protocol):
|
||||
def transcribe(self, pcm: bytes) -> str:
|
||||
"""Return verbatim recognized text for mono, little-endian PCM16 at 16 kHz."""
|
||||
|
||||
|
||||
class FasterWhisperEngine:
|
||||
"""CPU/int8 recognition. Construct only on a background worker or setup CLI."""
|
||||
|
||||
def __init__(self, model_path: Path | str | None = None) -> None:
|
||||
path = Path(model_path).expanduser().resolve() if model_path else default_model_path()
|
||||
# In particular, require tokenizer.json: upstream may otherwise fetch a
|
||||
# tokenizer even when WhisperModel(local_files_only=True) is requested.
|
||||
if _missing_model_files(path):
|
||||
raise LocalTranscriptionError("model_missing")
|
||||
try:
|
||||
import numpy as np
|
||||
from faster_whisper import WhisperModel
|
||||
from faster_whisper.vad import get_vad_model
|
||||
except ModuleNotFoundError:
|
||||
raise LocalTranscriptionError("dependency_missing") from None
|
||||
except (ImportError, OSError):
|
||||
raise LocalTranscriptionError("dependency_unavailable") from None
|
||||
self._np = np
|
||||
try:
|
||||
# Load local dictionaries before ready, so a broken/frozen install
|
||||
# cannot claim successful ASR and then fail on its first sentence.
|
||||
simplified_transcript("")
|
||||
except ImportError:
|
||||
raise LocalTranscriptionError("dependency_missing") from None
|
||||
except Exception:
|
||||
raise LocalTranscriptionError("conversion_unavailable") from None
|
||||
try:
|
||||
self._model = WhisperModel(
|
||||
str(path),
|
||||
device="cpu",
|
||||
compute_type="int8",
|
||||
cpu_threads=min(4, max(1, (os.cpu_count() or 2) // 2)),
|
||||
num_workers=1,
|
||||
local_files_only=True,
|
||||
)
|
||||
# Silero's ONNX asset ships inside faster-whisper; preflight it so
|
||||
# a packaged application cannot report ready with a missing VAD.
|
||||
get_vad_model()
|
||||
except Exception:
|
||||
raise LocalTranscriptionError("model_load_failed") from None
|
||||
|
||||
def transcribe(self, pcm: bytes) -> str:
|
||||
if not pcm or len(pcm) % 2 or len(pcm) > MAX_PCM_BYTES:
|
||||
raise LocalTranscriptionError("invalid_audio")
|
||||
if not any(pcm):
|
||||
return ""
|
||||
audio = self._np.frombuffer(pcm, dtype="<i2").astype(self._np.float32) / 32768.0
|
||||
segments, _ = self._model.transcribe(
|
||||
audio,
|
||||
language="zh",
|
||||
task="transcribe",
|
||||
beam_size=5,
|
||||
temperature=0.0,
|
||||
condition_on_previous_text=False,
|
||||
vad_filter=True,
|
||||
vad_parameters={
|
||||
"min_speech_duration_ms": 0,
|
||||
"min_silence_duration_ms": 500,
|
||||
"speech_pad_ms": 400,
|
||||
},
|
||||
)
|
||||
# Iterating is essential: faster-whisper returns a lazy generator. Do
|
||||
# not rewrite numbers, units, negations, repetitions, or short answers.
|
||||
return "".join(segment.text for segment in segments).strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Chunk:
|
||||
role: str
|
||||
sequence: int
|
||||
timestamp_ms: int
|
||||
pcm: bytes
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _Session:
|
||||
id: str
|
||||
initialized: bool = False
|
||||
stopping: bool = False
|
||||
stopped: bool = False
|
||||
partial: bool = False
|
||||
last_sequence: dict[str, int] = field(default_factory=dict)
|
||||
chunks: deque[_Chunk] = field(default_factory=deque)
|
||||
errors: deque[str] = field(default_factory=deque)
|
||||
reported_errors: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
class LocalTranscriptionController:
|
||||
"""One bounded daemon worker, isolated by session object identity.
|
||||
|
||||
``start`` accepts a new session, replacing queued old work. ``submit`` only
|
||||
acknowledges enqueueing, while ``ready`` confirms successful model loading.
|
||||
``stop`` rejects further submissions and drains accepted chunks in order.
|
||||
``close`` cancels immediately without waiting for native inference. No
|
||||
callbacks from an old generation occur after start/close returns.
|
||||
|
||||
The callback must be quick, thread-safe, and must not join/wait for another
|
||||
thread that is calling this controller. It is invoked under a reentrant
|
||||
lock, which makes close/session replacement atomic with callback delivery.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
on_event: Callable[[dict[str, Any]], None],
|
||||
engine_factory: Callable[[], SpeechEngine] | None = None,
|
||||
*,
|
||||
max_pending_chunks: int = 12,
|
||||
) -> None:
|
||||
if type(max_pending_chunks) is not int or not 1 <= max_pending_chunks <= 64:
|
||||
raise ValueError("max_pending_chunks must be an integer between 1 and 64")
|
||||
self._on_event = on_event
|
||||
self._engine_factory = engine_factory or FasterWhisperEngine
|
||||
self._max_pending_chunks = max_pending_chunks
|
||||
self._condition = threading.Condition(threading.RLock())
|
||||
self._session: _Session | None = None
|
||||
self._engine: SpeechEngine | None = None
|
||||
self._closed = False
|
||||
self._worker: threading.Thread | None = None
|
||||
|
||||
def start(self, session_id: str) -> bool:
|
||||
if not isinstance(session_id, str) or not _SESSION_ID.fullmatch(session_id):
|
||||
return False
|
||||
with self._condition:
|
||||
if self._closed:
|
||||
return False
|
||||
if self._session and self._session.id == session_id:
|
||||
return not (self._session.stopping or self._session.stopped)
|
||||
if self._session:
|
||||
self._session.chunks.clear()
|
||||
self._session.errors.clear()
|
||||
self._session = _Session(session_id)
|
||||
if self._worker is None:
|
||||
self._worker = threading.Thread(
|
||||
target=self._run, name="doctor-local-asr", daemon=True
|
||||
)
|
||||
self._worker.start()
|
||||
self._condition.notify_all()
|
||||
return True
|
||||
|
||||
def submit(
|
||||
self,
|
||||
session_id: str,
|
||||
role: str,
|
||||
sequence: int,
|
||||
timestamp_ms: int,
|
||||
pcm_base64: str,
|
||||
) -> bool:
|
||||
with self._condition:
|
||||
session = self._session
|
||||
if (
|
||||
self._closed
|
||||
or session is None
|
||||
or session.id != session_id
|
||||
or session.stopping
|
||||
or session.stopped
|
||||
):
|
||||
return False
|
||||
if (
|
||||
role not in ("doctor", "patient")
|
||||
or type(sequence) is not int
|
||||
or not 0 <= sequence <= MAX_SEQUENCE
|
||||
or type(timestamp_ms) is not int
|
||||
or not 0 < timestamp_ms <= MAX_TIMESTAMP_MS
|
||||
):
|
||||
self._error_locked(session, "invalid_metadata")
|
||||
return False
|
||||
if (
|
||||
not isinstance(pcm_base64, str)
|
||||
or not pcm_base64
|
||||
or len(pcm_base64) > MAX_BASE64_CHARS
|
||||
):
|
||||
self._error_locked(session, "invalid_audio")
|
||||
return False
|
||||
try:
|
||||
pcm = base64.b64decode(pcm_base64, validate=True)
|
||||
except (ValueError, binascii.Error):
|
||||
self._error_locked(session, "invalid_audio")
|
||||
return False
|
||||
if not pcm or len(pcm) % 2 or len(pcm) > MAX_PCM_BYTES:
|
||||
self._error_locked(session, "invalid_audio")
|
||||
return False
|
||||
previous = session.last_sequence.get(role)
|
||||
if previous is not None and sequence <= previous:
|
||||
self._error_locked(session, "sequence_out_of_order")
|
||||
return False
|
||||
if previous is not None and sequence != previous + 1:
|
||||
self._error_locked(session, "sequence_gap")
|
||||
session.last_sequence[role] = sequence
|
||||
if len(session.chunks) >= self._max_pending_chunks:
|
||||
self._error_locked(session, "overloaded")
|
||||
return False
|
||||
session.chunks.append(_Chunk(role, sequence, timestamp_ms, pcm))
|
||||
self._condition.notify_all()
|
||||
return True
|
||||
|
||||
def stop(self, session_id: str) -> bool:
|
||||
with self._condition:
|
||||
session = self._session
|
||||
if self._closed or session is None or session.id != session_id:
|
||||
return False
|
||||
session.stopping = True
|
||||
self._condition.notify_all()
|
||||
return True
|
||||
|
||||
def close(self) -> None:
|
||||
with self._condition:
|
||||
self._closed = True
|
||||
if self._session:
|
||||
self._session.chunks.clear()
|
||||
self._session.errors.clear()
|
||||
self._session = None
|
||||
self._condition.notify_all()
|
||||
|
||||
def _error_locked(self, session: _Session, code: str) -> None:
|
||||
session.partial = True
|
||||
# At most one pending/report per fixed error code, even when the browser
|
||||
# continues submitting into an overloaded queue. No unbounded UI queue.
|
||||
if code not in session.reported_errors:
|
||||
session.reported_errors.add(code)
|
||||
session.errors.append(code)
|
||||
self._condition.notify_all()
|
||||
|
||||
def _emit(self, session: _Session, event: dict[str, Any]) -> None:
|
||||
with self._condition:
|
||||
if self._closed or self._session is not session:
|
||||
return
|
||||
# A detached UI must not kill the worker or expose transcript
|
||||
# contents through exception logging.
|
||||
with suppress(Exception):
|
||||
self._on_event({"sessionId": session.id, **event})
|
||||
|
||||
def _initialize(self, session: _Session) -> None:
|
||||
try:
|
||||
if self._engine is None:
|
||||
self._engine = self._engine_factory()
|
||||
except Exception as error:
|
||||
code = error.code if isinstance(error, LocalTranscriptionError) else "model_load_failed"
|
||||
with self._condition:
|
||||
if self._closed or self._session is not session:
|
||||
return
|
||||
session.initialized = True
|
||||
session.stopping = True
|
||||
session.chunks.clear()
|
||||
self._error_locked(session, code)
|
||||
return
|
||||
with self._condition:
|
||||
if self._closed or self._session is not session:
|
||||
return
|
||||
session.initialized = True
|
||||
if not session.stopping:
|
||||
self._emit(session, {"type": "ready"})
|
||||
|
||||
def _transcribe(self, session: _Session, chunk: _Chunk) -> None:
|
||||
try:
|
||||
# Reject only exact digital silence here. VAD in the real engine
|
||||
# handles room noise without an amplitude cutoff that loses quiet
|
||||
# numerals or one-word negations.
|
||||
if not any(chunk.pcm):
|
||||
return
|
||||
assert self._engine is not None
|
||||
text = self._engine.transcribe(chunk.pcm)
|
||||
if not isinstance(text, str) or len(text) > MAX_TRANSCRIPT_CHARS:
|
||||
raise LocalTranscriptionError("transcription_failed")
|
||||
# Normalize before persistence and captions, so AI evidence and the
|
||||
# doctor's visible transcript refer to the exact same text.
|
||||
text = simplified_transcript(text.strip())
|
||||
if text:
|
||||
self._emit(
|
||||
session,
|
||||
{
|
||||
"type": "segment",
|
||||
"segmentId": f"{session.id}:{chunk.role}:{chunk.sequence}",
|
||||
"role": chunk.role,
|
||||
"text": text,
|
||||
"timestamp": chunk.timestamp_ms,
|
||||
"isCompleted": True,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
with self._condition:
|
||||
if not self._closed and self._session is session:
|
||||
self._error_locked(session, "transcription_failed")
|
||||
|
||||
def _run(self) -> None:
|
||||
try:
|
||||
while True:
|
||||
with self._condition:
|
||||
while True:
|
||||
if self._closed:
|
||||
return
|
||||
session = self._session
|
||||
if session is not None and not session.stopped:
|
||||
if session.errors:
|
||||
action, payload = "error", session.errors.popleft()
|
||||
break
|
||||
if not session.initialized:
|
||||
action, payload = "initialize", None
|
||||
break
|
||||
if session.chunks:
|
||||
action, payload = "chunk", session.chunks.popleft()
|
||||
break
|
||||
if session.stopping:
|
||||
session.stopped = True
|
||||
action, payload = "stopped", None
|
||||
break
|
||||
self._condition.wait()
|
||||
if action == "initialize":
|
||||
self._initialize(session)
|
||||
elif action == "chunk":
|
||||
self._transcribe(session, payload)
|
||||
payload = None # Release raw PCM before waiting for work.
|
||||
elif action == "error":
|
||||
self._emit(
|
||||
session,
|
||||
{"type": "error", "code": payload, "message": _MESSAGES[payload]},
|
||||
)
|
||||
else:
|
||||
self._emit(
|
||||
session,
|
||||
{
|
||||
"type": "stopped",
|
||||
"status": "partial" if session.partial else "completed",
|
||||
},
|
||||
)
|
||||
finally:
|
||||
self._engine = None
|
||||
@@ -11,6 +11,7 @@ import base64
|
||||
import binascii
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
@@ -34,7 +35,8 @@ from .launcher import (
|
||||
VideoTicketError,
|
||||
require_supported_backend,
|
||||
)
|
||||
from .lifecycle import OrderedCallLifecycle
|
||||
from .lifecycle import OrderedCallLifecycle, normalize_transcript_segment
|
||||
from .live_ai import LiveAiController
|
||||
from .screenshot import ScreenshotPreviewSize, capture_view_region
|
||||
from .security import TrustedDocumentError, TrustedDocumentPolicy
|
||||
|
||||
@@ -296,6 +298,16 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
}
|
||||
)
|
||||
|
||||
@Slot(str, str, int, float, str) # type: ignore[misc]
|
||||
def appendLocalTranscriptionChunk( # noqa: N802
|
||||
self, session_id: str, role: str, sequence: int, timestamp: float, encoded: str
|
||||
) -> None:
|
||||
if len(encoded) > 512_000:
|
||||
return
|
||||
self._callback({"source": "doctor-call", "event": "local-transcription-chunk",
|
||||
"sessionId": session_id, "role": role, "sequence": sequence,
|
||||
"timestamp": timestamp, "data": encoded})
|
||||
|
||||
class _EmbeddedVideoWindow(QMainWindow): # type: ignore[misc, valid-type]
|
||||
status_changed = Signal(str) # type: ignore[misc]
|
||||
call_ended = Signal(str) # type: ignore[misc]
|
||||
@@ -304,7 +316,8 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
_policy_completed = Signal(str, object) # type: ignore[misc]
|
||||
_room_completed = Signal(str, bool, str) # type: ignore[misc]
|
||||
_screenshot_completed = Signal(bool, str) # type: ignore[misc]
|
||||
_transcription_completed = Signal(str, str, str, bool, str) # type: ignore[misc]
|
||||
_transcription_completed = Signal(str, str, str, bool, str, str) # type: ignore[misc]
|
||||
_local_transcription_event = Signal(object) # type: ignore[misc]
|
||||
_local_recording_completed = Signal( # type: ignore[misc]
|
||||
str, str, int, bool, str
|
||||
)
|
||||
@@ -364,6 +377,20 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
self._im_ai_timer.timeout.connect(self._flush_im_ai)
|
||||
if self._im_ai.enabled:
|
||||
self._im_ai_timer.start()
|
||||
self._live_ai = LiveAiController(
|
||||
repository, diagnosis_id=request.diagnosis_id, enabled=can_ai_assistant is True,
|
||||
)
|
||||
# The default recognizer is Tencent's existing call transcriber.
|
||||
# Keep local audio recording independent; no local model is loaded.
|
||||
self._local_transcription = None
|
||||
self._asr_session_id = ""
|
||||
self._asr_ready = False
|
||||
self._asr_partial = False
|
||||
self._transcription_final_status = "idle"
|
||||
self._video_ai_timer = QTimer(self)
|
||||
self._video_ai_timer.setInterval(100)
|
||||
self._video_ai_timer.timeout.connect(self._flush_video_ai)
|
||||
self._video_ai_timer.start()
|
||||
self._screenshot_preview_size = ScreenshotPreviewSize()
|
||||
self._local_audio_capture: _LocalAudioCapture | None = None
|
||||
self._local_audio_store: LocalAudioQueueStore | None = None
|
||||
@@ -414,6 +441,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
self._room_completed.connect(self._on_room_completed)
|
||||
self._screenshot_completed.connect(self._on_screenshot_completed)
|
||||
self._transcription_completed.connect(self._on_transcription_completed)
|
||||
self._local_transcription_event.connect(self._on_local_transcription)
|
||||
self._local_recording_completed.connect(self._on_local_recording_completed)
|
||||
self.web_view.loadFinished.connect(self._on_load_finished)
|
||||
self.web_view.setUrl(QUrl(self.location.url))
|
||||
@@ -554,6 +582,14 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
if self._closing:
|
||||
return
|
||||
event = str(message.get("event", ""))
|
||||
if event.startswith("local-transcription-"):
|
||||
# An old companion must not start a second recognizer.
|
||||
return
|
||||
if event == "video-ai-action":
|
||||
self._handle_video_assistance_message(message)
|
||||
return
|
||||
if event.startswith("transcription-") and not self._local_transcription_context_is_trusted():
|
||||
return
|
||||
if event in {"im-ai-request", "im-ai-cancel"}:
|
||||
self._handle_im_ai_message(message)
|
||||
return
|
||||
@@ -604,6 +640,9 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
self._abort_local_audio_recording(str(message.get("sessionId") or ""))
|
||||
return
|
||||
if event == "transcription-start-request":
|
||||
if (self._shutdown_requested or not self._start_requested
|
||||
or self._call_cycle_closed or not self.lifecycle.call_record_id):
|
||||
return
|
||||
self._start_transcription(
|
||||
str(message.get("sessionId") or ""),
|
||||
str(message.get("language") or "zh-CN"),
|
||||
@@ -612,6 +651,22 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
if event == "transcription-segment":
|
||||
self._save_transcript_segment(message)
|
||||
return
|
||||
if event == "transcription-ready":
|
||||
session_id = str(message.get("sessionId") or "")
|
||||
if (not self._shutdown_requested and session_id
|
||||
and session_id == str(self.lifecycle.transcription_session_id or "")):
|
||||
self._live_ai.transcription_running(session_id)
|
||||
return
|
||||
if event == "transcription-error":
|
||||
if (str(message.get("sessionId") or "")
|
||||
== str(self.lifecycle.transcription_session_id or "")
|
||||
and self.lifecycle.transcription_session_id):
|
||||
self._asr_partial = True
|
||||
self._live_ai.transcription_error(
|
||||
str(message.get("message") or "腾讯云实时转写未完成,请重试。")[:240],
|
||||
status="failed",
|
||||
)
|
||||
return
|
||||
if event == "transcription-stop":
|
||||
self._finish_transcription(
|
||||
str(message.get("sessionId") or ""),
|
||||
@@ -679,6 +734,110 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
if timer is not None:
|
||||
timer.stop()
|
||||
|
||||
def _local_transcription_context_is_trusted(self) -> bool:
|
||||
return (not self._closing and not self._released and self._injected
|
||||
and self._policy.allows_main_document(self._page.url().toString()))
|
||||
|
||||
def _handle_video_assistance_message(self, message: Mapping[str, Any]) -> None:
|
||||
if not self._local_transcription_context_is_trusted():
|
||||
return
|
||||
event = message.get("event")
|
||||
if event == "video-ai-action":
|
||||
if not self._shutdown_requested:
|
||||
self._live_ai.handle(str(message.get("action") or ""), message.get("enabled"))
|
||||
return
|
||||
session_id = str(message.get("sessionId") or "")
|
||||
if event == "local-transcription-start":
|
||||
if (self._shutdown_requested or not self._start_requested or self._call_cycle_closed
|
||||
or not self.lifecycle.call_record_id
|
||||
or re.fullmatch(r"[A-Za-z0-9_-]{1,128}", session_id) is None):
|
||||
return
|
||||
existing = str(self.lifecycle.transcription_session_id or "")
|
||||
if existing and existing != session_id:
|
||||
return
|
||||
was_ready = self._asr_session_id == session_id and self._asr_ready
|
||||
if self._asr_session_id != session_id:
|
||||
self._asr_partial = False
|
||||
self._asr_ready = False
|
||||
self._asr_session_id = session_id
|
||||
self._live_ai.transcription_starting()
|
||||
accepted = self._local_transcription.start(session_id)
|
||||
if accepted and was_ready:
|
||||
self._on_local_transcription({"type": "ready", "sessionId": session_id})
|
||||
elif not accepted:
|
||||
self._on_local_transcription({"type": "error", "sessionId": session_id,
|
||||
"message": "本次转写已结束,请重新发起通话。"})
|
||||
elif session_id and session_id == self._asr_session_id:
|
||||
if event == "local-transcription-chunk":
|
||||
if session_id != str(self.lifecycle.transcription_session_id or ""):
|
||||
return
|
||||
timestamp = message.get("timestamp")
|
||||
if (isinstance(timestamp, bool) or not isinstance(timestamp, (int, float))
|
||||
or not math.isfinite(timestamp) or not 0 < timestamp < 2**53):
|
||||
return
|
||||
self._local_transcription.submit(
|
||||
session_id, str(message.get("role") or ""), message.get("sequence"),
|
||||
int(timestamp), str(message.get("data") or ""),
|
||||
)
|
||||
elif event == "local-transcription-stop":
|
||||
self._local_transcription.stop(session_id)
|
||||
|
||||
def _emit_local_transcription(self, event: dict[str, Any]) -> None:
|
||||
with suppress(RuntimeError):
|
||||
self._local_transcription_event.emit(event)
|
||||
|
||||
def _on_local_transcription(self, event: Mapping[str, Any]) -> None:
|
||||
if (not self._local_transcription_context_is_trusted()
|
||||
or event.get("sessionId") != self._asr_session_id):
|
||||
return
|
||||
kind = event.get("type")
|
||||
if kind == "ready":
|
||||
self._asr_ready = True
|
||||
elif kind == "stopped":
|
||||
self._asr_ready = False
|
||||
elif kind == "segment":
|
||||
role = event.get("role")
|
||||
if role not in {"doctor", "patient"}:
|
||||
return
|
||||
self._save_transcript_segment({
|
||||
"sessionId": self._asr_session_id,
|
||||
"segment": {
|
||||
"segment_id": event.get("segmentId"),
|
||||
"speaker_role": role,
|
||||
"speaker_user_id": (self.request.user_id if role == "doctor"
|
||||
else self.request.target_user_id),
|
||||
"timestamp": event.get("timestamp"), "text": event.get("text"),
|
||||
},
|
||||
})
|
||||
elif kind == "error":
|
||||
self._asr_partial = True
|
||||
self._live_ai.transcription_error(
|
||||
str(event.get("message") or "本地转写未完成"),
|
||||
status="partial" if self._asr_ready else "failed",
|
||||
)
|
||||
payload = json.dumps(dict(event), ensure_ascii=True, separators=(",", ":"))
|
||||
self._page.runJavaScript(f"window.doctorConsultation?.localTranscriptionEvent?.({payload});")
|
||||
|
||||
def _flush_video_ai(self) -> None:
|
||||
if not self._local_transcription_context_is_trusted() or self._shutdown_requested:
|
||||
return
|
||||
self._live_ai.tick()
|
||||
state = self._live_ai.take_state()
|
||||
if state is not None:
|
||||
payload = json.dumps(state, ensure_ascii=True, separators=(",", ":"))
|
||||
self._page.runJavaScript(f"window.doctorConsultation?.videoAiState?.({payload});")
|
||||
|
||||
def _close_video_assistance(self, *, close_asr: bool = True) -> None:
|
||||
controller = getattr(self, "_live_ai", None)
|
||||
if controller is not None:
|
||||
controller.close()
|
||||
timer = getattr(self, "_video_ai_timer", None)
|
||||
if timer is not None:
|
||||
timer.stop()
|
||||
controller = getattr(self, "_local_transcription", None)
|
||||
if close_asr and controller is not None:
|
||||
controller.close()
|
||||
|
||||
def _open_diagnosis_safely(self) -> None:
|
||||
if self._closing or self._on_open_diagnosis is None:
|
||||
return
|
||||
@@ -1063,6 +1222,12 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
if self._closing or self._shutdown_requested or self._start_requested:
|
||||
return
|
||||
self._prepare_call_cycle()
|
||||
live_ai = getattr(self, "_live_ai", None)
|
||||
if live_ai is not None:
|
||||
live_ai.reset()
|
||||
self._asr_session_id = ""
|
||||
self._asr_ready = False
|
||||
self._asr_partial = False
|
||||
self._start_requested = True
|
||||
try:
|
||||
future = self.lifecycle.start(call_type=call_type)
|
||||
@@ -1141,6 +1306,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
session_id: str,
|
||||
segment_id: str,
|
||||
future: Future[bool],
|
||||
normalized_segment: Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
succeeded = bool(future.result())
|
||||
@@ -1155,7 +1321,8 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
}.get(operation, "")
|
||||
with suppress(RuntimeError):
|
||||
self._transcription_completed.emit(
|
||||
operation, session_id, segment_id, succeeded, message
|
||||
operation, session_id, segment_id, succeeded, message,
|
||||
json.dumps(normalized_segment if succeeded else None, ensure_ascii=True),
|
||||
)
|
||||
|
||||
def _on_transcription_completed(
|
||||
@@ -1165,16 +1332,47 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
segment_id: str,
|
||||
succeeded: bool,
|
||||
message: str,
|
||||
normalized_segment: str | Mapping[str, Any] | None = None,
|
||||
) -> None:
|
||||
if not self._local_transcription_context_is_trusted():
|
||||
return
|
||||
live_ai = getattr(self, "_live_ai", None)
|
||||
if (live_ai is not None
|
||||
and session_id == str(self.lifecycle.transcription_session_id or "")):
|
||||
if operation == "start" and succeeded:
|
||||
live_ai.bind(int(self.lifecycle.call_record_id), session_id)
|
||||
elif operation == "start":
|
||||
live_ai.transcription_error(message or "转写文字存储未就绪,请重试。", status="failed")
|
||||
elif operation == "segment":
|
||||
if succeeded:
|
||||
live_ai.changed(session_id)
|
||||
else:
|
||||
self._asr_partial = True
|
||||
live_ai.transcription_error("部分对话文字保存失败,分析资料可能不完整。")
|
||||
elif operation == "stop":
|
||||
live_ai.finish(session_id, self._transcription_final_status if succeeded else "partial")
|
||||
payload = json.dumps(str(message)[:200], ensure_ascii=True)
|
||||
state = "true" if succeeded else "false"
|
||||
if isinstance(normalized_segment, str):
|
||||
try:
|
||||
normalized_segment = json.loads(normalized_segment)
|
||||
except (ValueError, TypeError):
|
||||
normalized_segment = None
|
||||
caption = json.dumps(
|
||||
dict(normalized_segment)
|
||||
if succeeded and operation == "segment" and isinstance(normalized_segment, Mapping)
|
||||
else None,
|
||||
ensure_ascii=True,
|
||||
)
|
||||
self._page.runJavaScript(
|
||||
"window.doctorConsultation?.transcriptionResult?.("
|
||||
f"{json.dumps(operation)}, {json.dumps(session_id)}, "
|
||||
f"{json.dumps(segment_id)}, {state}, {payload});"
|
||||
f"{json.dumps(segment_id)}, {state}, {payload}, {caption});"
|
||||
)
|
||||
|
||||
def _start_transcription(self, session_id: str, language: str) -> None:
|
||||
if not self._local_transcription_context_is_trusted() or self._shutdown_requested:
|
||||
return
|
||||
try:
|
||||
future = self.lifecycle.start_transcription(session_id, language=language)
|
||||
except Exception as error:
|
||||
@@ -1182,6 +1380,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
"start", session_id, "", False, str(error)[:200]
|
||||
)
|
||||
return
|
||||
self._live_ai.transcription_starting()
|
||||
future.add_done_callback(
|
||||
lambda completed: self._notify_transcription_completed(
|
||||
"start", session_id, "", completed
|
||||
@@ -1189,6 +1388,8 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
)
|
||||
|
||||
def _save_transcript_segment(self, message: Mapping[str, Any]) -> None:
|
||||
if not self._local_transcription_context_is_trusted():
|
||||
return
|
||||
session_id = str(message.get("sessionId") or "").strip()
|
||||
segment = message.get("segment")
|
||||
segment_id = (
|
||||
@@ -1207,25 +1408,44 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
)
|
||||
return
|
||||
try:
|
||||
future = self.lifecycle.save_transcript_segment(segment)
|
||||
normalized = normalize_transcript_segment(segment, session_id)
|
||||
speaker = normalized["speaker_user_id"]
|
||||
normalized["speaker_role"] = (
|
||||
"doctor" if speaker == self.request.user_id
|
||||
else "patient" if speaker == self.request.target_user_id
|
||||
else "unknown"
|
||||
)
|
||||
future = self.lifecycle.save_transcript_segment(normalized)
|
||||
except Exception as error:
|
||||
self._on_transcription_completed(
|
||||
"segment", session_id, segment_id, False, str(error)[:200]
|
||||
)
|
||||
return
|
||||
caption = {
|
||||
"segmentId": normalized["segment_id"],
|
||||
"speakerUserId": normalized["speaker_user_id"],
|
||||
"sourceText": normalized["text"],
|
||||
"timestamp": normalized["timestamp"],
|
||||
"isCompleted": True,
|
||||
}
|
||||
future.add_done_callback(
|
||||
lambda completed: self._notify_transcription_completed(
|
||||
"segment", session_id, segment_id, completed
|
||||
"segment", session_id, segment_id, completed, caption
|
||||
)
|
||||
)
|
||||
|
||||
def _finish_transcription(self, session_id: str, status: str) -> None:
|
||||
if not self._local_transcription_context_is_trusted():
|
||||
return
|
||||
if session_id.strip() != str(self.lifecycle.transcription_session_id or ""):
|
||||
self._on_transcription_completed(
|
||||
"stop", session_id, "", False, "录音会话标识不匹配。"
|
||||
)
|
||||
return
|
||||
try:
|
||||
if getattr(self, "_asr_partial", False):
|
||||
status = "partial"
|
||||
self._transcription_final_status = status
|
||||
future = self.lifecycle.finish_transcription(status=status)
|
||||
except Exception as error:
|
||||
self._on_transcription_completed(
|
||||
@@ -1255,6 +1475,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
"""Let MediaRecorder finish and upload before WebEngine is destroyed."""
|
||||
|
||||
_EmbeddedVideoWindow._close_im_ai(self)
|
||||
_EmbeddedVideoWindow._close_video_assistance(self, close_asr=False)
|
||||
self._close_reason = reason
|
||||
if self._closing or self._released:
|
||||
return
|
||||
@@ -1300,6 +1521,7 @@ if WEBENGINE_AVAILABLE: # pragma: no cover - GUI behavior needs an integration
|
||||
if self._closing:
|
||||
return
|
||||
_EmbeddedVideoWindow._close_im_ai(self)
|
||||
_EmbeddedVideoWindow._close_video_assistance(self)
|
||||
self._on_compact_close = None
|
||||
self._closing = True
|
||||
self._media_active = False
|
||||
|
||||
Reference in New Issue
Block a user