235 lines
8.5 KiB
Python
235 lines
8.5 KiB
Python
"""Retry a failed storage start without duplicate requests or reopening a call."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import threading
|
|
from concurrent.futures import Future, ThreadPoolExecutor
|
|
|
|
import pytest
|
|
|
|
from doctor_workstation.video.launcher import VideoCallRequest
|
|
from doctor_workstation.video.lifecycle import OrderedCallLifecycle
|
|
|
|
|
|
class Repository:
|
|
def __init__(self, *, fail_first: bool = False, block_attempt: int = 0) -> None:
|
|
self.fail_first = fail_first
|
|
self.block_attempt = block_attempt
|
|
self.attempts: list[dict] = []
|
|
self.entered = threading.Event()
|
|
self.release = threading.Event()
|
|
|
|
def start_call(self, **_payload):
|
|
return {"call_record_id": 71}
|
|
|
|
def start_call_transcription(self, **payload):
|
|
self.attempts.append(payload)
|
|
attempt = len(self.attempts)
|
|
if attempt == self.block_attempt:
|
|
self.entered.set()
|
|
assert self.release.wait(3), "test did not release storage request"
|
|
if self.fail_first and attempt == 1:
|
|
raise RuntimeError("temporary storage failure")
|
|
|
|
def finish_call_transcription(self, **_payload):
|
|
return True
|
|
|
|
def end_call(self, **_payload):
|
|
return True
|
|
|
|
|
|
@pytest.fixture
|
|
def make_lifecycle():
|
|
lifecycles = []
|
|
|
|
def make(**options):
|
|
repository = Repository(**options)
|
|
request = VideoCallRequest(
|
|
sdk_app_id=1400123456,
|
|
user_id="doctor-test",
|
|
user_sig="test-ticket",
|
|
target_user_id="patient-test",
|
|
diagnosis_id=12,
|
|
)
|
|
lifecycle = OrderedCallLifecycle(request, repository, logging.getLogger(__name__))
|
|
lifecycles.append((lifecycle, repository))
|
|
assert lifecycle.start().result(timeout=2) is True
|
|
return lifecycle, repository
|
|
|
|
yield make
|
|
for lifecycle, repository in lifecycles:
|
|
repository.release.set()
|
|
lifecycle.end("test-cleanup").result(timeout=3)
|
|
assert lifecycle.wait(3)
|
|
|
|
|
|
def test_failed_storage_start_retries_same_session_then_reuses_success(make_lifecycle):
|
|
lifecycle, repository = make_lifecycle(fail_first=True)
|
|
failed = lifecycle.start_transcription("tr-retry", language="zh")
|
|
with pytest.raises(RuntimeError, match="temporary storage failure"):
|
|
failed.result(timeout=2)
|
|
|
|
retry = lifecycle.start_transcription("tr-retry", language="zh")
|
|
assert retry is not failed
|
|
assert retry.result(timeout=2) is True
|
|
assert lifecycle.start_transcription("tr-retry") is retry
|
|
assert lifecycle.transcription_session_id == "tr-retry"
|
|
assert len(repository.attempts) == 2
|
|
assert (
|
|
repository.attempts[0]
|
|
== repository.attempts[1]
|
|
== {
|
|
"diagnosis_id": 12,
|
|
"call_record_id": 71,
|
|
"transcription_session_id": "tr-retry",
|
|
"language": "zh",
|
|
}
|
|
)
|
|
|
|
|
|
def test_inflight_start_is_reused_without_blocking_caller(make_lifecycle):
|
|
lifecycle, repository = make_lifecycle(block_attempt=1)
|
|
first = lifecycle.start_transcription("tr-running")
|
|
try:
|
|
assert repository.entered.wait(2)
|
|
assert not first.done()
|
|
assert lifecycle.start_transcription("tr-running") is first
|
|
assert len(repository.attempts) == 1
|
|
finally:
|
|
repository.release.set()
|
|
assert first.result(timeout=2) is True
|
|
|
|
|
|
def test_concurrent_retry_callers_share_one_new_request(make_lifecycle):
|
|
lifecycle, repository = make_lifecycle(fail_first=True, block_attempt=2)
|
|
with pytest.raises(RuntimeError, match="temporary storage failure"):
|
|
lifecycle.start_transcription("tr-concurrent").result(timeout=2)
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=4) as callers:
|
|
futures = list(
|
|
callers.map(lambda _: lifecycle.start_transcription("tr-concurrent"), range(4))
|
|
)
|
|
assert repository.entered.wait(2)
|
|
assert all(future is futures[0] for future in futures)
|
|
assert not futures[0].done()
|
|
assert len(repository.attempts) == 2
|
|
finally:
|
|
repository.release.set()
|
|
assert futures[0].result(timeout=2) is True
|
|
|
|
|
|
def test_false_completion_can_retry_without_changing_identity(make_lifecycle):
|
|
lifecycle, repository = make_lifecycle()
|
|
# Seed the documented False completion (e.g. call-start dependency had not
|
|
# succeeded), then exercise retry against a now-available actual lifecycle.
|
|
previous: Future[bool] = Future()
|
|
previous.set_result(False)
|
|
lifecycle._transcription_session_id = "tr-false"
|
|
lifecycle._transcription_start_future = previous
|
|
|
|
retry = lifecycle.start_transcription("tr-false")
|
|
assert retry is not previous
|
|
assert retry.result(timeout=2) is True
|
|
assert lifecycle.start_transcription("tr-false") is retry
|
|
assert len(repository.attempts) == 1
|
|
assert repository.attempts[0]["transcription_session_id"] == "tr-false"
|
|
|
|
|
|
def test_cancelled_queued_start_can_retry_same_session(make_lifecycle):
|
|
lifecycle, repository = make_lifecycle()
|
|
entered, release = threading.Event(), threading.Event()
|
|
|
|
def occupy_worker():
|
|
entered.set()
|
|
assert release.wait(3)
|
|
|
|
lifecycle._worker.submit("test-block", occupy_worker)
|
|
try:
|
|
assert entered.wait(2)
|
|
cancelled = lifecycle.start_transcription("tr-cancelled")
|
|
assert cancelled.cancel()
|
|
retry = lifecycle.start_transcription("tr-cancelled")
|
|
assert retry is not cancelled
|
|
finally:
|
|
release.set()
|
|
assert retry.result(timeout=2) is True
|
|
assert len(repository.attempts) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["running", "succeeded", "failed", "false"])
|
|
def test_session_identity_cannot_change_even_after_failure(make_lifecycle, state):
|
|
lifecycle, repository = make_lifecycle(
|
|
fail_first=state == "failed", block_attempt=1 if state == "running" else 0
|
|
)
|
|
if state == "false":
|
|
first: Future[bool] = Future()
|
|
first.set_result(False)
|
|
lifecycle._transcription_session_id = "tr-original"
|
|
lifecycle._transcription_start_future = first
|
|
else:
|
|
first = lifecycle.start_transcription("tr-original")
|
|
if state == "running":
|
|
assert repository.entered.wait(2)
|
|
elif state == "failed":
|
|
with pytest.raises(RuntimeError, match="temporary storage failure"):
|
|
first.result(timeout=2)
|
|
else:
|
|
assert first.result(timeout=2)
|
|
try:
|
|
with pytest.raises(RuntimeError, match="another transcription session"):
|
|
lifecycle.start_transcription("tr-different")
|
|
assert lifecycle.transcription_session_id == "tr-original"
|
|
finally:
|
|
repository.release.set()
|
|
|
|
|
|
@pytest.mark.parametrize("terminal", ["finish", "end"])
|
|
@pytest.mark.parametrize("failed", [False, True])
|
|
def test_finish_or_end_blocks_restart_before_and_after_completion(make_lifecycle, terminal, failed):
|
|
lifecycle, repository = make_lifecycle(fail_first=failed)
|
|
first = lifecycle.start_transcription("tr-terminal")
|
|
if failed:
|
|
with pytest.raises(RuntimeError, match="temporary storage failure"):
|
|
first.result(timeout=2)
|
|
else:
|
|
assert first.result(timeout=2)
|
|
|
|
ending = (
|
|
lifecycle.finish_transcription(status="partial")
|
|
if terminal == "finish"
|
|
else lifecycle.end("test-terminal")
|
|
)
|
|
error = "already stopped" if terminal == "finish" else "already ended"
|
|
with pytest.raises(RuntimeError, match=error):
|
|
lifecycle.start_transcription("tr-terminal")
|
|
ending.result(timeout=2)
|
|
with pytest.raises(RuntimeError, match=error):
|
|
lifecycle.start_transcription("tr-terminal")
|
|
assert len(repository.attempts) == 1
|
|
|
|
|
|
@pytest.mark.parametrize("terminal", ["finish", "end"])
|
|
def test_terminal_request_blocks_start_even_while_original_start_is_running(
|
|
make_lifecycle, terminal
|
|
):
|
|
lifecycle, repository = make_lifecycle(block_attempt=1)
|
|
first = lifecycle.start_transcription("tr-draining")
|
|
try:
|
|
assert repository.entered.wait(2)
|
|
ending = (
|
|
lifecycle.finish_transcription()
|
|
if terminal == "finish"
|
|
else lifecycle.end("test-draining")
|
|
)
|
|
assert not first.done()
|
|
assert not ending.done()
|
|
error = "already stopped" if terminal == "finish" else "already ended"
|
|
with pytest.raises(RuntimeError, match=error):
|
|
lifecycle.start_transcription("tr-draining")
|
|
finally:
|
|
repository.release.set()
|
|
assert first.result(timeout=2) is True
|
|
assert ending.result(timeout=2) is True
|
|
assert len(repository.attempts) == 1
|