更新

backend/rpa_engine/batch_start.py

@@ -18,6 +18,10 @@ StartHandler = Callable[[int], Awaitable[dict[str, Any]]]
 JobToken = tuple[str, int]
 
 
+class _StartPreparationTimeout(Exception):
+    """Internal marker for the queue's own per-account deadline."""
+
+
 def _configured_concurrency() -> int:
     try:
         return max(1, min(8, int(os.getenv("KEFU_BATCH_START_CONCURRENCY", "2"))))
@@ -25,6 +29,16 @@ def _configured_concurrency() -> int:
         return 2
 
 
+def _configured_timeout_seconds() -> float:
+    try:
+        value = float(os.getenv("KEFU_BATCH_START_TIMEOUT_SECONDS", "90"))
+    except (TypeError, ValueError):
+        return 90.0
+    if value <= 0:
+        return 0.0
+    return max(5.0, min(600.0, value))
+
+
 def _utc_now() -> str:
     return datetime.now(timezone.utc).isoformat()
 
@@ -55,10 +69,16 @@ class BatchStartQueue:
         handler: StartHandler,
         concurrency: int | None = None,
         max_batches: int = 100,
+        timeout_seconds: float | None = None,
     ) -> None:
         self._handler = handler
         self.concurrency = max(1, int(concurrency or _configured_concurrency()))
         self.max_batches = max(10, int(max_batches or 100))
+        self.timeout_seconds = (
+            _configured_timeout_seconds()
+            if timeout_seconds is None
+            else max(0.0, float(timeout_seconds or 0.0))
+        )
         self._queue: asyncio.Queue[JobToken] = asyncio.Queue()
         self._pending_jobs: dict[int, JobToken] = {}
         self._active_tasks: dict[int, tuple[JobToken, asyncio.Task]] = {}
@@ -157,7 +177,21 @@ class BatchStartQueue:
                     )
                     self._active_tasks[account_id] = (job_token, handler_task)
 
-                result = await handler_task
+                if self.timeout_seconds > 0:
+                    try:
+                        result = await asyncio.wait_for(
+                            handler_task,
+                            timeout=self.timeout_seconds,
+                        )
+                    except asyncio.TimeoutError as exc:
+                        # wait_for cancels its task only when this queue's
+                        # deadline expires.  Preserve a TimeoutError raised by
+                        # the handler itself as its real account failure.
+                        if handler_task.cancelled():
+                            raise _StartPreparationTimeout from exc
+                        raise
+                else:
+                    result = await handler_task
                 async with self._lock:
                     record = self._batches.get(batch_id)
                     if record:
@@ -171,6 +205,28 @@ class BatchStartQueue:
                                 elapsed_seconds=round(time.monotonic() - started_at, 3),
                             )
                         record.updated_at = _utc_now()
+            except _StartPreparationTimeout:
+                elapsed = round(time.monotonic() - started_at, 3)
+                logger.warning(
+                    "Batch start timed out account=%s worker=%s after %.1fs",
+                    account_id,
+                    worker_number,
+                    self.timeout_seconds,
+                )
+                async with self._lock:
+                    record = self._batches.get(batch_id)
+                    if record:
+                        item = record.items[account_id]
+                        if item.get("status") != "cancelled":
+                            item.update(
+                                status="failed",
+                                message=(
+                                    f"启动准备超过 {self.timeout_seconds:g} 秒，"
+                                    "已跳过并继续处理后续账号"
+                                ),
+                                elapsed_seconds=elapsed,
+                            )
+                        record.updated_at = _utc_now()
             except asyncio.CancelledError:
                 async with self._lock:
                     record = self._batches.get(batch_id)