更新

backend/rpa_engine/batch_start.py

@@ -22,11 +22,30 @@ class _StartPreparationTimeout(Exception):
     """Internal marker for the queue's own per-account deadline."""
 
 
+DEFAULT_CONCURRENCY = 6
+MAX_CONCURRENCY = 32
+
+
 def _configured_concurrency() -> int:
+    """Admission width for account preparation.
+
+    Every account spends most of its startup waiting: for the shared network
+    lane, for a WebSocket handshake, for signing work in a thread.  Admitting
+    only two at a time therefore left the network lane idle and made a fleet of
+    several hundred accounts take tens of minutes.  Actual outbound traffic is
+    still capped by the traffic controller, so a wider admission window fills
+    the existing lane instead of adding load.
+    """
     try:
-        return max(1, min(8, int(os.getenv("KEFU_BATCH_START_CONCURRENCY", "2"))))
+        return max(
+            1,
+            min(
+                MAX_CONCURRENCY,
+                int(os.getenv("KEFU_BATCH_START_CONCURRENCY", str(DEFAULT_CONCURRENCY))),
+            ),
+        )
     except (TypeError, ValueError):
-        return 2
+        return DEFAULT_CONCURRENCY
 
 
 def _configured_timeout_seconds() -> float:
@@ -84,19 +103,25 @@ class BatchStartQueue:
         self._active_tasks: dict[int, tuple[JobToken, asyncio.Task]] = {}
         self._batches: dict[str, _BatchRecord] = {}
         self._workers: list[asyncio.Task] = []
+        self._worker_sequence = 0
         self._lock = asyncio.Lock()
         self._stopping = False
 
     async def _ensure_workers(self) -> None:
         async with self._lock:
             self._workers = [task for task in self._workers if not task.done()]
-            if self._workers or self._stopping:
+            if self._stopping:
                 return
-            for index in range(self.concurrency):
+            # Top up to the configured width instead of only starting from
+            # zero.  A worker that died on an unexpected error used to shrink
+            # the queue permanently, so later batches crawled through a single
+            # remaining worker with no way to recover short of a restart.
+            while len(self._workers) < self.concurrency:
+                self._worker_sequence += 1
                 self._workers.append(
                     asyncio.create_task(
-                        self._worker(index + 1),
-                        name=f"account-batch-start-{index + 1}",
+                        self._worker(self._worker_sequence),
+                        name=f"account-batch-start-{self._worker_sequence}",
                     )
                 )
 

backend/rpa_engine/douyin_im/traffic_control.py

@@ -393,9 +393,8 @@ class TrafficController:
                 1.0,
             )
         )
-        self._background = asyncio.Semaphore(
-            _env_int("KEFU_BACKGROUND_NETWORK_CONCURRENCY", 2)
-        )
+        background_capacity = _env_int("KEFU_BACKGROUND_NETWORK_CONCURRENCY", 4)
+        self._background = asyncio.Semaphore(background_capacity)
         # Recurring polls can create hundreds of waiters when many accounts
         # are online.  Admit at most one normal waiter to the semaphore at a
         # time so startup validation can join near the front instead of being
@@ -405,6 +404,17 @@ class TrafficController:
         self._background_normal_admission = asyncio.Lock()
         self._background_startup_clear = asyncio.Event()
         self._background_startup_clear.set()
+        # Starting several hundred accounts keeps startup waiters queued for
+        # many minutes on end.  Leave one slot of the shared lane for recurring
+        # work so hosted accounts keep receiving messages during a bulk start
+        # instead of going silent until the last account is up.
+        self._background_startup = asyncio.Semaphore(
+            max(1, background_capacity - 1)
+        )
+        self._background_normal_max_defer = _env_float(
+            "KEFU_BACKGROUND_NORMAL_MAX_DEFER_SECONDS",
+            5.0,
+        )
         self._browser = asyncio.Semaphore(
             _env_int("KEFU_BROWSER_START_CONCURRENCY", 1)
         )
@@ -449,8 +459,15 @@ class TrafficController:
             if startup:
                 self.background_startup_waiting += 1
                 self._background_startup_clear.clear()
+                startup_reservation = False
                 try:
+                    await self._background_startup.acquire()
+                    startup_reservation = True
                     await self._background.acquire()
+                except BaseException:
+                    if startup_reservation:
+                        self._background_startup.release()
+                    raise
                 finally:
                     self.background_startup_waiting -= 1
                     if self.background_startup_waiting == 0:
@@ -460,7 +477,23 @@ class TrafficController:
                 # the shared semaphore.  A later startup request therefore
                 # has at most one normal request ahead of it, not hundreds.
                 async with self._background_normal_admission:
-                    await self._background_startup_clear.wait()
+                    # Yield to pending startup work, but only while a startup
+                    # waiter could still claim a slot, and never for longer
+                    # than the deferral budget.  A batch of several hundred
+                    # accounts otherwise keeps startup waiters pending for the
+                    # whole run, which stalled every recurring poll behind it.
+                    if (
+                        self._background_normal_max_defer > 0
+                        and not self._background_startup_clear.is_set()
+                        and not self._background_startup.locked()
+                    ):
+                        try:
+                            await asyncio.wait_for(
+                                self._background_startup_clear.wait(),
+                                timeout=self._background_normal_max_defer,
+                            )
+                        except asyncio.TimeoutError:
+                            pass
                     await self._background.acquire()
         except BaseException:
             self.background_waiting -= 1
@@ -484,6 +517,7 @@ class TrafficController:
             self._background_owner.reset(token)
             if startup:
                 self.background_startup_active -= 1
+                self._background_startup.release()
             self.background_active -= 1
             self._background.release()