更新

backend/rpa_engine/douyin_im/traffic_control.py

@@ -393,9 +393,8 @@ class TrafficController:
                 1.0,
             )
         )
-        self._background = asyncio.Semaphore(
-            _env_int("KEFU_BACKGROUND_NETWORK_CONCURRENCY", 2)
-        )
+        background_capacity = _env_int("KEFU_BACKGROUND_NETWORK_CONCURRENCY", 4)
+        self._background = asyncio.Semaphore(background_capacity)
         # Recurring polls can create hundreds of waiters when many accounts
         # are online.  Admit at most one normal waiter to the semaphore at a
         # time so startup validation can join near the front instead of being
@@ -405,6 +404,17 @@ class TrafficController:
         self._background_normal_admission = asyncio.Lock()
         self._background_startup_clear = asyncio.Event()
         self._background_startup_clear.set()
+        # Starting several hundred accounts keeps startup waiters queued for
+        # many minutes on end.  Leave one slot of the shared lane for recurring
+        # work so hosted accounts keep receiving messages during a bulk start
+        # instead of going silent until the last account is up.
+        self._background_startup = asyncio.Semaphore(
+            max(1, background_capacity - 1)
+        )
+        self._background_normal_max_defer = _env_float(
+            "KEFU_BACKGROUND_NORMAL_MAX_DEFER_SECONDS",
+            5.0,
+        )
         self._browser = asyncio.Semaphore(
             _env_int("KEFU_BROWSER_START_CONCURRENCY", 1)
         )
@@ -449,8 +459,15 @@ class TrafficController:
             if startup:
                 self.background_startup_waiting += 1
                 self._background_startup_clear.clear()
+                startup_reservation = False
                 try:
+                    await self._background_startup.acquire()
+                    startup_reservation = True
                     await self._background.acquire()
+                except BaseException:
+                    if startup_reservation:
+                        self._background_startup.release()
+                    raise
                 finally:
                     self.background_startup_waiting -= 1
                     if self.background_startup_waiting == 0:
@@ -460,7 +477,23 @@ class TrafficController:
                 # the shared semaphore.  A later startup request therefore
                 # has at most one normal request ahead of it, not hundreds.
                 async with self._background_normal_admission:
-                    await self._background_startup_clear.wait()
+                    # Yield to pending startup work, but only while a startup
+                    # waiter could still claim a slot, and never for longer
+                    # than the deferral budget.  A batch of several hundred
+                    # accounts otherwise keeps startup waiters pending for the
+                    # whole run, which stalled every recurring poll behind it.
+                    if (
+                        self._background_normal_max_defer > 0
+                        and not self._background_startup_clear.is_set()
+                        and not self._background_startup.locked()
+                    ):
+                        try:
+                            await asyncio.wait_for(
+                                self._background_startup_clear.wait(),
+                                timeout=self._background_normal_max_defer,
+                            )
+                        except asyncio.TimeoutError:
+                            pass
                     await self._background.acquire()
         except BaseException:
             self.background_waiting -= 1
@@ -484,6 +517,7 @@ class TrafficController:
             self._background_owner.reset(token)
             if startup:
                 self.background_startup_active -= 1
+                self._background_startup.release()
             self.background_active -= 1
             self._background.release()