更新:

This commit is contained in:
Your Name
2026-07-17 10:21:18 +08:00
parent 530e7f839d
commit 2f2150d2f6
6 changed files with 596 additions and 30 deletions
@@ -818,7 +818,22 @@ class DouyinImHttpClient:
"""通过 IM API 发送 Protobuf 编码的私信(带接口签名)
content 可为纯文本,或 JSON 格式的结构化回复(文本/网址/卡片)。
所有出站发送(自动回复/手动 API/脚本)都经过全局限流器,
防止大量托管账号同时发送占满出站带宽或触发平台级风控。
"""
from .rate_limit import get_send_limiter
async with get_send_limiter():
return await self._send_text_message_unthrottled(
conversation_id, content, conversation_short_id
)
async def _send_text_message_unthrottled(
self,
conversation_id: str,
content: str,
conversation_short_id: str = "",
) -> bool:
from .auth import DouyinAuth
from .proto_builder import ProtoBuilder
from .reply_payload import (
+117
View File
@@ -0,0 +1,117 @@
"""全局出站发送限流器。
目的:托管账号数量多(如 1000 个)时,避免自动回复/手动发送在同一时刻
大量并发打向抖音接口,占满服务器出站带宽或触发平台级风控。
机制(两层,先到先过):
1. 并发上限(Semaphore):同时在途的发送请求数不超过 KEFU_SEND_MAX_CONCURRENCY
2. 令牌桶(QPS):平均发送速率不超过 KEFU_SEND_RATE_PER_SEC,允许 KEFU_SEND_BURST 的突发。
限流器按事件循环惰性创建(backend 主循环一个实例;独立脚本各自一个),
所有账号共享同一个实例,因此是跨账号的全局闸门。
环境变量:
KEFU_SEND_MAX_CONCURRENCY 同时在途发送数上限,默认 20,<=0 表示不限
KEFU_SEND_RATE_PER_SEC 平均每秒发送数上限,默认 10,<=0 表示不限
KEFU_SEND_BURST 令牌桶容量(突发上限),默认与并发上限相同
"""
from __future__ import annotations
import asyncio
import logging
import os
import time
logger = logging.getLogger("douyin_im.rate_limit")
def _env_float(name: str, default: float) -> float:
try:
return float(os.getenv(name, "") or default)
except ValueError:
return default
class GlobalSendLimiter:
"""并发上限 + 令牌桶。用法:async with limiter: await 发送。"""
def __init__(
self,
max_concurrency: int = 20,
rate_per_sec: float = 10.0,
burst: float | None = None,
):
self.max_concurrency = int(max_concurrency)
self.rate_per_sec = float(rate_per_sec)
self.burst = float(burst if burst is not None else max(1, max_concurrency))
self._sem = (
asyncio.Semaphore(self.max_concurrency) if self.max_concurrency > 0 else None
)
self._tokens = self.burst
self._last_refill = time.monotonic()
self._token_lock = asyncio.Lock()
# 统计:便于日志观察限流是否生效
self._waited_total = 0.0
self._acquired_count = 0
async def _take_token(self) -> None:
if self.rate_per_sec <= 0:
return
async with self._token_lock:
while True:
now = time.monotonic()
self._tokens = min(
self.burst, self._tokens + (now - self._last_refill) * self.rate_per_sec
)
self._last_refill = now
if self._tokens >= 1.0:
self._tokens -= 1.0
return
await asyncio.sleep((1.0 - self._tokens) / self.rate_per_sec)
async def __aenter__(self) -> "GlobalSendLimiter":
start = time.monotonic()
if self._sem is not None:
await self._sem.acquire()
try:
await self._take_token()
except BaseException:
if self._sem is not None:
self._sem.release()
raise
waited = time.monotonic() - start
self._waited_total += waited
self._acquired_count += 1
if waited > 1.0:
logger.info(
"send throttled: waited %.1fs (in-flight cap=%s, rate=%s/s)",
waited, self.max_concurrency or "", self.rate_per_sec or "",
)
return self
async def __aexit__(self, *exc) -> None:
if self._sem is not None:
self._sem.release()
# 每个事件循环一个实例(backend 主循环即全局唯一;脚本自用循环互不影响)
_limiters: dict[int, GlobalSendLimiter] = {}
def get_send_limiter() -> GlobalSendLimiter:
loop = asyncio.get_running_loop()
key = id(loop)
limiter = _limiters.get(key)
if limiter is None:
max_conc = int(_env_float("KEFU_SEND_MAX_CONCURRENCY", 20))
rate = _env_float("KEFU_SEND_RATE_PER_SEC", 10.0)
burst = _env_float("KEFU_SEND_BURST", max(1, max_conc))
limiter = GlobalSendLimiter(max_conc, rate, burst)
_limiters[key] = limiter
logger.info(
"global send limiter ready: concurrency=%s, rate=%s/s, burst=%s",
max_conc if max_conc > 0 else "unlimited",
rate if rate > 0 else "unlimited",
burst,
)
return limiter
+21 -1
View File
@@ -1,5 +1,7 @@
import asyncio
import logging
import os
import random
import time
from typing import Awaitable, Callable, Optional
@@ -374,6 +376,22 @@ class DouyinImService:
)
await self._ws_client.start()
# 轮询错峰:多账号同时托管时,若所有账号按同一节奏轮询,请求会在同一
# 时刻叠峰。这里给每个账号随机相位偏移 + 每轮 ±20% 抖动,把请求摊平。
# WS 可用时轮询只是兜底,可以适当放缓(KEFU_IM_POLL_INTERVAL_SECONDS 可调)。
try:
poll_interval = float(os.getenv("KEFU_IM_POLL_INTERVAL_SECONDS", "") or 15)
except ValueError:
poll_interval = 15.0
poll_interval = max(5.0, poll_interval)
if has_ws:
poll_interval = max(poll_interval, 30.0)
# 首轮轮询前的随机延迟(相位偏移),批量启动时错开各账号的首波请求
await asyncio.sleep(random.uniform(0.5, min(10.0, poll_interval)))
if not self._running:
return
try:
await self._poll_conversations()
except Exception as e:
@@ -387,10 +405,12 @@ class DouyinImService:
)
loop_count = 0
next_poll = time.monotonic() + poll_interval * random.uniform(0.8, 1.2)
while self._running:
loop_count += 1
try:
if loop_count % 3 == 1:
if time.monotonic() >= next_poll:
next_poll = time.monotonic() + poll_interval * random.uniform(0.8, 1.2)
await self._poll_conversations()
if loop_count % 6 == 1:
logger.info(f"IM direct tick #{loop_count} account={self.account_id}")
+33
View File
@@ -49,6 +49,36 @@ async def _launch_chromium(pw, args: list[str], headless: Optional[bool] = None)
return await pw.chromium.launch(**launch_kwargs)
# ---------------------------------------------------------------------------
# 启动错峰门:批量启动大量账号时,把各 worker 的启动时刻按固定间隔排开,
# 避免同一瞬间大量凭证校验/WS 建连/首轮拉取叠峰。
# 空闲时单个账号启动无需等待;只有短时间内大量启动才会排队。
# KEFU_WORKER_START_INTERVAL_SECONDS:相邻两个 worker 启动的最小间隔,默认 1.5s,<=0 关闭。
# ---------------------------------------------------------------------------
_start_gate = {"lock": None, "next_at": 0.0}
async def _startup_stagger(account_id: int) -> None:
try:
interval = float(os.getenv("KEFU_WORKER_START_INTERVAL_SECONDS", "") or 1.5)
except ValueError:
interval = 1.5
if interval <= 0:
return
if _start_gate["lock"] is None:
_start_gate["lock"] = asyncio.Lock()
async with _start_gate["lock"]:
now = time.monotonic()
wait = max(0.0, _start_gate["next_at"] - now)
_start_gate["next_at"] = max(now, _start_gate["next_at"]) + interval
if wait > 0:
if wait > 5:
logger.info(
f"Account {account_id}: start queued, waiting {wait:.1f}s to smooth batch startup"
)
await asyncio.sleep(wait)
def format_error(exc: BaseException) -> str:
message = str(exc).strip()
if "Target page, context or browser has been closed" in message:
@@ -995,6 +1025,9 @@ class DouyinWorker:
logger.info(f"Starting worker loop for account {self.account_id}")
try:
await _startup_stagger(self.account_id)
if self.stopping:
return
storage_state = await self._load_storage_state()
cookie_info = analyze_cookie(
json.dumps(storage_state, ensure_ascii=False) if storage_state else None