diff --git a/backend/rpa_engine/douyin_im/http_client.py b/backend/rpa_engine/douyin_im/http_client.py index 73ff628..5aee173 100644 --- a/backend/rpa_engine/douyin_im/http_client.py +++ b/backend/rpa_engine/douyin_im/http_client.py @@ -818,7 +818,22 @@ class DouyinImHttpClient: """通过 IM API 发送 Protobuf 编码的私信(带接口签名) content 可为纯文本,或 JSON 格式的结构化回复(文本/网址/卡片)。 + 所有出站发送(自动回复/手动 API/脚本)都经过全局限流器, + 防止大量托管账号同时发送占满出站带宽或触发平台级风控。 """ + from .rate_limit import get_send_limiter + + async with get_send_limiter(): + return await self._send_text_message_unthrottled( + conversation_id, content, conversation_short_id + ) + + async def _send_text_message_unthrottled( + self, + conversation_id: str, + content: str, + conversation_short_id: str = "", + ) -> bool: from .auth import DouyinAuth from .proto_builder import ProtoBuilder from .reply_payload import ( diff --git a/backend/rpa_engine/douyin_im/rate_limit.py b/backend/rpa_engine/douyin_im/rate_limit.py new file mode 100644 index 0000000..d3b4180 --- /dev/null +++ b/backend/rpa_engine/douyin_im/rate_limit.py @@ -0,0 +1,117 @@ +"""全局出站发送限流器。 + +目的:托管账号数量多(如 1000 个)时,避免自动回复/手动发送在同一时刻 +大量并发打向抖音接口,占满服务器出站带宽或触发平台级风控。 + +机制(两层,先到先过): + 1. 并发上限(Semaphore):同时在途的发送请求数不超过 KEFU_SEND_MAX_CONCURRENCY; + 2. 令牌桶(QPS):平均发送速率不超过 KEFU_SEND_RATE_PER_SEC,允许 KEFU_SEND_BURST 的突发。 + +限流器按事件循环惰性创建(backend 主循环一个实例;独立脚本各自一个), +所有账号共享同一个实例,因此是跨账号的全局闸门。 + +环境变量: + KEFU_SEND_MAX_CONCURRENCY 同时在途发送数上限,默认 20,<=0 表示不限 + KEFU_SEND_RATE_PER_SEC 平均每秒发送数上限,默认 10,<=0 表示不限 + KEFU_SEND_BURST 令牌桶容量(突发上限),默认与并发上限相同 +""" +from __future__ import annotations + +import asyncio +import logging +import os +import time + +logger = logging.getLogger("douyin_im.rate_limit") + + +def _env_float(name: str, default: float) -> float: + try: + return float(os.getenv(name, "") or default) + except ValueError: + return default + + +class GlobalSendLimiter: + """并发上限 + 令牌桶。用法:async with limiter: await 发送。""" + + def __init__( + self, + max_concurrency: int = 20, + rate_per_sec: float = 10.0, + burst: float | None = None, + ): + self.max_concurrency = int(max_concurrency) + self.rate_per_sec = float(rate_per_sec) + self.burst = float(burst if burst is not None else max(1, max_concurrency)) + self._sem = ( + asyncio.Semaphore(self.max_concurrency) if self.max_concurrency > 0 else None + ) + self._tokens = self.burst + self._last_refill = time.monotonic() + self._token_lock = asyncio.Lock() + # 统计:便于日志观察限流是否生效 + self._waited_total = 0.0 + self._acquired_count = 0 + + async def _take_token(self) -> None: + if self.rate_per_sec <= 0: + return + async with self._token_lock: + while True: + now = time.monotonic() + self._tokens = min( + self.burst, self._tokens + (now - self._last_refill) * self.rate_per_sec + ) + self._last_refill = now + if self._tokens >= 1.0: + self._tokens -= 1.0 + return + await asyncio.sleep((1.0 - self._tokens) / self.rate_per_sec) + + async def __aenter__(self) -> "GlobalSendLimiter": + start = time.monotonic() + if self._sem is not None: + await self._sem.acquire() + try: + await self._take_token() + except BaseException: + if self._sem is not None: + self._sem.release() + raise + waited = time.monotonic() - start + self._waited_total += waited + self._acquired_count += 1 + if waited > 1.0: + logger.info( + "send throttled: waited %.1fs (in-flight cap=%s, rate=%s/s)", + waited, self.max_concurrency or "∞", self.rate_per_sec or "∞", + ) + return self + + async def __aexit__(self, *exc) -> None: + if self._sem is not None: + self._sem.release() + + +# 每个事件循环一个实例(backend 主循环即全局唯一;脚本自用循环互不影响) +_limiters: dict[int, GlobalSendLimiter] = {} + + +def get_send_limiter() -> GlobalSendLimiter: + loop = asyncio.get_running_loop() + key = id(loop) + limiter = _limiters.get(key) + if limiter is None: + max_conc = int(_env_float("KEFU_SEND_MAX_CONCURRENCY", 20)) + rate = _env_float("KEFU_SEND_RATE_PER_SEC", 10.0) + burst = _env_float("KEFU_SEND_BURST", max(1, max_conc)) + limiter = GlobalSendLimiter(max_conc, rate, burst) + _limiters[key] = limiter + logger.info( + "global send limiter ready: concurrency=%s, rate=%s/s, burst=%s", + max_conc if max_conc > 0 else "unlimited", + rate if rate > 0 else "unlimited", + burst, + ) + return limiter diff --git a/backend/rpa_engine/douyin_im/service.py b/backend/rpa_engine/douyin_im/service.py index 3d4773d..5fdb1bf 100644 --- a/backend/rpa_engine/douyin_im/service.py +++ b/backend/rpa_engine/douyin_im/service.py @@ -1,5 +1,7 @@ import asyncio import logging +import os +import random import time from typing import Awaitable, Callable, Optional @@ -374,6 +376,22 @@ class DouyinImService: ) await self._ws_client.start() + # 轮询错峰:多账号同时托管时,若所有账号按同一节奏轮询,请求会在同一 + # 时刻叠峰。这里给每个账号随机相位偏移 + 每轮 ±20% 抖动,把请求摊平。 + # WS 可用时轮询只是兜底,可以适当放缓(KEFU_IM_POLL_INTERVAL_SECONDS 可调)。 + try: + poll_interval = float(os.getenv("KEFU_IM_POLL_INTERVAL_SECONDS", "") or 15) + except ValueError: + poll_interval = 15.0 + poll_interval = max(5.0, poll_interval) + if has_ws: + poll_interval = max(poll_interval, 30.0) + + # 首轮轮询前的随机延迟(相位偏移),批量启动时错开各账号的首波请求 + await asyncio.sleep(random.uniform(0.5, min(10.0, poll_interval))) + if not self._running: + return + try: await self._poll_conversations() except Exception as e: @@ -387,10 +405,12 @@ class DouyinImService: ) loop_count = 0 + next_poll = time.monotonic() + poll_interval * random.uniform(0.8, 1.2) while self._running: loop_count += 1 try: - if loop_count % 3 == 1: + if time.monotonic() >= next_poll: + next_poll = time.monotonic() + poll_interval * random.uniform(0.8, 1.2) await self._poll_conversations() if loop_count % 6 == 1: logger.info(f"IM direct tick #{loop_count} account={self.account_id}") diff --git a/backend/rpa_engine/playwright_worker.py b/backend/rpa_engine/playwright_worker.py index 04c8edd..eb9f01f 100644 --- a/backend/rpa_engine/playwright_worker.py +++ b/backend/rpa_engine/playwright_worker.py @@ -49,6 +49,36 @@ async def _launch_chromium(pw, args: list[str], headless: Optional[bool] = None) return await pw.chromium.launch(**launch_kwargs) +# --------------------------------------------------------------------------- +# 启动错峰门:批量启动大量账号时,把各 worker 的启动时刻按固定间隔排开, +# 避免同一瞬间大量凭证校验/WS 建连/首轮拉取叠峰。 +# 空闲时单个账号启动无需等待;只有短时间内大量启动才会排队。 +# KEFU_WORKER_START_INTERVAL_SECONDS:相邻两个 worker 启动的最小间隔,默认 1.5s,<=0 关闭。 +# --------------------------------------------------------------------------- +_start_gate = {"lock": None, "next_at": 0.0} + + +async def _startup_stagger(account_id: int) -> None: + try: + interval = float(os.getenv("KEFU_WORKER_START_INTERVAL_SECONDS", "") or 1.5) + except ValueError: + interval = 1.5 + if interval <= 0: + return + if _start_gate["lock"] is None: + _start_gate["lock"] = asyncio.Lock() + async with _start_gate["lock"]: + now = time.monotonic() + wait = max(0.0, _start_gate["next_at"] - now) + _start_gate["next_at"] = max(now, _start_gate["next_at"]) + interval + if wait > 0: + if wait > 5: + logger.info( + f"Account {account_id}: start queued, waiting {wait:.1f}s to smooth batch startup" + ) + await asyncio.sleep(wait) + + def format_error(exc: BaseException) -> str: message = str(exc).strip() if "Target page, context or browser has been closed" in message: @@ -995,6 +1025,9 @@ class DouyinWorker: logger.info(f"Starting worker loop for account {self.account_id}") try: + await _startup_stagger(self.account_id) + if self.stopping: + return storage_state = await self._load_storage_state() cookie_info = analyze_cookie( json.dumps(storage_state, ensure_ascii=False) if storage_state else None diff --git a/douyin-login-launcher/desktop_app.py b/douyin-login-launcher/desktop_app.py index 742d8b1..c88e7b5 100644 --- a/douyin-login-launcher/desktop_app.py +++ b/douyin-login-launcher/desktop_app.py @@ -1,10 +1,12 @@ """抖音托管客服 · 桌面整合版 一个软件搞定一切: - 1) 启动后默认打开云端网站(dev.zhenyangtang.com.cn); - 2) 页面右下角自动出现「一键本地登录」悬浮按钮; - 3) 点它 → 弹出你的托管账号列表 → 选一个 → 本机直接打开一个 - 已登录该托管账号的浏览器,进入抖音。 + 1) 启动后先显示「站点选择」界面(内置 dev / dev1,可自行添加更多站点, + 可勾选“记住选择”下次直接进入); + 2) 进入站点后,页面右下角自动出现「一键本地登录」悬浮按钮, + 以及「切换站点」按钮可随时换环境; + 3) 点「一键本地登录」→ 弹出你的托管账号列表 → 选一个 → 本机直接打开 + 一个已登录该托管账号的浏览器,进入抖音。 原理:桌面壳(pywebview)把云端网页装进原生窗口,并注入一段脚本。该脚本 用网页里已有的登录令牌(localStorage.kefu_token)调用云端接口取账号与凭证, @@ -18,6 +20,7 @@ from __future__ import annotations import ctypes +import json import os import subprocess import threading @@ -29,10 +32,276 @@ from browser_open import normalize_storage_state, open_logged_in_browser from updater import check_update, download_installer from version import __version__ -CLOUD_URL = "https://dev.zhenyangtang.com.cn/" -CLOUD_HOST = urlparse(CLOUD_URL).hostname or "" WINDOW_TITLE = f"抖音托管客服 · 桌面版 v{__version__}" +# --------------------------------------------------------------------------- +# 多站点配置:内置站点 + 用户自定义站点(保存在用户目录,升级软件不丢失)。 +# 以后要新增内置环境,往 BUILTIN_SITES 里加一行即可。 +# --------------------------------------------------------------------------- +BUILTIN_SITES = [ + {"name": "节点1", "url": "https://dev.zhenyangtang.com.cn/"}, + {"name": "节点2", "url": "https://dev1.zhenyangtang.com.cn/"}, +] +CONFIG_PATH = os.path.join(os.path.expanduser("~"), ".douyin_desktop_config.json") + + +def load_config() -> dict: + try: + with open(CONFIG_PATH, "r", encoding="utf-8") as f: + data = json.load(f) + return data if isinstance(data, dict) else {} + except Exception: # noqa: BLE001 + return {} + + +def save_config(cfg: dict) -> None: + try: + with open(CONFIG_PATH, "w", encoding="utf-8") as f: + json.dump(cfg, f, ensure_ascii=False, indent=2) + except Exception: # noqa: BLE001 + pass + + +def normalize_site_url(url: str) -> str: + url = (url or "").strip() + if not url: + return "" + if not url.startswith(("http://", "https://")): + url = "https://" + url + if not url.endswith("/"): + url += "/" + return url + + +def get_all_sites() -> list[dict]: + """内置站点 + 自定义站点(去重,按 URL)。""" + cfg = load_config() + sites: list[dict] = [] + seen: set[str] = set() + for s in BUILTIN_SITES: + u = normalize_site_url(s["url"]) + sites.append({"name": s["name"], "url": u, "builtin": True}) + seen.add(u) + for s in cfg.get("custom_sites", []): + u = normalize_site_url(s.get("url", "")) + if u and u not in seen: + sites.append({"name": s.get("name") or urlparse(u).hostname, "url": u, "builtin": False}) + seen.add(u) + return sites + + +def site_hosts() -> set[str]: + return {urlparse(s["url"]).hostname or "" for s in get_all_sites()} - {""} + + +# --------------------------------------------------------------------------- +# 站点选择界面(软件启动首页)。与注入浮层同一套视觉语言:#fe2c55 主色 + 玻璃拟态。 +# --------------------------------------------------------------------------- +PICKER_HTML = r""" + + + +选择站点 + + + +
+
+
+ +
+

抖音托管客服 · 桌面版

+
请选择要进入的站点,也可以在下方添加新站点
+
+
+
+
正在加载站点…
+
+ +
+ + + +
+
+
+
+ + + +""" + # 注入到云端页面的脚本:加悬浮按钮 + 账号选择浮层,调用本机桥打开浏览器。 # 设计:Glassmorphism 浮层 + SVG 图标(无 emoji)+ 悬浮过渡 + 可访问性(焦点/Esc/减少动效)。 @@ -47,7 +316,8 @@ INJECT_JS = r""" close: '', chevron: '', monitor: '', - alert: '' + alert: '', + swap: '' }; var ROOT = null; // Shadow root,隔离宿主站点 CSS,避免错位/层级冲突 @@ -91,6 +361,16 @@ INJECT_JS = r""" .${NS}-btn:focus-visible{outline:3px solid rgba(254,44,85,.4);outline-offset:2px;} .${NS}-btn svg{width:18px;height:18px;} + .${NS}-switch{position:fixed;right:24px;bottom:76px;z-index:2147483646;display:inline-flex; + align-items:center;gap:6px;background:rgba(15,23,42,.72);color:#fff; + border:1px solid rgba(255,255,255,.18);border-radius:999px;padding:8px 14px 8px 12px; + font-size:12.5px;font-weight:500;cursor:pointer;backdrop-filter:blur(8px);-webkit-backdrop-filter:blur(8px); + box-shadow:0 6px 18px rgba(15,23,42,.3);transition:background .2s ease,transform .2s ease;} + .${NS}-switch:hover{background:rgba(15,23,42,.88);} + .${NS}-switch:active{transform:translateY(1px);} + .${NS}-switch:focus-visible{outline:2px solid #fe2c55;outline-offset:2px;} + .${NS}-switch svg{width:14px;height:14px;} + .${NS}-mask{position:fixed;inset:0;z-index:2147483647;background:rgba(15,23,42,.45); display:flex;align-items:center;justify-content:center;padding:20px; animation:${NS}-fade .18s ease;} @@ -316,13 +596,25 @@ INJECT_JS = r""" function ensureButton() { var root = ensureRoot(); - if (root.querySelector('.' + NS + '-btn')) return; - var btn = document.createElement('button'); - btn.className = NS + '-btn'; - btn.innerHTML = ICON.login + '一键本地登录'; - btn.title = '在本机打开已登录托管账号的抖音浏览器'; - btn.addEventListener('click', showPanel); - root.appendChild(btn); + if (!root.querySelector('.' + NS + '-btn')) { + var btn = document.createElement('button'); + btn.className = NS + '-btn'; + btn.innerHTML = ICON.login + '一键本地登录'; + btn.title = '在本机打开已登录托管账号的抖音浏览器'; + btn.addEventListener('click', showPanel); + root.appendChild(btn); + } + if (!root.querySelector('.' + NS + '-switch')) { + var sw = document.createElement('button'); + sw.className = NS + '-switch'; + sw.innerHTML = ICON.swap + '切换站点'; + sw.title = '返回站点选择界面'; + sw.addEventListener('click', function () { + var a = api(); + if (a && a.go_picker) a.go_picker(); + }); + root.appendChild(sw); + } } // 供 Python 端回报本地登录错误:右下角弹出可关闭的提示条 @@ -415,6 +707,76 @@ class Api: self._window = None self._update_info: dict | None = None + # ---- 站点选择 ---- + def get_sites(self) -> dict: + cfg = load_config() + return {"sites": get_all_sites(), "remember": bool(cfg.get("default_url"))} + + def add_site(self, name: str, url: str) -> dict: + u = normalize_site_url(url) + host = urlparse(u).hostname if u else None + if not u or not host or "." not in host: + return {"error": "网址格式不对,示例:https://dev1.zhenyangtang.com.cn/", "sites": get_all_sites()} + if any(s["url"] == u for s in get_all_sites()): + return {"error": "该站点已存在", "sites": get_all_sites()} + cfg = load_config() + custom = cfg.get("custom_sites", []) + custom.append({"name": (name or "").strip() or host, "url": u}) + cfg["custom_sites"] = custom + save_config(cfg) + return {"sites": get_all_sites()} + + def remove_site(self, url: str) -> dict: + u = normalize_site_url(url) + cfg = load_config() + cfg["custom_sites"] = [ + s for s in cfg.get("custom_sites", []) if normalize_site_url(s.get("url", "")) != u + ] + if normalize_site_url(cfg.get("default_url", "")) == u: + cfg.pop("default_url", None) + save_config(cfg) + return {"sites": get_all_sites()} + + def ping_site(self, url: str) -> dict: + """测量站点响应时间(毫秒)。服务器有响应即算可达,包括 4xx/5xx。""" + import time + from urllib import error as _err, request as _req + + u = normalize_site_url(url) + if not u: + return {"ok": False} + req = _req.Request(u, method="HEAD", headers={"User-Agent": "Mozilla/5.0"}) + start = time.perf_counter() + try: + with _req.urlopen(req, timeout=8): + pass + except _err.HTTPError: + pass # 服务器已响应(如 403/405),延迟有效 + except Exception: # noqa: BLE001 + return {"ok": False} + return {"ok": True, "ms": int((time.perf_counter() - start) * 1000)} + + def open_site(self, url: str, remember: bool = False) -> dict: + u = normalize_site_url(url) + cfg = load_config() + if remember: + cfg["default_url"] = u + else: + cfg.pop("default_url", None) + save_config(cfg) + if self._window: + self._window.load_url(u) + return {"ok": True} + + def go_picker(self) -> dict: + """从站点页面返回选择界面,并取消“记住选择”。""" + cfg = load_config() + cfg.pop("default_url", None) + save_config(cfg) + if self._window: + self._window.load_html(PICKER_HTML) + return {"ok": True} + # ---- 本地登录 ---- def _notify(self, msg: str, ok: bool = False) -> None: if not self._window: @@ -547,25 +909,39 @@ def _check_update_async(window, api) -> None: def main() -> None: api = Api() - # 直接打开云端主界面,更新检测放到后台线程做,避免启动时同步联网卡住窗口。 - window = webview.create_window( - WINDOW_TITLE, - CLOUD_URL, - js_api=api, - width=1280, - height=860, - text_select=True, - ) + # 上次勾选了“记住选择”则直接进该站点,否则先显示站点选择界面。 + # 更新检测放到后台线程做,避免启动时同步联网卡住窗口。 + cfg = load_config() + default_url = normalize_site_url(cfg.get("default_url", "")) + if default_url and any(s["url"] == default_url for s in get_all_sites()): + window = webview.create_window( + WINDOW_TITLE, + default_url, + js_api=api, + width=1280, + height=860, + text_select=True, + ) + else: + window = webview.create_window( + WINDOW_TITLE, + html=PICKER_HTML, + js_api=api, + width=1280, + height=860, + text_select=True, + ) api._window = window _update_checked = {"done": False} def on_loaded(): - # 仅在云端站点页面注入悬浮按钮脚本;about:blank 等不注入。 + # 仅在已配置站点的页面注入悬浮按钮脚本;选择页/about:blank 等不注入。 try: current = window.get_current_url() or "" except Exception: # noqa: BLE001 current = "" - if CLOUD_HOST and CLOUD_HOST in current: + host = urlparse(current).hostname or "" + if host and host in site_hosts(): try: window.evaluate_js(INJECT_JS) except Exception as e: # noqa: BLE001 diff --git a/douyin-login-launcher/使用说明.txt b/douyin-login-launcher/使用说明.txt index 2fbec2f..814c26b 100644 --- a/douyin-login-launcher/使用说明.txt +++ b/douyin-login-launcher/使用说明.txt @@ -11,10 +11,15 @@ 【推荐:桌面整合版 desktop_app.py】——一个软件搞定 ==================================================================== 效果: - 打开软件 → 默认加载云端网站 https://dev.zhenyangtang.com.cn/ - → 你在里面正常登录后台 - → 页面右下角出现「🚀 一键本地登录」按钮 - → 点它 → 弹出托管账号列表 → 选一个 + 打开软件 → 先显示「站点选择」界面,内置两个站点: + - 节点1 https://dev.zhenyangtang.com.cn/ + - 节点2 https://dev1.zhenyangtang.com.cn/ + 以后有新环境,可直接在界面下方填名称+网址「添加」(保存在本机,升级不丢)。 + 勾选「记住选择」后,下次启动直接进入该站点,跳过选择页。 + → 选一个站点进入 → 你在里面正常登录后台 + → 页面右下角出现「一键本地登录」按钮(上方还有「切换站点」按钮, + 点击可随时返回站点选择界面,并自动取消“记住选择”) + → 点「一键本地登录」→ 弹出托管账号列表 → 选一个 → 本机直接打开一个“已登录该托管账号”的浏览器进入抖音。 运行方式一(推荐,无黑窗):