更新
This commit is contained in:
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""用户可添加抖音账号数量限制(限制功能已移除,保留接口兼容)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import Account, User
|
||||
from .roles import is_admin
|
||||
|
||||
UNLIMITED_ACCOUNTS = -1
|
||||
|
||||
|
||||
def normalize_max_accounts(value: int | None, role: str = "operator") -> int:
|
||||
if is_admin(role):
|
||||
return UNLIMITED_ACCOUNTS
|
||||
if value is None:
|
||||
return 3
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 3
|
||||
if parsed < 0:
|
||||
return UNLIMITED_ACCOUNTS
|
||||
return parsed
|
||||
|
||||
|
||||
def account_limit_for_user(user: User) -> int | None:
|
||||
"""账号数量/并发限制已移除,始终不限制。"""
|
||||
return None
|
||||
|
||||
|
||||
async def count_user_accounts(db: AsyncSession, user_id: int) -> int:
|
||||
result = await db.execute(
|
||||
select(func.count()).select_from(Account).where(Account.owner_id == user_id)
|
||||
)
|
||||
return int(result.scalar() or 0)
|
||||
|
||||
|
||||
async def count_user_account_breakdown(db: AsyncSession, user_id: int) -> dict[str, int]:
|
||||
"""统计用户名下托管账号:总数 / 可用 / 额度停用。"""
|
||||
total = await count_user_accounts(db, user_id)
|
||||
if total <= 0:
|
||||
return {"total": 0, "active": 0, "disabled": 0}
|
||||
disabled_result = await db.execute(
|
||||
select(func.count())
|
||||
.select_from(Account)
|
||||
.where(Account.owner_id == user_id, Account.quota_disabled.is_(True))
|
||||
)
|
||||
disabled = int(disabled_result.scalar() or 0)
|
||||
active = max(0, total - disabled)
|
||||
return {"total": total, "active": active, "disabled": disabled}
|
||||
|
||||
|
||||
async def ensure_can_add_account(db: AsyncSession, user: User) -> None:
|
||||
"""账号数量限制已移除,任何用户可添加任意数量账号。"""
|
||||
return
|
||||
@@ -0,0 +1,68 @@
|
||||
"""账号额度与 quota_disabled 同步。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import Account, User
|
||||
from .account_limits import account_limit_for_user
|
||||
|
||||
QUOTA_DISABLED_MESSAGE = "账号额度不足,该账号已被系统停用"
|
||||
|
||||
|
||||
async def sync_user_account_quota(
|
||||
db: AsyncSession,
|
||||
user: User,
|
||||
*,
|
||||
stop_worker: Callable[[int], Awaitable[None]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""按 max_accounts 同步停用状态:保留最早创建的账号,超额账号禁用并停止托管。"""
|
||||
limit = account_limit_for_user(user)
|
||||
|
||||
result = await db.execute(
|
||||
select(Account)
|
||||
.where(Account.owner_id == user.id)
|
||||
.order_by(Account.created_at.asc(), Account.id.asc())
|
||||
)
|
||||
accounts = list(result.scalars().all())
|
||||
|
||||
disabled_ids: list[int] = []
|
||||
enabled_ids: list[int] = []
|
||||
|
||||
if limit is None:
|
||||
for acc in accounts:
|
||||
if acc.quota_disabled:
|
||||
acc.quota_disabled = False
|
||||
if (acc.error_message or "").strip() == QUOTA_DISABLED_MESSAGE:
|
||||
acc.error_message = None
|
||||
enabled_ids.append(acc.id)
|
||||
return {"disabled_ids": disabled_ids, "enabled_ids": enabled_ids, "limit": None}
|
||||
|
||||
for idx, acc in enumerate(accounts):
|
||||
if idx < limit:
|
||||
if acc.quota_disabled:
|
||||
acc.quota_disabled = False
|
||||
if (acc.error_message or "").strip() == QUOTA_DISABLED_MESSAGE:
|
||||
acc.error_message = None
|
||||
enabled_ids.append(acc.id)
|
||||
else:
|
||||
if not acc.quota_disabled:
|
||||
acc.quota_disabled = True
|
||||
acc.error_message = QUOTA_DISABLED_MESSAGE
|
||||
acc.status = "offline"
|
||||
acc.qr_code_base64 = None
|
||||
disabled_ids.append(acc.id)
|
||||
if stop_worker:
|
||||
await stop_worker(acc.id)
|
||||
|
||||
return {"disabled_ids": disabled_ids, "enabled_ids": enabled_ids, "limit": limit}
|
||||
|
||||
|
||||
async def default_stop_worker(account_id: int) -> None:
|
||||
from main import manager
|
||||
|
||||
await manager.stop_worker(account_id)
|
||||
@@ -0,0 +1,50 @@
|
||||
from fastapi import Depends, HTTPException, status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.database import get_db
|
||||
from models.models import User
|
||||
from .jwt_utils import decode_access_token
|
||||
from .roles import can_manage_users, can_write, is_admin
|
||||
|
||||
bearer_scheme = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
async def get_current_user(
|
||||
credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> User:
|
||||
if not credentials or not credentials.credentials:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="未登录或令牌缺失")
|
||||
payload = decode_access_token(credentials.credentials)
|
||||
if not payload or not payload.get("sub"):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="登录已过期,请重新登录")
|
||||
try:
|
||||
user_id = int(payload["sub"])
|
||||
except (TypeError, ValueError):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="无效令牌")
|
||||
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user or not user.is_active:
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户不存在或已禁用")
|
||||
return user
|
||||
|
||||
|
||||
async def require_admin(user: User = Depends(get_current_user)) -> User:
|
||||
if not is_admin(user.role):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="需要管理员权限")
|
||||
return user
|
||||
|
||||
|
||||
async def require_write(user: User = Depends(get_current_user)) -> User:
|
||||
if not can_write(user.role):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="当前角色只读,无法执行此操作")
|
||||
return user
|
||||
|
||||
|
||||
async def require_user_manager(user: User = Depends(get_current_user)) -> User:
|
||||
if not can_manage_users(user.role):
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="需要管理员权限")
|
||||
return user
|
||||
@@ -0,0 +1,378 @@
|
||||
"""发送邮箱验证邮件(标准库 SMTP)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import smtplib
|
||||
import ssl
|
||||
from dataclasses import asdict
|
||||
from email.mime.multipart import MIMEMultipart
|
||||
from email.mime.text import MIMEText
|
||||
from typing import Any
|
||||
|
||||
from .email_templates import build_password_reset_content, build_test_email_content, build_verify_email_content
|
||||
from .system_settings import SystemSettingsData
|
||||
|
||||
logger = logging.getLogger("auth.email")
|
||||
|
||||
SMTP_TIMEOUT_SECONDS = 45
|
||||
# 阿里企业邮自定义域名 SMTP(如 smtp.xxiaw.com)常出现 535,官方地址可正常认证
|
||||
ALIBABA_SMTP_OFFICIAL = "smtp.mxhichina.com"
|
||||
_SSL_CONTEXT = ssl.create_default_context()
|
||||
|
||||
|
||||
def _ssl_context(*, insecure: bool = False) -> ssl.SSLContext:
|
||||
if not insecure:
|
||||
return _SSL_CONTEXT
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
return ctx
|
||||
|
||||
|
||||
def _connect_ssl(host: str, port: int) -> smtplib.SMTP:
|
||||
try:
|
||||
return smtplib.SMTP_SSL(
|
||||
host,
|
||||
port,
|
||||
timeout=SMTP_TIMEOUT_SECONDS,
|
||||
context=_ssl_context(insecure=False),
|
||||
)
|
||||
except ssl.SSLCertVerificationError:
|
||||
logger.warning(
|
||||
"SMTP SSL certificate hostname mismatch for %s, retry without verify",
|
||||
host,
|
||||
)
|
||||
return smtplib.SMTP_SSL(
|
||||
host,
|
||||
port,
|
||||
timeout=SMTP_TIMEOUT_SECONDS,
|
||||
context=_ssl_context(insecure=True),
|
||||
)
|
||||
|
||||
|
||||
def _starttls(server: smtplib.SMTP, host: str) -> None:
|
||||
try:
|
||||
server.starttls(context=_ssl_context(insecure=False))
|
||||
except ssl.SSLCertVerificationError:
|
||||
logger.warning(
|
||||
"SMTP STARTTLS certificate hostname mismatch for %s, retry without verify",
|
||||
host,
|
||||
)
|
||||
server.starttls(context=_ssl_context(insecure=True))
|
||||
|
||||
|
||||
def build_verification_link(token: str, settings: SystemSettingsData) -> str:
|
||||
return f"{settings.app_url_normalized()}/#/login?verify_token={token}"
|
||||
|
||||
|
||||
def build_password_reset_link(token: str, settings: SystemSettingsData) -> str:
|
||||
return f"{settings.app_url_normalized()}/#/login?reset_token={token}"
|
||||
|
||||
|
||||
def resolve_smtp_mode(settings: SystemSettingsData) -> tuple[int, bool, bool]:
|
||||
"""返回 (port, use_ssl, use_starttls)。"""
|
||||
port = int(settings.smtp_port or 587)
|
||||
use_ssl = bool(settings.smtp_use_ssl) or port == 465
|
||||
# 465 只能 implicit SSL,绝不能 plain + STARTTLS
|
||||
if port == 465:
|
||||
use_ssl = True
|
||||
use_starttls = False
|
||||
elif port == 587:
|
||||
use_ssl = False
|
||||
use_starttls = bool(settings.smtp_use_tls)
|
||||
else:
|
||||
use_ssl = bool(settings.smtp_use_ssl)
|
||||
use_starttls = bool(settings.smtp_use_tls) and not use_ssl
|
||||
return port, use_ssl, use_starttls
|
||||
|
||||
|
||||
def _open_smtp(settings: SystemSettingsData) -> smtplib.SMTP:
|
||||
host = settings.smtp_host.strip()
|
||||
port, use_ssl, use_starttls = resolve_smtp_mode(settings)
|
||||
logger.info(
|
||||
"SMTP connect host=%s port=%s ssl=%s starttls=%s user=%s",
|
||||
host,
|
||||
port,
|
||||
use_ssl,
|
||||
use_starttls,
|
||||
settings.smtp_user or "",
|
||||
)
|
||||
|
||||
if use_ssl:
|
||||
server = _connect_ssl(host, port)
|
||||
server.ehlo()
|
||||
return server
|
||||
|
||||
server = smtplib.SMTP(host, port, timeout=SMTP_TIMEOUT_SECONDS)
|
||||
server.ehlo()
|
||||
if use_starttls:
|
||||
_starttls(server, host)
|
||||
server.ehlo()
|
||||
return server
|
||||
|
||||
|
||||
def _close_smtp(server: smtplib.SMTP) -> None:
|
||||
try:
|
||||
server.quit()
|
||||
except Exception:
|
||||
try:
|
||||
server.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _login_smtp(server: smtplib.SMTP, settings: SystemSettingsData) -> tuple[smtplib.SMTP, bool]:
|
||||
"""登录 SMTP,返回 (server, 是否使用了阿里官方地址回退)。"""
|
||||
user = (settings.smtp_user or "").strip()
|
||||
pwd = settings.smtp_password
|
||||
if not user:
|
||||
return server, False
|
||||
if not pwd:
|
||||
raise RuntimeError("未填写 SMTP 密码,请在设置中填写登录密码或客户端安全密码后重试")
|
||||
try:
|
||||
server.login(user, pwd)
|
||||
return server, False
|
||||
except smtplib.SMTPAuthenticationError as exc:
|
||||
host = (settings.smtp_host or "").strip().lower()
|
||||
if host == ALIBABA_SMTP_OFFICIAL:
|
||||
raise RuntimeError(_format_login_error(exc, settings)) from exc
|
||||
logger.warning(
|
||||
"SMTP auth failed on %s (%s), retrying via %s",
|
||||
host,
|
||||
exc,
|
||||
ALIBABA_SMTP_OFFICIAL,
|
||||
)
|
||||
_close_smtp(server)
|
||||
fallback = SystemSettingsData(**{**asdict(settings), "smtp_host": ALIBABA_SMTP_OFFICIAL})
|
||||
fallback_server = _open_smtp(fallback)
|
||||
try:
|
||||
fallback_server.login(user, pwd)
|
||||
except Exception as retry_exc:
|
||||
_close_smtp(fallback_server)
|
||||
raise RuntimeError(_format_login_error(exc, settings)) from retry_exc
|
||||
return fallback_server, True
|
||||
|
||||
|
||||
def _format_login_error(exc: Exception, settings: SystemSettingsData) -> str:
|
||||
if isinstance(exc, smtplib.SMTPAuthenticationError):
|
||||
return _format_smtp_error(exc, settings)
|
||||
if isinstance(exc, smtplib.SMTPServerDisconnected):
|
||||
return (
|
||||
"SMTP 服务器在登录时断开连接,通常不是加密方式问题,而是密码错误或账号未授权 SMTP。"
|
||||
f"请确认用户名 {settings.smtp_user or ''} 与密码/客户端安全密码正确,"
|
||||
"并在阿里企业邮后台开启「允许使用第三方客户端」。"
|
||||
"若使用自定义域名 SMTP(如 smtp.xxiaw.com)仍失败,可改用 smtp.mxhichina.com 后重试。"
|
||||
)
|
||||
return _format_smtp_error(exc, settings)
|
||||
|
||||
|
||||
def _format_smtp_error(exc: Exception, settings: SystemSettingsData) -> str:
|
||||
msg = str(exc).strip() or exc.__class__.__name__
|
||||
port, use_ssl, use_starttls = resolve_smtp_mode(settings)
|
||||
hints: list[str] = []
|
||||
|
||||
if isinstance(exc, ssl.SSLCertVerificationError):
|
||||
hints.append(
|
||||
f"SSL 证书域名与 {settings.smtp_host} 不匹配(阿里企业邮自定义域名常见),"
|
||||
"系统已尝试跳过证书校验;若仍失败请改用 smtp.mxhichina.com"
|
||||
)
|
||||
return f"{msg}。{' '.join(hints)}"
|
||||
|
||||
if isinstance(exc, smtplib.SMTPAuthenticationError):
|
||||
host = (settings.smtp_host or "").strip().lower()
|
||||
if host != ALIBABA_SMTP_OFFICIAL and host.startswith("smtp."):
|
||||
hints.append(
|
||||
f"若使用自定义域名 SMTP({settings.smtp_host}),阿里企业邮常返回 535;"
|
||||
f"请将 SMTP 服务器改为 {ALIBABA_SMTP_OFFICIAL} 后重试"
|
||||
)
|
||||
hints.append(
|
||||
"SMTP 登录失败:请确认登录用户名为完整邮箱地址,"
|
||||
"密码为邮箱登录密码或阿里企业邮「客户端安全密码」(非网页登录密码时需在邮箱设置中单独生成)"
|
||||
)
|
||||
hints.append("请在阿里企业邮管理后台确认已开启「允许使用第三方客户端」")
|
||||
return f"{msg}。{' '.join(hints)}"
|
||||
|
||||
lower = msg.lower()
|
||||
if "connection unexpectedly closed" in lower and port == 465 and not use_ssl:
|
||||
hints.append("465 端口必须使用 SSL 加密,不能勾选 STARTTLS")
|
||||
elif "connection unexpectedly closed" in lower and port == 465:
|
||||
if use_ssl:
|
||||
hints.append(
|
||||
"连接已建立但在后续步骤失败:请重点检查 SMTP 密码/客户端安全密码,"
|
||||
"或尝试将服务器改为 smtp.mxhichina.com"
|
||||
)
|
||||
else:
|
||||
hints.append("465 端口必须使用 SSL 加密,请选择「SSL / TLS(端口 465)」")
|
||||
if "timed out" in lower or "timeout" in lower or "10060" in lower:
|
||||
if port == 587:
|
||||
hints.append(
|
||||
f"587 端口 STARTTLS 连接 {settings.smtp_host} 超时:"
|
||||
"该服务器可能仅支持 465 SSL,请将端口改为 465 并选择 SSL 加密"
|
||||
)
|
||||
else:
|
||||
hints.append(
|
||||
f"无法在 {SMTP_TIMEOUT_SECONDS}s 内连接 {settings.smtp_host}:{port},"
|
||||
"请检查服务器地址、端口、防火墙/安全组是否放行出站 SMTP"
|
||||
)
|
||||
if "authentication" in lower or "535" in lower:
|
||||
hints.append("认证失败:请检查 SMTP 密码/客户端安全密码是否正确")
|
||||
|
||||
if hints:
|
||||
return f"{msg}。{' '.join(hints)}"
|
||||
return msg
|
||||
|
||||
|
||||
def diagnose_smtp(settings: SystemSettingsData) -> dict[str, Any]:
|
||||
"""分步检测 SMTP 连接,便于定位配置问题。"""
|
||||
host = (settings.smtp_host or "").strip()
|
||||
port, use_ssl, use_starttls = resolve_smtp_mode(settings)
|
||||
result: dict[str, Any] = {
|
||||
"host": host,
|
||||
"port": port,
|
||||
"mode": "ssl" if use_ssl else ("starttls" if use_starttls else "plain"),
|
||||
"steps": [],
|
||||
"ok": False,
|
||||
}
|
||||
if not host:
|
||||
result["steps"].append({"step": "config", "ok": False, "message": "未填写 SMTP 服务器"})
|
||||
return result
|
||||
|
||||
server: smtplib.SMTP | None = None
|
||||
try:
|
||||
server = _open_smtp(settings)
|
||||
result["steps"].append({"step": "connect", "ok": True, "message": "连接成功"})
|
||||
except Exception as exc:
|
||||
result["steps"].append(
|
||||
{"step": "connect", "ok": False, "message": _format_smtp_error(exc, settings)}
|
||||
)
|
||||
return result
|
||||
|
||||
try:
|
||||
if settings.smtp_user:
|
||||
if not settings.smtp_password:
|
||||
raise RuntimeError("未填写 SMTP 密码,请在设置中填写登录密码或客户端安全密码后重试")
|
||||
host_before = (settings.smtp_host or "").strip().lower()
|
||||
server, used_fallback = _login_smtp(server, settings)
|
||||
login_msg = "登录成功"
|
||||
if used_fallback and host_before != ALIBABA_SMTP_OFFICIAL:
|
||||
login_msg = (
|
||||
f"登录成功(已通过 {ALIBABA_SMTP_OFFICIAL} 认证,"
|
||||
f"建议将 SMTP 服务器改为 {ALIBABA_SMTP_OFFICIAL})"
|
||||
)
|
||||
result["steps"].append({"step": "login", "ok": True, "message": login_msg})
|
||||
else:
|
||||
result["steps"].append({"step": "login", "ok": True, "message": "未配置用户名,跳过登录"})
|
||||
result["ok"] = True
|
||||
except Exception as exc:
|
||||
result["steps"].append(
|
||||
{"step": "login", "ok": False, "message": _format_login_error(exc, settings)}
|
||||
)
|
||||
finally:
|
||||
if server:
|
||||
try:
|
||||
server.quit()
|
||||
except Exception:
|
||||
server.close()
|
||||
return result
|
||||
|
||||
|
||||
def _send_sync(
|
||||
settings: SystemSettingsData,
|
||||
to_email: str,
|
||||
subject: str,
|
||||
html_body: str,
|
||||
text_body: str,
|
||||
) -> None:
|
||||
if not settings.smtp_configured():
|
||||
raise RuntimeError("SMTP 未配置,无法发送邮件")
|
||||
|
||||
smtp_from = (settings.smtp_from or settings.smtp_user or "").strip()
|
||||
msg = MIMEMultipart("alternative")
|
||||
msg["Subject"] = subject
|
||||
msg["From"] = smtp_from
|
||||
msg["To"] = to_email
|
||||
msg.attach(MIMEText(text_body, "plain", "utf-8"))
|
||||
msg.attach(MIMEText(html_body, "html", "utf-8"))
|
||||
|
||||
server = _open_smtp(settings)
|
||||
try:
|
||||
if settings.smtp_user:
|
||||
try:
|
||||
server, _ = _login_smtp(server, settings)
|
||||
except RuntimeError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RuntimeError(_format_login_error(exc, settings)) from exc
|
||||
server.sendmail(smtp_from, [to_email], msg.as_string())
|
||||
except RuntimeError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise RuntimeError(_format_smtp_error(exc, settings)) from exc
|
||||
finally:
|
||||
try:
|
||||
server.quit()
|
||||
except Exception:
|
||||
server.close()
|
||||
|
||||
|
||||
async def send_password_reset_email(
|
||||
to_email: str,
|
||||
username: str,
|
||||
token: str,
|
||||
settings: SystemSettingsData,
|
||||
) -> str:
|
||||
"""发送密码重置邮件,返回重置链接。"""
|
||||
link = build_password_reset_link(token, settings)
|
||||
expire_hours = int(settings.email_verify_token_hours or 24)
|
||||
subject, text_body, html_body = build_password_reset_content(
|
||||
settings, username, to_email, link, expire_hours
|
||||
)
|
||||
|
||||
if settings.smtp_configured():
|
||||
await asyncio.to_thread(_send_sync, settings, to_email, subject, html_body, text_body)
|
||||
logger.info("Password reset email sent to %s", to_email)
|
||||
else:
|
||||
logger.warning(
|
||||
"SMTP 未配置,密码重置链接: user=%s email=%s link=%s",
|
||||
username,
|
||||
to_email,
|
||||
link,
|
||||
)
|
||||
return link
|
||||
|
||||
|
||||
async def send_verification_email(
|
||||
to_email: str,
|
||||
username: str,
|
||||
token: str,
|
||||
settings: SystemSettingsData,
|
||||
) -> str:
|
||||
"""发送验证邮件,返回验证链接。"""
|
||||
link = build_verification_link(token, settings)
|
||||
subject, text_body, html_body = build_verify_email_content(
|
||||
settings, username, to_email, link
|
||||
)
|
||||
|
||||
if settings.smtp_configured():
|
||||
await asyncio.to_thread(_send_sync, settings, to_email, subject, html_body, text_body)
|
||||
logger.info("Verification email sent to %s", to_email)
|
||||
else:
|
||||
logger.warning(
|
||||
"SMTP 未配置,验证链接: user=%s email=%s link=%s",
|
||||
username,
|
||||
to_email,
|
||||
link,
|
||||
)
|
||||
return link
|
||||
|
||||
|
||||
async def send_test_email(to_email: str, settings: SystemSettingsData) -> None:
|
||||
diagnosis = await asyncio.to_thread(diagnose_smtp, settings)
|
||||
if not diagnosis.get("ok"):
|
||||
failed = next((s for s in diagnosis.get("steps", []) if not s.get("ok")), None)
|
||||
raise RuntimeError(failed["message"] if failed else "SMTP 连接检测失败")
|
||||
|
||||
subject, text_body, html_body = build_test_email_content(settings, to_email)
|
||||
await asyncio.to_thread(_send_sync, settings, to_email, subject, html_body, text_body)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""邮件模板渲染。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
|
||||
from .system_settings import (
|
||||
APP_NAME,
|
||||
DEFAULT_EMAIL_TEST_BODY,
|
||||
DEFAULT_EMAIL_TEST_HTML,
|
||||
DEFAULT_EMAIL_TEST_SUBJECT,
|
||||
DEFAULT_EMAIL_VERIFY_BODY,
|
||||
DEFAULT_EMAIL_VERIFY_HTML,
|
||||
DEFAULT_EMAIL_VERIFY_SUBJECT,
|
||||
DEFAULT_PASSWORD_RESET_BODY,
|
||||
DEFAULT_PASSWORD_RESET_HTML,
|
||||
DEFAULT_PASSWORD_RESET_SUBJECT,
|
||||
SystemSettingsData,
|
||||
)
|
||||
|
||||
_TEMPLATE_VAR_PATTERN = re.compile(r"\{(\w+)\}")
|
||||
|
||||
|
||||
def render_email_template(template: str, **context: str) -> str:
|
||||
"""安全渲染邮件模板,未知占位符保留原样。"""
|
||||
value = template or ""
|
||||
|
||||
def replacer(match: re.Match) -> str:
|
||||
key = match.group(1)
|
||||
return context.get(key, match.group(0))
|
||||
|
||||
return _TEMPLATE_VAR_PATTERN.sub(replacer, value)
|
||||
|
||||
|
||||
def _text_to_html(text: str) -> str:
|
||||
escaped = html.escape(text or "")
|
||||
return f'<pre style="font-family:sans-serif;white-space:pre-wrap;margin:0;">{escaped}</pre>'
|
||||
|
||||
|
||||
def build_verify_email_content(
|
||||
settings: SystemSettingsData,
|
||||
username: str,
|
||||
to_email: str,
|
||||
link: str,
|
||||
) -> tuple[str, str, str]:
|
||||
context = {
|
||||
"username": username,
|
||||
"email": to_email,
|
||||
"link": link,
|
||||
"app_name": APP_NAME,
|
||||
}
|
||||
subject_tpl = settings.email_verify_subject or DEFAULT_EMAIL_VERIFY_SUBJECT
|
||||
body_tpl = settings.email_verify_body or DEFAULT_EMAIL_VERIFY_BODY
|
||||
html_tpl = settings.email_verify_html or ""
|
||||
|
||||
subject = render_email_template(subject_tpl, **context)
|
||||
text_body = render_email_template(body_tpl, **context)
|
||||
if html_tpl.strip():
|
||||
html_body = render_email_template(html_tpl, **context)
|
||||
else:
|
||||
html_body = _text_to_html(text_body)
|
||||
return subject, text_body, html_body
|
||||
|
||||
|
||||
def build_password_reset_content(
|
||||
settings: SystemSettingsData,
|
||||
username: str,
|
||||
to_email: str,
|
||||
link: str,
|
||||
expire_hours: int,
|
||||
) -> tuple[str, str, str]:
|
||||
context = {
|
||||
"username": username,
|
||||
"email": to_email,
|
||||
"link": link,
|
||||
"app_name": APP_NAME,
|
||||
"expire_hours": str(expire_hours),
|
||||
}
|
||||
subject = render_email_template(DEFAULT_PASSWORD_RESET_SUBJECT, **context)
|
||||
text_body = render_email_template(DEFAULT_PASSWORD_RESET_BODY, **context)
|
||||
html_body = render_email_template(DEFAULT_PASSWORD_RESET_HTML, **context)
|
||||
return subject, text_body, html_body
|
||||
|
||||
|
||||
def build_test_email_content(settings: SystemSettingsData, to_email: str) -> tuple[str, str, str]:
|
||||
context = {
|
||||
"username": "测试用户",
|
||||
"email": to_email,
|
||||
"link": settings.app_url_normalized(),
|
||||
"app_name": APP_NAME,
|
||||
}
|
||||
subject_tpl = settings.email_test_subject or DEFAULT_EMAIL_TEST_SUBJECT
|
||||
body_tpl = settings.email_test_body or DEFAULT_EMAIL_TEST_BODY
|
||||
html_tpl = settings.email_test_html or ""
|
||||
|
||||
subject = render_email_template(subject_tpl, **context)
|
||||
text_body = render_email_template(body_tpl, **context)
|
||||
if html_tpl.strip():
|
||||
html_body = render_email_template(html_tpl, **context)
|
||||
else:
|
||||
html_body = _text_to_html(text_body)
|
||||
return subject, text_body, html_body
|
||||
@@ -0,0 +1,78 @@
|
||||
"""邮箱验证令牌创建与校验。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import EmailVerificationToken, User
|
||||
|
||||
|
||||
def _token_expires_at(hours: int) -> datetime:
|
||||
return datetime.utcnow() + timedelta(hours=max(1, min(168, int(hours or 24))))
|
||||
|
||||
|
||||
async def invalidate_user_tokens(db: AsyncSession, user_id: int) -> None:
|
||||
await db.execute(
|
||||
delete(EmailVerificationToken).where(EmailVerificationToken.user_id == user_id)
|
||||
)
|
||||
|
||||
|
||||
async def create_verification_token(
|
||||
db: AsyncSession,
|
||||
user: User,
|
||||
expire_hours: int = 24,
|
||||
) -> str:
|
||||
await invalidate_user_tokens(db, user.id)
|
||||
token = secrets.token_urlsafe(32)
|
||||
db.add(
|
||||
EmailVerificationToken(
|
||||
user_id=user.id,
|
||||
token=token,
|
||||
expires_at=_token_expires_at(expire_hours),
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
return token
|
||||
|
||||
|
||||
async def verify_email_token(db: AsyncSession, token: str) -> User | None:
|
||||
value = (token or "").strip()
|
||||
if not value:
|
||||
return None
|
||||
|
||||
result = await db.execute(
|
||||
select(EmailVerificationToken, User)
|
||||
.join(User, User.id == EmailVerificationToken.user_id)
|
||||
.where(EmailVerificationToken.token == value)
|
||||
)
|
||||
row = result.first()
|
||||
if not row:
|
||||
return None
|
||||
|
||||
record, user = row
|
||||
if record.expires_at < datetime.utcnow():
|
||||
await db.delete(record)
|
||||
await db.flush()
|
||||
return None
|
||||
|
||||
user.email_verified = True
|
||||
user.email_verified_at = datetime.utcnow()
|
||||
await db.delete(record)
|
||||
await db.flush()
|
||||
return user
|
||||
|
||||
|
||||
def mask_email(email: str) -> str:
|
||||
value = (email or "").strip()
|
||||
if "@" not in value:
|
||||
return value
|
||||
local, domain = value.split("@", 1)
|
||||
if len(local) <= 2:
|
||||
masked_local = local[0] + "*"
|
||||
else:
|
||||
masked_local = local[0] + "*" * (len(local) - 2) + local[-1]
|
||||
return f"{masked_local}@{domain}"
|
||||
@@ -0,0 +1,24 @@
|
||||
import os
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Optional
|
||||
|
||||
from jose import JWTError, jwt
|
||||
|
||||
SECRET_KEY = os.getenv("KEFU_SECRET_KEY", "kefu-dev-secret-change-in-production")
|
||||
ALGORITHM = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES = int(os.getenv("KEFU_TOKEN_EXPIRE_MINUTES", str(60 * 24)))
|
||||
|
||||
|
||||
def create_access_token(subject: str, extra: Optional[dict[str, Any]] = None) -> str:
|
||||
expire = datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
payload = {"sub": subject, "exp": expire}
|
||||
if extra:
|
||||
payload.update(extra)
|
||||
return jwt.encode(payload, SECRET_KEY, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def decode_access_token(token: str) -> Optional[dict[str, Any]]:
|
||||
try:
|
||||
return jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
|
||||
except JWTError:
|
||||
return None
|
||||
@@ -0,0 +1,64 @@
|
||||
"""密码重置令牌创建与校验。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import PasswordResetToken, User
|
||||
|
||||
|
||||
def _token_expires_at(hours: int) -> datetime:
|
||||
return datetime.utcnow() + timedelta(hours=max(1, min(168, int(hours or 24))))
|
||||
|
||||
|
||||
async def invalidate_user_reset_tokens(db: AsyncSession, user_id: int) -> None:
|
||||
await db.execute(
|
||||
delete(PasswordResetToken).where(PasswordResetToken.user_id == user_id)
|
||||
)
|
||||
|
||||
|
||||
async def create_password_reset_token(
|
||||
db: AsyncSession,
|
||||
user: User,
|
||||
expire_hours: int = 24,
|
||||
) -> str:
|
||||
await invalidate_user_reset_tokens(db, user.id)
|
||||
token = secrets.token_urlsafe(32)
|
||||
db.add(
|
||||
PasswordResetToken(
|
||||
user_id=user.id,
|
||||
token=token,
|
||||
expires_at=_token_expires_at(expire_hours),
|
||||
)
|
||||
)
|
||||
await db.flush()
|
||||
return token
|
||||
|
||||
|
||||
async def verify_password_reset_token(db: AsyncSession, token: str) -> User | None:
|
||||
value = (token or "").strip()
|
||||
if not value:
|
||||
return None
|
||||
|
||||
result = await db.execute(
|
||||
select(PasswordResetToken, User)
|
||||
.join(User, User.id == PasswordResetToken.user_id)
|
||||
.where(PasswordResetToken.token == value)
|
||||
)
|
||||
row = result.first()
|
||||
if not row:
|
||||
return None
|
||||
|
||||
record, user = row
|
||||
if record.expires_at < datetime.utcnow():
|
||||
await db.delete(record)
|
||||
await db.flush()
|
||||
return None
|
||||
|
||||
await db.delete(record)
|
||||
await db.flush()
|
||||
return user
|
||||
@@ -0,0 +1,11 @@
|
||||
from passlib.context import CryptContext
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def verify_password(plain: str, hashed: str) -> bool:
|
||||
return pwd_context.verify(plain, hashed)
|
||||
@@ -0,0 +1,31 @@
|
||||
from typing import Iterable
|
||||
|
||||
ROLE_ADMIN = "admin"
|
||||
ROLE_OPERATOR = "operator"
|
||||
ROLE_VIEWER = "viewer"
|
||||
|
||||
ALL_ROLES = (ROLE_ADMIN, ROLE_OPERATOR, ROLE_VIEWER)
|
||||
|
||||
ROLE_LABELS = {
|
||||
ROLE_ADMIN: "管理员",
|
||||
ROLE_OPERATOR: "运营",
|
||||
ROLE_VIEWER: "只读",
|
||||
}
|
||||
|
||||
|
||||
def is_admin(role: str) -> bool:
|
||||
return role == ROLE_ADMIN
|
||||
|
||||
|
||||
def can_write(role: str) -> bool:
|
||||
return role in (ROLE_ADMIN, ROLE_OPERATOR)
|
||||
|
||||
|
||||
def can_manage_users(role: str) -> bool:
|
||||
return role == ROLE_ADMIN
|
||||
|
||||
|
||||
def ensure_role(role: str) -> str:
|
||||
if role not in ALL_ROLES:
|
||||
raise ValueError(f"无效角色: {role}")
|
||||
return role
|
||||
@@ -0,0 +1,511 @@
|
||||
import logging
|
||||
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, status
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.database import get_db
|
||||
from models.models import User
|
||||
from .account_limits import (
|
||||
UNLIMITED_ACCOUNTS,
|
||||
count_user_account_breakdown,
|
||||
normalize_max_accounts,
|
||||
)
|
||||
from .account_quota import default_stop_worker, sync_user_account_quota
|
||||
from .dependencies import get_current_user, require_user_manager
|
||||
from .email_service import (
|
||||
build_password_reset_link,
|
||||
build_verification_link,
|
||||
send_password_reset_email,
|
||||
send_verification_email,
|
||||
)
|
||||
from .email_verification import create_verification_token, mask_email, verify_email_token
|
||||
from .password_reset import create_password_reset_token, verify_password_reset_token
|
||||
from .jwt_utils import create_access_token
|
||||
from .passwords import hash_password, verify_password
|
||||
from .roles import ALL_ROLES, ROLE_LABELS, ROLE_OPERATOR, ensure_role, is_admin
|
||||
from .schemas import (
|
||||
LoginRequest,
|
||||
MessageResponse,
|
||||
ForgotPasswordRequest,
|
||||
ForgotPasswordResponse,
|
||||
RegisterRequest,
|
||||
RegisterResponse,
|
||||
ResendVerificationRequest,
|
||||
ResetPasswordRequest,
|
||||
RolesResponse,
|
||||
RoleInfo,
|
||||
TokenResponse,
|
||||
UserCreate,
|
||||
UserResponse,
|
||||
UserUpdate,
|
||||
VerifyEmailRequest,
|
||||
)
|
||||
from .system_settings import SystemSettingsData, load_settings
|
||||
|
||||
logger = logging.getLogger("auth.router")
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["auth"])
|
||||
|
||||
|
||||
async def _build_user_response(db: AsyncSession, user: User, with_count: bool = False) -> UserResponse:
|
||||
payload = UserResponse.model_validate(user)
|
||||
if with_count:
|
||||
breakdown = await count_user_account_breakdown(db, user.id)
|
||||
payload.account_count = breakdown["total"]
|
||||
payload.active_account_count = breakdown["active"]
|
||||
payload.disabled_account_count = breakdown["disabled"]
|
||||
return payload
|
||||
|
||||
|
||||
async def _ensure_email_available(
|
||||
db: AsyncSession,
|
||||
email: str | None,
|
||||
exclude_user_id: int | None = None,
|
||||
) -> str | None:
|
||||
value = (str(email).strip().lower() if email else "") or None
|
||||
if not value:
|
||||
return None
|
||||
stmt = select(User).where(User.email == value)
|
||||
if exclude_user_id:
|
||||
stmt = stmt.where(User.id != exclude_user_id)
|
||||
exists = await db.execute(stmt)
|
||||
if exists.scalar_one_or_none():
|
||||
raise HTTPException(status_code=400, detail="邮箱已被其他用户使用")
|
||||
return value
|
||||
|
||||
|
||||
def _apply_email_verified(user: User, verified: bool | None) -> None:
|
||||
if verified is None:
|
||||
return
|
||||
user.email_verified = verified
|
||||
if verified:
|
||||
if not user.email_verified_at:
|
||||
user.email_verified_at = datetime.utcnow()
|
||||
else:
|
||||
user.email_verified_at = None
|
||||
|
||||
|
||||
def _email_not_verified_detail(user: User) -> dict:
|
||||
return {
|
||||
"code": "email_not_verified",
|
||||
"message": "邮箱尚未验证,请先完成邮箱验证后再登录",
|
||||
"email": mask_email(user.email or ""),
|
||||
}
|
||||
|
||||
|
||||
def _email_not_bound_detail() -> dict:
|
||||
return {
|
||||
"code": "email_not_bound",
|
||||
"message": "该账号未绑定邮箱,请联系管理员绑定邮箱后再登录",
|
||||
}
|
||||
|
||||
|
||||
def _require_email_access(user: User, settings: SystemSettingsData) -> None:
|
||||
if is_admin(user.role):
|
||||
return
|
||||
if settings.email_binding_required and not user.email:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=_email_not_bound_detail(),
|
||||
)
|
||||
if not settings.email_verification_required:
|
||||
return
|
||||
if user.email_verified:
|
||||
return
|
||||
if user.email:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=_email_not_verified_detail(user),
|
||||
)
|
||||
|
||||
|
||||
async def _issue_token(user: User) -> TokenResponse:
|
||||
token = create_access_token(str(user.id), {"role": user.role, "username": user.username})
|
||||
return TokenResponse(access_token=token)
|
||||
|
||||
|
||||
async def _send_user_verification(
|
||||
db: AsyncSession,
|
||||
user: User,
|
||||
settings: SystemSettingsData,
|
||||
) -> tuple[bool, str | None]:
|
||||
if not user.email:
|
||||
raise HTTPException(status_code=400, detail="该账号未绑定邮箱")
|
||||
token = await create_verification_token(
|
||||
db, user, expire_hours=settings.email_verify_token_hours
|
||||
)
|
||||
await db.commit()
|
||||
link = build_verification_link(token, settings)
|
||||
if not settings.smtp_configured():
|
||||
logger.warning(
|
||||
"SMTP 未配置,验证链接: user=%s email=%s link=%s",
|
||||
user.username,
|
||||
user.email,
|
||||
link,
|
||||
)
|
||||
dev_url = link if settings.debug_show_verify_link else None
|
||||
return False, dev_url
|
||||
try:
|
||||
await send_verification_email(user.email, user.username, token, settings)
|
||||
return True, None
|
||||
except Exception as exc:
|
||||
logger.exception("Send verification email failed for user=%s", user.username)
|
||||
dev_url = link if settings.debug_show_verify_link else None
|
||||
if dev_url:
|
||||
return False, dev_url
|
||||
raise HTTPException(status_code=400, detail=f"邮件发送失败:{exc}") from exc
|
||||
|
||||
|
||||
@router.post("/login", response_model=TokenResponse)
|
||||
async def login(body: LoginRequest, db: AsyncSession = Depends(get_db)):
|
||||
settings = await load_settings(db)
|
||||
result = await db.execute(select(User).where(User.username == body.username.strip()))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user or not verify_password(body.password, user.password_hash):
|
||||
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
|
||||
if not user.is_active:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="账号已禁用")
|
||||
_require_email_access(user, settings)
|
||||
return await _issue_token(user)
|
||||
|
||||
|
||||
@router.post("/register", response_model=RegisterResponse)
|
||||
async def register(body: RegisterRequest, db: AsyncSession = Depends(get_db)):
|
||||
settings = await load_settings(db)
|
||||
if not settings.registration_enabled:
|
||||
raise HTTPException(status_code=403, detail="当前未开放用户注册")
|
||||
|
||||
username = body.username.strip()
|
||||
email = str(body.email).strip().lower()
|
||||
|
||||
exists = await db.execute(select(User).where(User.username == username))
|
||||
if exists.scalar_one_or_none():
|
||||
raise HTTPException(status_code=400, detail="用户名已存在")
|
||||
|
||||
exists = await db.execute(select(User).where(User.email == email))
|
||||
if exists.scalar_one_or_none():
|
||||
raise HTTPException(status_code=400, detail="邮箱已被注册")
|
||||
|
||||
verification_required = settings.email_verification_required
|
||||
|
||||
user = User(
|
||||
username=username,
|
||||
email=email,
|
||||
password_hash=hash_password(body.password),
|
||||
display_name=body.display_name or username,
|
||||
role=ROLE_OPERATOR,
|
||||
is_active=True,
|
||||
email_verified=not verification_required,
|
||||
email_verified_at=datetime.utcnow() if not verification_required else None,
|
||||
max_accounts=max(0, int(settings.default_register_max_accounts or 3)),
|
||||
)
|
||||
db.add(user)
|
||||
await db.flush()
|
||||
|
||||
if not verification_required:
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
return RegisterResponse(
|
||||
message="注册成功,可直接登录",
|
||||
email=mask_email(user.email),
|
||||
verification_sent=False,
|
||||
verification_required=False,
|
||||
)
|
||||
|
||||
verification_sent, dev_url = await _send_user_verification(db, user, settings)
|
||||
await db.refresh(user)
|
||||
message = "注册成功,验证邮件已发送,请查收并完成验证后再登录"
|
||||
if not verification_sent:
|
||||
message = (
|
||||
"注册成功。验证邮件未能发出,请使用下方链接或联系管理员检查 SMTP 配置"
|
||||
if dev_url
|
||||
else "注册成功。邮件服务未配置或发送失败,请联系管理员"
|
||||
)
|
||||
|
||||
return RegisterResponse(
|
||||
message=message,
|
||||
email=mask_email(user.email),
|
||||
verification_sent=verification_sent,
|
||||
verification_required=True,
|
||||
dev_verify_url=dev_url,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/verify-email", response_model=MessageResponse)
|
||||
async def verify_email(body: VerifyEmailRequest, db: AsyncSession = Depends(get_db)):
|
||||
user = await verify_email_token(db, body.token.strip())
|
||||
if not user:
|
||||
raise HTTPException(status_code=400, detail="验证链接无效或已过期")
|
||||
await db.commit()
|
||||
return MessageResponse(message="邮箱验证成功,现在可以登录了")
|
||||
|
||||
|
||||
@router.get("/verify-email", response_model=MessageResponse)
|
||||
async def verify_email_get(token: str = Query(..., min_length=8), db: AsyncSession = Depends(get_db)):
|
||||
user = await verify_email_token(db, token.strip())
|
||||
if not user:
|
||||
raise HTTPException(status_code=400, detail="验证链接无效或已过期")
|
||||
await db.commit()
|
||||
return MessageResponse(message="邮箱验证成功,现在可以登录了")
|
||||
|
||||
|
||||
@router.post("/forgot-password", response_model=ForgotPasswordResponse)
|
||||
async def forgot_password(body: ForgotPasswordRequest, db: AsyncSession = Depends(get_db)):
|
||||
settings = await load_settings(db)
|
||||
if not settings.smtp_configured():
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="邮件服务未配置,请联系管理员重置密码",
|
||||
)
|
||||
|
||||
email = str(body.email).strip().lower() if body.email else None
|
||||
username = body.username.strip() if body.username else None
|
||||
if not email and not username:
|
||||
raise HTTPException(status_code=400, detail="请提供邮箱或用户名")
|
||||
|
||||
user = None
|
||||
if email:
|
||||
result = await db.execute(select(User).where(User.email == email))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user and username:
|
||||
result = await db.execute(select(User).where(User.username == username))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
generic = ForgotPasswordResponse(
|
||||
message="若账号存在且已绑定邮箱,重置邮件将发送到注册邮箱",
|
||||
email=mask_email(user.email if user and user.email else (email or "")),
|
||||
reset_sent=True,
|
||||
)
|
||||
|
||||
if not user or not user.email or not user.is_active:
|
||||
return generic
|
||||
|
||||
token = await create_password_reset_token(
|
||||
db, user, expire_hours=settings.email_verify_token_hours
|
||||
)
|
||||
await db.commit()
|
||||
link = build_password_reset_link(token, settings)
|
||||
try:
|
||||
await send_password_reset_email(user.email, user.username, token, settings)
|
||||
except Exception as exc:
|
||||
logger.exception("Send password reset email failed for user=%s", user.username)
|
||||
dev_url = link if settings.debug_show_verify_link else None
|
||||
if dev_url:
|
||||
return ForgotPasswordResponse(
|
||||
message="邮件发送失败,请使用下方开发重置链接",
|
||||
email=mask_email(user.email),
|
||||
reset_sent=False,
|
||||
dev_reset_url=dev_url,
|
||||
)
|
||||
raise HTTPException(status_code=400, detail=f"邮件发送失败:{exc}") from exc
|
||||
return ForgotPasswordResponse(
|
||||
message="重置邮件已发送,请查收并按邮件说明设置新密码",
|
||||
email=mask_email(user.email),
|
||||
reset_sent=True,
|
||||
dev_reset_url=None,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/reset-password", response_model=MessageResponse)
|
||||
async def reset_password(body: ResetPasswordRequest, db: AsyncSession = Depends(get_db)):
|
||||
user = await verify_password_reset_token(db, body.token.strip())
|
||||
if not user:
|
||||
raise HTTPException(status_code=400, detail="重置链接无效或已过期")
|
||||
if not user.is_active:
|
||||
raise HTTPException(status_code=400, detail="账号已禁用,请联系管理员")
|
||||
user.password_hash = hash_password(body.password)
|
||||
await db.commit()
|
||||
return MessageResponse(message="密码已重置,请使用新密码登录")
|
||||
|
||||
|
||||
@router.post("/resend-verification", response_model=RegisterResponse)
|
||||
async def resend_verification(body: ResendVerificationRequest, db: AsyncSession = Depends(get_db)):
|
||||
settings = await load_settings(db)
|
||||
if not settings.email_verification_required:
|
||||
raise HTTPException(status_code=400, detail="当前系统未开启邮箱验证")
|
||||
email = str(body.email).strip().lower() if body.email else None
|
||||
username = body.username.strip() if body.username else None
|
||||
if not email and not username:
|
||||
raise HTTPException(status_code=400, detail="请提供邮箱或用户名")
|
||||
|
||||
user = None
|
||||
if email:
|
||||
result = await db.execute(select(User).where(User.email == email))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user and username:
|
||||
result = await db.execute(select(User).where(User.username == username))
|
||||
user = result.scalar_one_or_none()
|
||||
|
||||
if not user:
|
||||
return RegisterResponse(
|
||||
message="若账号存在且未验证,验证邮件将发送到注册邮箱",
|
||||
email=mask_email(email or ""),
|
||||
verification_sent=True,
|
||||
verification_required=True,
|
||||
)
|
||||
|
||||
if user.email_verified:
|
||||
raise HTTPException(status_code=400, detail="该账号邮箱已验证,可直接登录")
|
||||
|
||||
verification_sent, dev_url = await _send_user_verification(db, user, settings)
|
||||
message = (
|
||||
"验证邮件已重新发送,请查收"
|
||||
if verification_sent
|
||||
else "邮件未能发出,请使用下方验证链接或联系管理员检查 SMTP 配置"
|
||||
)
|
||||
return RegisterResponse(
|
||||
message=message,
|
||||
email=mask_email(user.email or ""),
|
||||
verification_sent=verification_sent,
|
||||
verification_required=True,
|
||||
dev_verify_url=dev_url,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/me", response_model=UserResponse)
|
||||
async def get_me(user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
|
||||
return await _build_user_response(db, user, with_count=True)
|
||||
|
||||
|
||||
@router.get("/roles", response_model=RolesResponse)
|
||||
async def list_roles(_: User = Depends(get_current_user)):
|
||||
return RolesResponse(
|
||||
roles=[RoleInfo(value=r, label=ROLE_LABELS.get(r, r)) for r in ALL_ROLES]
|
||||
)
|
||||
|
||||
|
||||
users_router = APIRouter(prefix="/api/users", tags=["users"])
|
||||
|
||||
|
||||
@users_router.get("", response_model=list[UserResponse])
|
||||
async def list_users(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_user_manager),
|
||||
):
|
||||
result = await db.execute(select(User).order_by(User.id.asc()))
|
||||
users = result.scalars().all()
|
||||
responses = []
|
||||
for user in users:
|
||||
responses.append(await _build_user_response(db, user, with_count=True))
|
||||
return responses
|
||||
|
||||
|
||||
@users_router.post("", response_model=UserResponse)
|
||||
async def create_user(
|
||||
body: UserCreate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_user_manager),
|
||||
):
|
||||
settings = await load_settings(db)
|
||||
exists = await db.execute(select(User).where(User.username == body.username))
|
||||
if exists.scalar_one_or_none():
|
||||
raise HTTPException(status_code=400, detail="用户名已存在")
|
||||
try:
|
||||
role = ensure_role(body.role)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
email = await _ensure_email_available(db, str(body.email) if body.email else None)
|
||||
if settings.email_binding_required and not is_admin(role) and not email:
|
||||
raise HTTPException(status_code=400, detail="系统已开启「登录必须绑定邮箱」,请填写邮箱")
|
||||
if email:
|
||||
email_verified = body.email_verified
|
||||
else:
|
||||
email_verified = not settings.email_binding_required
|
||||
user = User(
|
||||
username=body.username.strip(),
|
||||
password_hash=hash_password(body.password),
|
||||
display_name=body.display_name or body.username,
|
||||
role=role,
|
||||
is_active=True,
|
||||
email=email,
|
||||
email_verified=email_verified,
|
||||
email_verified_at=datetime.utcnow() if email and email_verified else None,
|
||||
max_accounts=normalize_max_accounts(body.max_accounts, role),
|
||||
)
|
||||
db.add(user)
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
return await _build_user_response(db, user, with_count=True)
|
||||
|
||||
|
||||
@users_router.put("/{user_id}", response_model=UserResponse)
|
||||
async def update_user(
|
||||
user_id: int,
|
||||
body: UserUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current: User = Depends(require_user_manager),
|
||||
):
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user:
|
||||
raise HTTPException(status_code=404, detail="用户不存在")
|
||||
settings = await load_settings(db)
|
||||
if user.id == current.id and body.is_active is False:
|
||||
raise HTTPException(status_code=400, detail="不能禁用当前登录账号")
|
||||
if body.display_name is not None:
|
||||
user.display_name = body.display_name
|
||||
if body.role is not None:
|
||||
prev_role = user.role
|
||||
try:
|
||||
user.role = ensure_role(body.role)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
if is_admin(user.role):
|
||||
user.max_accounts = UNLIMITED_ACCOUNTS
|
||||
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
|
||||
elif is_admin(prev_role) and not is_admin(user.role):
|
||||
settings = await load_settings(db)
|
||||
user.max_accounts = max(0, int(settings.default_register_max_accounts or 3))
|
||||
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
|
||||
if body.is_active is not None:
|
||||
user.is_active = body.is_active
|
||||
if body.password:
|
||||
user.password_hash = hash_password(body.password)
|
||||
|
||||
updates = body.model_dump(exclude_unset=True)
|
||||
if "max_accounts" in updates and not is_admin(user.role):
|
||||
user.max_accounts = normalize_max_accounts(updates["max_accounts"], user.role)
|
||||
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
|
||||
if "email" in updates:
|
||||
raw_email = updates.get("email")
|
||||
user.email = await _ensure_email_available(
|
||||
db,
|
||||
str(raw_email) if raw_email else None,
|
||||
exclude_user_id=user.id,
|
||||
)
|
||||
if not user.email:
|
||||
user.email_verified = not (
|
||||
settings.email_binding_required and not is_admin(user.role)
|
||||
)
|
||||
user.email_verified_at = None
|
||||
if "email_verified" in updates:
|
||||
if not user.email:
|
||||
raise HTTPException(status_code=400, detail="未绑定邮箱时无法设置验证状态")
|
||||
_apply_email_verified(user, updates["email_verified"])
|
||||
|
||||
if settings.email_binding_required and not is_admin(user.role) and not user.email:
|
||||
raise HTTPException(status_code=400, detail="系统已开启「登录必须绑定邮箱」,该用户需绑定邮箱")
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(user)
|
||||
return await _build_user_response(db, user, with_count=True)
|
||||
|
||||
|
||||
@users_router.delete("/{user_id}")
|
||||
async def delete_user(
|
||||
user_id: int,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
current: User = Depends(require_user_manager),
|
||||
):
|
||||
if user_id == current.id:
|
||||
raise HTTPException(status_code=400, detail="不能删除当前登录账号")
|
||||
result = await db.execute(select(User).where(User.id == user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user:
|
||||
raise HTTPException(status_code=404, detail="用户不存在")
|
||||
await db.delete(user)
|
||||
await db.commit()
|
||||
return {"message": "用户已删除"}
|
||||
@@ -0,0 +1,108 @@
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
|
||||
from .roles import ALL_ROLES
|
||||
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
|
||||
class RegisterRequest(BaseModel):
|
||||
username: str = Field(min_length=2, max_length=50)
|
||||
email: EmailStr
|
||||
password: str = Field(min_length=6, max_length=128)
|
||||
display_name: Optional[str] = None
|
||||
|
||||
|
||||
class VerifyEmailRequest(BaseModel):
|
||||
token: str = Field(min_length=8, max_length=128)
|
||||
|
||||
|
||||
class ResendVerificationRequest(BaseModel):
|
||||
email: Optional[EmailStr] = None
|
||||
username: Optional[str] = None
|
||||
|
||||
|
||||
class ForgotPasswordRequest(BaseModel):
|
||||
email: Optional[EmailStr] = None
|
||||
username: Optional[str] = None
|
||||
|
||||
|
||||
class ResetPasswordRequest(BaseModel):
|
||||
token: str = Field(min_length=8, max_length=128)
|
||||
password: str = Field(min_length=6, max_length=128)
|
||||
|
||||
|
||||
class ForgotPasswordResponse(BaseModel):
|
||||
message: str
|
||||
email: str
|
||||
reset_sent: bool
|
||||
dev_reset_url: Optional[str] = None
|
||||
|
||||
|
||||
class TokenResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
|
||||
|
||||
class RegisterResponse(BaseModel):
|
||||
message: str
|
||||
email: str
|
||||
verification_sent: bool
|
||||
verification_required: bool = True
|
||||
dev_verify_url: Optional[str] = None
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
message: str
|
||||
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
id: int
|
||||
username: str
|
||||
email: Optional[str] = None
|
||||
display_name: Optional[str] = None
|
||||
role: str
|
||||
is_active: bool
|
||||
email_verified: bool = False
|
||||
max_accounts: int = 3
|
||||
account_count: Optional[int] = None
|
||||
active_account_count: Optional[int] = None
|
||||
disabled_account_count: Optional[int] = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class UserCreate(BaseModel):
|
||||
username: str = Field(min_length=2, max_length=50)
|
||||
password: str = Field(min_length=6, max_length=128)
|
||||
display_name: Optional[str] = None
|
||||
role: str = "operator"
|
||||
email: Optional[EmailStr] = None
|
||||
email_verified: bool = True
|
||||
max_accounts: int = Field(default=3, ge=0, le=999)
|
||||
|
||||
|
||||
class UserUpdate(BaseModel):
|
||||
display_name: Optional[str] = None
|
||||
role: Optional[str] = None
|
||||
is_active: Optional[bool] = None
|
||||
password: Optional[str] = Field(default=None, min_length=6, max_length=128)
|
||||
email: Optional[EmailStr] = None
|
||||
email_verified: Optional[bool] = None
|
||||
max_accounts: Optional[int] = Field(default=None, ge=0, le=999)
|
||||
|
||||
|
||||
class RoleInfo(BaseModel):
|
||||
value: str
|
||||
label: str
|
||||
|
||||
|
||||
class RolesResponse(BaseModel):
|
||||
roles: list[RoleInfo]
|
||||
@@ -0,0 +1,109 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from sqlalchemy import or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import Account, AutoReplyRule, MessageLog, ReceivedMessageLog, SystemLog, User
|
||||
from .roles import is_admin
|
||||
|
||||
|
||||
async def get_owned_account(
|
||||
db: AsyncSession,
|
||||
user: User,
|
||||
account_id: int,
|
||||
*,
|
||||
write: bool = False,
|
||||
) -> Account:
|
||||
result = await db.execute(select(Account).where(Account.id == account_id))
|
||||
account = result.scalar_one_or_none()
|
||||
if not account:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="账号不存在")
|
||||
if is_admin(user.role):
|
||||
return account
|
||||
if account.owner_id != user.id:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="无权访问该账号")
|
||||
if write and user.role == "viewer":
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="只读用户无法修改")
|
||||
return account
|
||||
|
||||
|
||||
def accounts_for_user(user: User):
|
||||
stmt = select(Account)
|
||||
if not is_admin(user.role):
|
||||
stmt = stmt.where(Account.owner_id == user.id)
|
||||
return stmt
|
||||
|
||||
|
||||
async def owned_account_ids(db: AsyncSession, user: User) -> Optional[set[int]]:
|
||||
if is_admin(user.role):
|
||||
return None
|
||||
result = await db.execute(select(Account.id).where(Account.owner_id == user.id))
|
||||
return {row[0] for row in result.all()}
|
||||
|
||||
|
||||
def logs_for_user(user: User, account_id: Optional[int] = None):
|
||||
stmt = select(MessageLog)
|
||||
if account_id is not None:
|
||||
stmt = stmt.where(MessageLog.account_id == account_id)
|
||||
if not is_admin(user.role):
|
||||
owned = select(Account.id).where(Account.owner_id == user.id)
|
||||
stmt = stmt.where(MessageLog.account_id.in_(owned))
|
||||
return stmt
|
||||
|
||||
|
||||
def received_logs_for_user(user: User, account_id: Optional[int] = None):
|
||||
stmt = select(ReceivedMessageLog)
|
||||
if account_id is not None:
|
||||
stmt = stmt.where(ReceivedMessageLog.account_id == account_id)
|
||||
if not is_admin(user.role):
|
||||
owned = select(Account.id).where(Account.owner_id == user.id)
|
||||
stmt = stmt.where(ReceivedMessageLog.account_id.in_(owned))
|
||||
return stmt
|
||||
|
||||
|
||||
def rules_for_user(user: User, account_id: Optional[int] = None):
|
||||
stmt = select(AutoReplyRule)
|
||||
if account_id is not None:
|
||||
stmt = stmt.where(AutoReplyRule.account_id == account_id)
|
||||
if is_admin(user.role):
|
||||
return stmt
|
||||
owned = select(Account.id).where(Account.owner_id == user.id)
|
||||
return stmt.where(AutoReplyRule.account_id.in_(owned))
|
||||
|
||||
|
||||
async def get_accessible_rule(db: AsyncSession, user: User, rule_id: int, *, write: bool = False) -> AutoReplyRule:
|
||||
result = await db.execute(select(AutoReplyRule).where(AutoReplyRule.id == rule_id))
|
||||
rule = result.scalar_one_or_none()
|
||||
if not rule:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="规则不存在")
|
||||
|
||||
if is_admin(user.role):
|
||||
if write and user.role == "viewer":
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="只读用户无法修改")
|
||||
return rule
|
||||
|
||||
if rule.account_id is None:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="无权访问全局规则")
|
||||
|
||||
account = await get_owned_account(db, user, rule.account_id, write=write)
|
||||
if rule.owner_id and rule.owner_id != user.id and account.owner_id != user.id:
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="无权访问该规则")
|
||||
if write and user.role == "viewer":
|
||||
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="只读用户无法修改")
|
||||
return rule
|
||||
|
||||
|
||||
def system_logs_for_user(user: User, account_id: Optional[int] = None):
|
||||
stmt = select(SystemLog)
|
||||
if account_id is not None:
|
||||
stmt = stmt.where(SystemLog.account_id == account_id)
|
||||
if not is_admin(user.role):
|
||||
owned = select(Account.id).where(Account.owner_id == user.id)
|
||||
stmt = stmt.where(
|
||||
or_(
|
||||
SystemLog.account_id.in_(owned),
|
||||
SystemLog.account_id.is_(None),
|
||||
)
|
||||
)
|
||||
return stmt
|
||||
@@ -0,0 +1,346 @@
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
|
||||
from dataclasses import asdict
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.database import get_db
|
||||
from models.db_config import (
|
||||
PASSWORD_PLACEHOLDER as DB_PASSWORD_PLACEHOLDER,
|
||||
database_config_to_response,
|
||||
persist_database_config,
|
||||
test_database_connection,
|
||||
)
|
||||
from models.db_transfer import inspect_sqlite_source, migrate_sqlite_to_target
|
||||
from models.models import User
|
||||
from .dependencies import require_admin
|
||||
from .email_service import send_test_email
|
||||
from .system_settings import (
|
||||
PASSWORD_PLACEHOLDER,
|
||||
SystemSettingsData,
|
||||
load_settings,
|
||||
save_settings,
|
||||
settings_to_admin_response,
|
||||
settings_to_payment_response,
|
||||
settings_to_public,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/settings", tags=["settings"])
|
||||
|
||||
|
||||
class PublicSettingsResponse(BaseModel):
|
||||
registration_enabled: bool
|
||||
email_verification_required: bool
|
||||
email_binding_required: bool
|
||||
|
||||
|
||||
class SystemSettingsResponse(BaseModel):
|
||||
registration_enabled: bool
|
||||
email_verification_required: bool
|
||||
email_binding_required: bool = False
|
||||
email_verify_token_hours: int
|
||||
auto_reply_delay_seconds: int = 0
|
||||
auto_reply_cooldown_seconds: int = 60
|
||||
app_url: str
|
||||
default_register_max_accounts: int = 3
|
||||
smtp_host: str
|
||||
smtp_port: int
|
||||
smtp_user: str
|
||||
smtp_password: str = ""
|
||||
smtp_password_configured: bool = False
|
||||
smtp_from: str
|
||||
smtp_use_tls: bool
|
||||
smtp_use_ssl: bool = False
|
||||
debug_show_verify_link: bool
|
||||
email_verify_subject: str
|
||||
email_verify_body: str
|
||||
email_verify_html: str
|
||||
email_test_subject: str
|
||||
email_test_body: str
|
||||
email_test_html: str
|
||||
|
||||
|
||||
class SystemSettingsUpdate(BaseModel):
|
||||
registration_enabled: bool | None = None
|
||||
email_verification_required: bool | None = None
|
||||
email_binding_required: bool | None = None
|
||||
email_verify_token_hours: int | None = Field(default=None, ge=1, le=168)
|
||||
auto_reply_delay_seconds: int | None = Field(default=None, ge=0, le=86400)
|
||||
auto_reply_cooldown_seconds: int | None = Field(default=None, ge=0, le=86400)
|
||||
app_url: str | None = None
|
||||
default_register_max_accounts: int | None = Field(default=None, ge=0, le=999)
|
||||
smtp_host: str | None = None
|
||||
smtp_port: int | None = Field(default=None, ge=1, le=65535)
|
||||
smtp_user: str | None = None
|
||||
smtp_password: str | None = None
|
||||
smtp_from: str | None = None
|
||||
smtp_use_tls: bool | None = None
|
||||
smtp_use_ssl: bool | None = None
|
||||
debug_show_verify_link: bool | None = None
|
||||
email_verify_subject: str | None = None
|
||||
email_verify_body: str | None = None
|
||||
email_verify_html: str | None = None
|
||||
email_test_subject: str | None = None
|
||||
email_test_body: str | None = None
|
||||
email_test_html: str | None = None
|
||||
|
||||
|
||||
class TestEmailRequest(BaseModel):
|
||||
to_email: EmailStr
|
||||
smtp_host: str | None = None
|
||||
smtp_port: int | None = Field(default=None, ge=1, le=65535)
|
||||
smtp_user: str | None = None
|
||||
smtp_password: str | None = None
|
||||
smtp_from: str | None = None
|
||||
smtp_use_tls: bool | None = None
|
||||
smtp_use_ssl: bool | None = None
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
message: str
|
||||
|
||||
|
||||
class DatabaseSettingsResponse(BaseModel):
|
||||
db_type: str
|
||||
db_host: str = ""
|
||||
db_port: int = 0
|
||||
db_user: str = ""
|
||||
db_password: str = ""
|
||||
db_password_configured: bool = False
|
||||
db_name: str = ""
|
||||
db_path: str = ""
|
||||
database_url_display: str = ""
|
||||
supported_types: list[str] = []
|
||||
|
||||
|
||||
class DatabaseSettingsUpdate(BaseModel):
|
||||
db_type: str
|
||||
db_host: str | None = None
|
||||
db_port: int | None = Field(default=None, ge=1, le=65535)
|
||||
db_user: str | None = None
|
||||
db_password: str | None = None
|
||||
db_name: str | None = None
|
||||
db_path: str | None = None
|
||||
|
||||
|
||||
class DatabaseTestRequest(BaseModel):
|
||||
db_type: str
|
||||
db_host: str | None = None
|
||||
db_port: int | None = Field(default=None, ge=1, le=65535)
|
||||
db_user: str | None = None
|
||||
db_password: str | None = None
|
||||
db_name: str | None = None
|
||||
db_path: str | None = None
|
||||
|
||||
|
||||
class DatabaseMigratePreviewResponse(BaseModel):
|
||||
source_path: str
|
||||
exists: bool
|
||||
tables: dict[str, int]
|
||||
total_rows: int
|
||||
|
||||
|
||||
class DatabaseMigrateRequest(DatabaseTestRequest):
|
||||
source_db_path: str | None = None
|
||||
clear_target: bool = False
|
||||
|
||||
|
||||
class DatabaseMigrateResponse(BaseModel):
|
||||
message: str
|
||||
source_path: str
|
||||
target_type: str
|
||||
target_url: str
|
||||
tables: dict[str, int]
|
||||
total_rows: int
|
||||
|
||||
|
||||
class PaymentSettingsResponse(BaseModel):
|
||||
app_url: str
|
||||
payment_enabled: bool = False
|
||||
payment_demo_mode: bool = True
|
||||
wechat_pay_enabled: bool = False
|
||||
alipay_pay_enabled: bool = False
|
||||
account_slot_unit_price: float = 9.9
|
||||
account_slot_purchase_min: int = 1
|
||||
account_slot_purchase_max: int = 20
|
||||
wechat_app_id: str = ""
|
||||
wechat_mch_id: str = ""
|
||||
wechat_api_v3_key: str = ""
|
||||
wechat_api_v3_key_configured: bool = False
|
||||
wechat_cert_serial: str = ""
|
||||
wechat_private_key: str = ""
|
||||
wechat_private_key_configured: bool = False
|
||||
wechat_pay_configured: bool = False
|
||||
alipay_app_id: str = ""
|
||||
alipay_private_key: str = ""
|
||||
alipay_private_key_configured: bool = False
|
||||
alipay_public_key: str = ""
|
||||
alipay_sandbox: bool = False
|
||||
alipay_configured: bool = False
|
||||
|
||||
|
||||
class PaymentSettingsUpdate(BaseModel):
|
||||
payment_enabled: bool | None = None
|
||||
payment_demo_mode: bool | None = None
|
||||
wechat_pay_enabled: bool | None = None
|
||||
alipay_pay_enabled: bool | None = None
|
||||
account_slot_unit_price: float | None = Field(default=None, ge=0.01, le=99999)
|
||||
account_slot_purchase_min: int | None = Field(default=None, ge=1, le=100)
|
||||
account_slot_purchase_max: int | None = Field(default=None, ge=1, le=100)
|
||||
wechat_app_id: str | None = None
|
||||
wechat_mch_id: str | None = None
|
||||
wechat_api_v3_key: str | None = None
|
||||
wechat_cert_serial: str | None = None
|
||||
wechat_private_key: str | None = None
|
||||
alipay_app_id: str | None = None
|
||||
alipay_private_key: str | None = None
|
||||
alipay_public_key: str | None = None
|
||||
alipay_sandbox: bool | None = None
|
||||
|
||||
|
||||
@router.get("/public", response_model=PublicSettingsResponse)
|
||||
async def get_public_settings(db: AsyncSession = Depends(get_db)):
|
||||
data = await load_settings(db)
|
||||
return PublicSettingsResponse(**settings_to_public(data))
|
||||
|
||||
|
||||
@router.get("", response_model=SystemSettingsResponse)
|
||||
async def get_system_settings(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
data = await load_settings(db)
|
||||
return SystemSettingsResponse(**settings_to_admin_response(data))
|
||||
|
||||
|
||||
@router.put("", response_model=SystemSettingsResponse)
|
||||
async def update_system_settings(
|
||||
body: SystemSettingsUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
updates = body.model_dump(exclude_unset=True)
|
||||
if "app_url" in updates and updates["app_url"]:
|
||||
updates["app_url"] = updates["app_url"].strip().rstrip("/")
|
||||
data = await save_settings(db, updates)
|
||||
return SystemSettingsResponse(**settings_to_admin_response(data))
|
||||
|
||||
|
||||
@router.get("/payment", response_model=PaymentSettingsResponse)
|
||||
async def get_payment_settings(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
data = await load_settings(db)
|
||||
return PaymentSettingsResponse(**settings_to_payment_response(data))
|
||||
|
||||
|
||||
@router.put("/payment", response_model=PaymentSettingsResponse)
|
||||
async def update_payment_settings(
|
||||
body: PaymentSettingsUpdate,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
updates = body.model_dump(exclude_unset=True)
|
||||
data = await save_settings(db, updates)
|
||||
return PaymentSettingsResponse(**settings_to_payment_response(data))
|
||||
|
||||
|
||||
@router.post("/test-email", response_model=MessageResponse)
|
||||
async def test_smtp_email(
|
||||
body: TestEmailRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
data = await load_settings(db)
|
||||
overrides = body.model_dump(exclude_unset=True, exclude={"to_email"})
|
||||
if overrides:
|
||||
merged = asdict(data)
|
||||
for key, value in overrides.items():
|
||||
if value is None:
|
||||
continue
|
||||
if key == "smtp_password":
|
||||
pwd = str(value).strip()
|
||||
if not pwd or pwd == PASSWORD_PLACEHOLDER:
|
||||
continue
|
||||
merged[key] = value
|
||||
data = SystemSettingsData(**merged)
|
||||
if not data.smtp_configured():
|
||||
raise HTTPException(status_code=400, detail="请先完整配置 SMTP 服务器与发件人")
|
||||
try:
|
||||
await send_test_email(str(body.to_email), data)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return MessageResponse(message=f"测试邮件已发送至 {body.to_email}")
|
||||
|
||||
|
||||
@router.get("/database", response_model=DatabaseSettingsResponse)
|
||||
async def get_database_settings(_: User = Depends(require_admin)):
|
||||
return DatabaseSettingsResponse(**database_config_to_response())
|
||||
|
||||
|
||||
@router.put("/database", response_model=MessageResponse)
|
||||
async def update_database_settings(
|
||||
body: DatabaseSettingsUpdate,
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
payload = body.model_dump(exclude_unset=True)
|
||||
if payload.get("db_password") in (None, "", DB_PASSWORD_PLACEHOLDER):
|
||||
payload.pop("db_password", None)
|
||||
try:
|
||||
await test_database_connection(payload)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=f"连接测试失败: {exc}") from exc
|
||||
try:
|
||||
persist_database_config(payload)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=f"写入配置失败: {exc}") from exc
|
||||
return MessageResponse(message="数据库配置已保存至 .env,请重启后端服务后生效")
|
||||
|
||||
|
||||
@router.post("/database/test", response_model=MessageResponse)
|
||||
async def test_database_settings(
|
||||
body: DatabaseTestRequest,
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
payload = body.model_dump(exclude_unset=True)
|
||||
if payload.get("db_password") in (None, "", DB_PASSWORD_PLACEHOLDER):
|
||||
payload.pop("db_password", None)
|
||||
try:
|
||||
await test_database_connection(payload)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
return MessageResponse(message="数据库连接测试成功")
|
||||
|
||||
|
||||
@router.get("/database/migrate/preview", response_model=DatabaseMigratePreviewResponse)
|
||||
async def preview_database_migration(
|
||||
source_db_path: str | None = None,
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
return DatabaseMigratePreviewResponse(**await inspect_sqlite_source(source_db_path))
|
||||
|
||||
|
||||
@router.post("/database/migrate", response_model=DatabaseMigrateResponse)
|
||||
async def migrate_database_data(
|
||||
body: DatabaseMigrateRequest,
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
payload = body.model_dump(exclude_unset=True)
|
||||
clear_target = bool(payload.pop("clear_target", False))
|
||||
if payload.get("db_password") in (None, "", DB_PASSWORD_PLACEHOLDER):
|
||||
payload.pop("db_password", None)
|
||||
try:
|
||||
await test_database_connection(payload)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=400, detail=f"目标库连接失败: {exc}") from exc
|
||||
try:
|
||||
result = await migrate_sqlite_to_target(payload, clear_target=clear_target)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=500, detail=f"迁移失败: {exc}") from exc
|
||||
return DatabaseMigrateResponse(**result)
|
||||
@@ -0,0 +1,366 @@
|
||||
"""系统功能配置(数据库存储 + 内存缓存)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict, dataclass, fields
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import AppConfig
|
||||
|
||||
CONFIG_ROW_ID = 1
|
||||
PASSWORD_PLACEHOLDER = "******"
|
||||
APP_NAME = "抖音回复助手"
|
||||
|
||||
DEFAULT_EMAIL_VERIFY_SUBJECT = f"【{APP_NAME}】请验证您的邮箱"
|
||||
DEFAULT_EMAIL_VERIFY_BODY = (
|
||||
"您好 {username},\n\n"
|
||||
"感谢注册{app_name}。请点击以下链接完成邮箱验证:\n"
|
||||
"{link}\n\n"
|
||||
"如非本人操作,请忽略此邮件。\n"
|
||||
)
|
||||
DEFAULT_EMAIL_VERIFY_HTML = (
|
||||
"<p>您好 <strong>{username}</strong>,</p>"
|
||||
"<p>感谢注册{app_name}。请点击下方按钮完成邮箱验证:</p>"
|
||||
'<p><a href="{link}" style="display:inline-block;padding:10px 18px;'
|
||||
'background:#aa3bff;color:#fff;text-decoration:none;border-radius:6px;">'
|
||||
"验证邮箱</a></p>"
|
||||
'<p>或复制链接到浏览器:<br><a href="{link}">{link}</a></p>'
|
||||
'<p style="color:#888;font-size:12px;">如非本人操作,请忽略此邮件。</p>'
|
||||
)
|
||||
DEFAULT_EMAIL_TEST_SUBJECT = f"【{APP_NAME}】SMTP 测试邮件"
|
||||
DEFAULT_EMAIL_TEST_BODY = "这是一封 SMTP 配置测试邮件。若您收到此邮件,说明邮件服务已配置正确。"
|
||||
DEFAULT_EMAIL_TEST_HTML = (
|
||||
"<p>这是一封 <strong>SMTP 配置测试</strong> 邮件。</p>"
|
||||
"<p>若您收到此邮件,说明邮件服务已配置正确。</p>"
|
||||
)
|
||||
DEFAULT_PASSWORD_RESET_SUBJECT = f"【{APP_NAME}】重置您的登录密码"
|
||||
DEFAULT_PASSWORD_RESET_BODY = (
|
||||
"您好 {username},\n\n"
|
||||
"我们收到了重置 {app_name} 账号密码的请求。请点击以下链接设置新密码:\n"
|
||||
"{link}\n\n"
|
||||
"链接有效期 {expire_hours} 小时。如非本人操作,请忽略此邮件。\n"
|
||||
)
|
||||
DEFAULT_PASSWORD_RESET_HTML = (
|
||||
"<p>您好 <strong>{username}</strong>,</p>"
|
||||
"<p>我们收到了重置 {app_name} 账号密码的请求。请点击下方按钮设置新密码:</p>"
|
||||
'<p><a href="{link}" style="display:inline-block;padding:10px 18px;'
|
||||
'background:#aa3bff;color:#fff;text-decoration:none;border-radius:6px;">'
|
||||
"重置密码</a></p>"
|
||||
'<p>或复制链接到浏览器:<br><a href="{link}">{link}</a></p>'
|
||||
'<p style="color:#888;font-size:12px;">链接有效期 {expire_hours} 小时。如非本人操作,请忽略此邮件。</p>'
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SystemSettingsData:
|
||||
registration_enabled: bool = True
|
||||
email_verification_required: bool = True
|
||||
email_binding_required: bool = False
|
||||
email_verify_token_hours: int = 24
|
||||
auto_reply_delay_seconds: int = 0
|
||||
auto_reply_cooldown_seconds: int = 60
|
||||
app_url: str = "http://localhost:8800"
|
||||
smtp_host: str = ""
|
||||
smtp_port: int = 587
|
||||
smtp_user: str = ""
|
||||
smtp_password: str = ""
|
||||
smtp_from: str = ""
|
||||
smtp_use_tls: bool = True
|
||||
smtp_use_ssl: bool = False
|
||||
debug_show_verify_link: bool = False
|
||||
default_register_max_accounts: int = 3
|
||||
payment_enabled: bool = False
|
||||
payment_demo_mode: bool = True
|
||||
wechat_pay_enabled: bool = False
|
||||
alipay_pay_enabled: bool = False
|
||||
account_slot_unit_price: float = 9.9
|
||||
account_slot_purchase_min: int = 1
|
||||
account_slot_purchase_max: int = 20
|
||||
wechat_app_id: str = ""
|
||||
wechat_mch_id: str = ""
|
||||
wechat_api_v3_key: str = ""
|
||||
wechat_cert_serial: str = ""
|
||||
wechat_private_key: str = ""
|
||||
alipay_app_id: str = ""
|
||||
alipay_private_key: str = ""
|
||||
alipay_public_key: str = ""
|
||||
alipay_sandbox: bool = False
|
||||
email_verify_subject: str = DEFAULT_EMAIL_VERIFY_SUBJECT
|
||||
email_verify_body: str = DEFAULT_EMAIL_VERIFY_BODY
|
||||
email_verify_html: str = DEFAULT_EMAIL_VERIFY_HTML
|
||||
email_test_subject: str = DEFAULT_EMAIL_TEST_SUBJECT
|
||||
email_test_body: str = DEFAULT_EMAIL_TEST_BODY
|
||||
email_test_html: str = DEFAULT_EMAIL_TEST_HTML
|
||||
|
||||
def smtp_configured(self) -> bool:
|
||||
sender = (self.smtp_from or self.smtp_user or "").strip()
|
||||
return bool(self.smtp_host.strip() and sender)
|
||||
|
||||
def app_url_normalized(self) -> str:
|
||||
return (self.app_url or "http://localhost:8800").rstrip("/")
|
||||
|
||||
def wechat_pay_configured(self) -> bool:
|
||||
return bool(
|
||||
self.wechat_app_id.strip()
|
||||
and self.wechat_mch_id.strip()
|
||||
and self.wechat_api_v3_key.strip()
|
||||
and self.wechat_cert_serial.strip()
|
||||
and self.wechat_private_key.strip()
|
||||
)
|
||||
|
||||
def alipay_configured(self) -> bool:
|
||||
return bool(
|
||||
self.alipay_app_id.strip()
|
||||
and self.alipay_private_key.strip()
|
||||
and self.alipay_public_key.strip()
|
||||
)
|
||||
|
||||
def payment_channel_available(self, channel: str) -> bool:
|
||||
if channel == "wechat":
|
||||
return self.wechat_pay_enabled and self.wechat_pay_configured()
|
||||
if channel == "alipay":
|
||||
return self.alipay_pay_enabled and self.alipay_configured()
|
||||
return False
|
||||
|
||||
def payment_channel_selectable(self, channel: str) -> bool:
|
||||
"""用户可选的支付渠道(含演示模式)。"""
|
||||
if channel == "wechat":
|
||||
if not self.wechat_pay_enabled:
|
||||
return False
|
||||
return self.wechat_pay_configured() or (
|
||||
self.payment_demo_mode and self.payment_enabled
|
||||
)
|
||||
if channel == "alipay":
|
||||
if not self.alipay_pay_enabled:
|
||||
return False
|
||||
return self.alipay_configured() or (
|
||||
self.payment_demo_mode and self.payment_enabled
|
||||
)
|
||||
return False
|
||||
|
||||
def available_payment_channels(self) -> list[str]:
|
||||
channels = []
|
||||
if self.payment_channel_selectable("wechat"):
|
||||
channels.append("wechat")
|
||||
if self.payment_channel_selectable("alipay"):
|
||||
channels.append("alipay")
|
||||
return channels
|
||||
|
||||
|
||||
_settings_cache: SystemSettingsData | None = None
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, default: bool) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.lower() in ("1", "true", "yes", "on")
|
||||
if value is None:
|
||||
return default
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _parse_settings(raw: dict[str, Any]) -> SystemSettingsData:
|
||||
base = SystemSettingsData()
|
||||
allowed = {f.name for f in fields(SystemSettingsData)}
|
||||
merged: dict[str, Any] = {}
|
||||
for key in allowed:
|
||||
if key in raw:
|
||||
merged[key] = raw[key]
|
||||
if "smtp_port" in merged:
|
||||
try:
|
||||
merged["smtp_port"] = int(merged["smtp_port"])
|
||||
except (TypeError, ValueError):
|
||||
merged["smtp_port"] = base.smtp_port
|
||||
if "email_verify_token_hours" in merged:
|
||||
try:
|
||||
merged["email_verify_token_hours"] = max(
|
||||
1, min(168, int(merged["email_verify_token_hours"]))
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
merged["email_verify_token_hours"] = base.email_verify_token_hours
|
||||
if "auto_reply_cooldown_seconds" in merged:
|
||||
try:
|
||||
merged["auto_reply_cooldown_seconds"] = max(
|
||||
0, min(86400, int(merged["auto_reply_cooldown_seconds"]))
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
merged["auto_reply_cooldown_seconds"] = base.auto_reply_cooldown_seconds
|
||||
if "auto_reply_delay_seconds" in merged:
|
||||
try:
|
||||
merged["auto_reply_delay_seconds"] = max(
|
||||
0, min(86400, int(merged["auto_reply_delay_seconds"]))
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
merged["auto_reply_delay_seconds"] = base.auto_reply_delay_seconds
|
||||
if "default_register_max_accounts" in merged:
|
||||
try:
|
||||
merged["default_register_max_accounts"] = max(
|
||||
0, min(999, int(merged["default_register_max_accounts"]))
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
merged["default_register_max_accounts"] = base.default_register_max_accounts
|
||||
if "account_slot_unit_price" in merged:
|
||||
try:
|
||||
merged["account_slot_unit_price"] = max(
|
||||
0.01, min(99999.0, float(merged["account_slot_unit_price"]))
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
merged["account_slot_unit_price"] = base.account_slot_unit_price
|
||||
for int_key, lo, hi in (
|
||||
("account_slot_purchase_min", 1, 100),
|
||||
("account_slot_purchase_max", 1, 100),
|
||||
):
|
||||
if int_key in merged:
|
||||
try:
|
||||
merged[int_key] = max(lo, min(hi, int(merged[int_key])))
|
||||
except (TypeError, ValueError):
|
||||
merged[int_key] = getattr(base, int_key)
|
||||
for bool_key in (
|
||||
"registration_enabled",
|
||||
"email_verification_required",
|
||||
"email_binding_required",
|
||||
"smtp_use_tls",
|
||||
"smtp_use_ssl",
|
||||
"debug_show_verify_link",
|
||||
"payment_enabled",
|
||||
"payment_demo_mode",
|
||||
"wechat_pay_enabled",
|
||||
"alipay_pay_enabled",
|
||||
"alipay_sandbox",
|
||||
):
|
||||
if bool_key in merged:
|
||||
merged[bool_key] = _coerce_bool(merged[bool_key], getattr(base, bool_key))
|
||||
return SystemSettingsData(**{**asdict(base), **merged})
|
||||
|
||||
|
||||
def get_cached_settings() -> SystemSettingsData:
|
||||
global _settings_cache
|
||||
if _settings_cache is None:
|
||||
_settings_cache = SystemSettingsData()
|
||||
return _settings_cache
|
||||
|
||||
|
||||
def set_cached_settings(data: SystemSettingsData) -> None:
|
||||
global _settings_cache
|
||||
_settings_cache = data
|
||||
|
||||
|
||||
async def load_settings(db: AsyncSession) -> SystemSettingsData:
|
||||
result = await db.execute(select(AppConfig).where(AppConfig.id == CONFIG_ROW_ID))
|
||||
row = result.scalar_one_or_none()
|
||||
if not row or not row.data:
|
||||
data = SystemSettingsData()
|
||||
set_cached_settings(data)
|
||||
return data
|
||||
try:
|
||||
payload = json.loads(row.data)
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
data = _parse_settings(payload if isinstance(payload, dict) else {})
|
||||
set_cached_settings(data)
|
||||
return data
|
||||
|
||||
|
||||
async def ensure_default_settings(db: AsyncSession) -> SystemSettingsData:
|
||||
result = await db.execute(select(AppConfig).where(AppConfig.id == CONFIG_ROW_ID))
|
||||
row = result.scalar_one_or_none()
|
||||
if row:
|
||||
return await load_settings(db)
|
||||
data = SystemSettingsData()
|
||||
row = AppConfig(
|
||||
id=CONFIG_ROW_ID,
|
||||
data=json.dumps(asdict(data), ensure_ascii=False),
|
||||
updated_at=datetime.utcnow(),
|
||||
)
|
||||
db.add(row)
|
||||
await db.commit()
|
||||
set_cached_settings(data)
|
||||
return data
|
||||
|
||||
|
||||
async def save_settings(db: AsyncSession, updates: dict[str, Any]) -> SystemSettingsData:
|
||||
current = await load_settings(db)
|
||||
merged = asdict(current)
|
||||
for key, value in updates.items():
|
||||
if key not in merged or value is None:
|
||||
continue
|
||||
if key == "smtp_password":
|
||||
pwd = str(value).strip()
|
||||
if not pwd or pwd == PASSWORD_PLACEHOLDER:
|
||||
continue
|
||||
merged[key] = pwd
|
||||
continue
|
||||
if key in ("wechat_api_v3_key", "wechat_private_key", "alipay_private_key"):
|
||||
secret = str(value).strip()
|
||||
if not secret or secret == PASSWORD_PLACEHOLDER:
|
||||
continue
|
||||
merged[key] = secret
|
||||
continue
|
||||
merged[key] = value
|
||||
data = _parse_settings(merged)
|
||||
|
||||
result = await db.execute(select(AppConfig).where(AppConfig.id == CONFIG_ROW_ID))
|
||||
row = result.scalar_one_or_none()
|
||||
if not row:
|
||||
row = AppConfig(id=CONFIG_ROW_ID, data="{}", updated_at=datetime.utcnow())
|
||||
db.add(row)
|
||||
row.data = json.dumps(asdict(data), ensure_ascii=False)
|
||||
row.updated_at = datetime.utcnow()
|
||||
await db.commit()
|
||||
set_cached_settings(data)
|
||||
return data
|
||||
|
||||
|
||||
def settings_to_public(data: SystemSettingsData) -> dict[str, bool]:
|
||||
return {
|
||||
"registration_enabled": data.registration_enabled,
|
||||
"email_verification_required": data.email_verification_required,
|
||||
"email_binding_required": data.email_binding_required,
|
||||
}
|
||||
|
||||
|
||||
def settings_to_admin_response(data: SystemSettingsData) -> dict[str, Any]:
|
||||
payload = asdict(data)
|
||||
payload["smtp_password"] = PASSWORD_PLACEHOLDER if data.smtp_password else ""
|
||||
payload["smtp_password_configured"] = bool(data.smtp_password)
|
||||
return payload
|
||||
|
||||
|
||||
PAYMENT_SETTING_KEYS = (
|
||||
"payment_enabled",
|
||||
"payment_demo_mode",
|
||||
"wechat_pay_enabled",
|
||||
"alipay_pay_enabled",
|
||||
"account_slot_unit_price",
|
||||
"account_slot_purchase_min",
|
||||
"account_slot_purchase_max",
|
||||
"wechat_app_id",
|
||||
"wechat_mch_id",
|
||||
"wechat_api_v3_key",
|
||||
"wechat_cert_serial",
|
||||
"wechat_private_key",
|
||||
"alipay_app_id",
|
||||
"alipay_private_key",
|
||||
"alipay_public_key",
|
||||
"alipay_sandbox",
|
||||
)
|
||||
|
||||
|
||||
def settings_to_payment_response(data: SystemSettingsData) -> dict[str, Any]:
|
||||
payload = {key: getattr(data, key) for key in PAYMENT_SETTING_KEYS}
|
||||
payload["app_url"] = data.app_url
|
||||
payload["wechat_api_v3_key"] = PASSWORD_PLACEHOLDER if data.wechat_api_v3_key else ""
|
||||
payload["wechat_api_v3_key_configured"] = bool(data.wechat_api_v3_key)
|
||||
payload["wechat_private_key"] = PASSWORD_PLACEHOLDER if data.wechat_private_key else ""
|
||||
payload["wechat_private_key_configured"] = bool(data.wechat_private_key)
|
||||
payload["alipay_private_key"] = PASSWORD_PLACEHOLDER if data.alipay_private_key else ""
|
||||
payload["alipay_private_key_configured"] = bool(data.alipay_private_key)
|
||||
payload["wechat_pay_configured"] = data.wechat_pay_configured()
|
||||
payload["alipay_configured"] = data.alipay_configured()
|
||||
return payload
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# 宝塔一键修复:权限 + 安装依赖 + 提示正确启动命令
|
||||
# SSH 执行: bash /www/wwwroot/douyin/backend/baota_fix.sh
|
||||
set -euo pipefail
|
||||
|
||||
BACKEND="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$BACKEND/.." && pwd)"
|
||||
# shellcheck source=lib/resolve_python.sh
|
||||
source "$BACKEND/lib/resolve_python.sh"
|
||||
|
||||
echo "========================================"
|
||||
echo " 宝塔环境修复"
|
||||
echo "========================================"
|
||||
|
||||
# 权限
|
||||
chown -R www:www "$ROOT" 2>/dev/null || true
|
||||
chmod +x "$BACKEND/baota_start.sh" "$BACKEND/baota_init.sh" 2>/dev/null || true
|
||||
chmod +x "$ROOT/install.sh" "$ROOT/start_web.sh" "$ROOT/start_backend.sh" 2>/dev/null || true
|
||||
if [ -d "$BACKEND/.venv/bin" ]; then
|
||||
chmod -R 755 "$BACKEND/.venv"
|
||||
chmod +x "$BACKEND/.venv/bin/"* 2>/dev/null || true
|
||||
fi
|
||||
echo "[OK] 权限已修复"
|
||||
|
||||
if [ -n "${KEFU_PYTHON:-}" ] && [ -x "$KEFU_PYTHON" ]; then
|
||||
PY="$KEFU_PYTHON"
|
||||
if ! "$PY" -c "import ssl" 2>/dev/null; then
|
||||
echo "[错误] KEFU_PYTHON 无 SSL: $KEFU_PYTHON"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
PY="$(resolve_python "$BACKEND" "$ROOT")" || {
|
||||
echo
|
||||
echo "请先编译可用 Python:"
|
||||
echo " AUTO_BUILD_PYTHON=1 bash $ROOT/install.sh"
|
||||
exit 1
|
||||
}
|
||||
fi
|
||||
echo "[OK] Python: $PY ($("$PY" -V 2>&1))"
|
||||
|
||||
if ! "$PY" -c "import ssl" 2>/dev/null; then
|
||||
echo "[错误] 该 Python 无 SSL,请换环境或执行: AUTO_BUILD_PYTHON=1 $ROOT/install.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "[1/3] pip install requirements..."
|
||||
"$PY" -m pip install -U pip
|
||||
"$PY" -m pip install -r "$BACKEND/requirements.txt"
|
||||
echo "[OK] Python 依赖已安装"
|
||||
|
||||
echo "[2/3] 校验 uvicorn..."
|
||||
"$PY" -c "import uvicorn, fastapi; print(' uvicorn + fastapi OK')"
|
||||
|
||||
echo "[3/3] 可选:完整初始化 Playwright/前端"
|
||||
echo " bash $BACKEND/baota_init.sh"
|
||||
echo
|
||||
echo "========================================"
|
||||
echo " 请在宝塔面板修改以下配置:"
|
||||
echo
|
||||
echo " 项目路径: $BACKEND"
|
||||
echo " 启动命令: bash $BACKEND/baota_start.sh"
|
||||
echo " 初始化命令: bash $BACKEND/baota_init.sh"
|
||||
echo " 依赖包: $BACKEND/requirements.txt"
|
||||
echo
|
||||
echo " 不要用:"
|
||||
echo " backend/.venv/bin/uvicorn (不存在)"
|
||||
echo " start_web.bat (Windows 脚本,Linux 不能运行)"
|
||||
echo "========================================"
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
# 宝塔 Python 项目 - 项目初始化命令
|
||||
# 在面板填写: /www/wwwroot/douyin/backend/baota_init.sh
|
||||
# 执行时机: 宝塔创建虚拟环境并 pip install requirements.txt 之后
|
||||
set -euo pipefail
|
||||
|
||||
BACKEND="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$BACKEND/.." && pwd)"
|
||||
# shellcheck source=lib/resolve_python.sh
|
||||
source "$BACKEND/lib/resolve_python.sh"
|
||||
|
||||
PY="$(resolve_python "$BACKEND" "$ROOT")"
|
||||
ensure_uvicorn "$PY" "$BACKEND"
|
||||
|
||||
echo "========================================"
|
||||
echo " 抖音客服 - 宝塔项目初始化"
|
||||
echo " Python: $("$PY" -V 2>&1) @ $PY"
|
||||
echo "========================================"
|
||||
|
||||
# 校验 Python
|
||||
"$PY" -c "import ssl; print('[OK] OpenSSL:', ssl.OPENSSL_VERSION)"
|
||||
|
||||
echo "[1/4] Playwright Chromium..."
|
||||
KEFU_PYTHON="$PY" bash "$BACKEND/baota_playwright.sh"
|
||||
|
||||
echo "[2/4] IM 签名 Node 依赖..."
|
||||
if command -v npm &>/dev/null; then
|
||||
cd "$BACKEND/rpa_engine/douyin_im/static"
|
||||
npm install --no-fund --no-audit
|
||||
else
|
||||
echo "[警告] 未找到 npm,请宝塔安装 Node 18+ 后重新运行本脚本"
|
||||
fi
|
||||
|
||||
echo "[3/4] 前端依赖与构建..."
|
||||
if command -v npm &>/dev/null && [ -f "$ROOT/frontend/package.json" ]; then
|
||||
cd "$ROOT/frontend"
|
||||
npm install --no-fund --no-audit
|
||||
npm run build
|
||||
else
|
||||
echo "[警告] 跳过前端构建"
|
||||
fi
|
||||
|
||||
echo "[4/4] 校验..."
|
||||
cd "$BACKEND"
|
||||
"$PY" -c "import fastapi, uvicorn, playwright; print(' 核心包 OK')"
|
||||
[ -f "$ROOT/frontend/dist/index.html" ] && echo " 前端静态 OK" || echo "[警告] 未找到 frontend/dist"
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
echo " 宝塔初始化完成"
|
||||
echo " 启动命令请用: bash $BACKEND/baota_start.sh"
|
||||
echo "========================================"
|
||||
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
# 安装 Playwright Chromium(Linux 宝塔/无桌面服务器)
|
||||
# 用法: bash /www/wwwroot/douyin/backend/baota_playwright.sh
|
||||
set -euo pipefail
|
||||
|
||||
BACKEND="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$BACKEND/.." && pwd)"
|
||||
# shellcheck source=lib/resolve_python.sh
|
||||
source "$BACKEND/lib/resolve_python.sh"
|
||||
|
||||
PY="${KEFU_PYTHON:-}"
|
||||
if [ -z "$PY" ]; then
|
||||
PY="$(resolve_python "$BACKEND" "$ROOT")"
|
||||
fi
|
||||
|
||||
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$ROOT/playwright-browsers}"
|
||||
mkdir -p "$PLAYWRIGHT_BROWSERS_PATH"
|
||||
chown -R www:www "$PLAYWRIGHT_BROWSERS_PATH" 2>/dev/null || true
|
||||
|
||||
echo "========================================"
|
||||
echo " Playwright Chromium 安装"
|
||||
echo " Python: $PY"
|
||||
echo " 目录: $PLAYWRIGHT_BROWSERS_PATH"
|
||||
echo "========================================"
|
||||
|
||||
echo "[1/4] 安装系统依赖(需 root)..."
|
||||
if [ "$(uname -s)" = "Linux" ]; then
|
||||
if [ "$(id -u)" -eq 0 ]; then
|
||||
# 优先用 dnf(阿里云 Linux 3 / Anolis / RHEL 8+),回退到 yum
|
||||
DNF=""
|
||||
if command -v dnf &>/dev/null; then
|
||||
DNF="dnf"
|
||||
elif command -v yum &>/dev/null; then
|
||||
DNF="yum"
|
||||
fi
|
||||
|
||||
if [ -n "$DNF" ]; then
|
||||
# Chromium 运行库 + Xvfb 虚拟显示(无桌面服务器跑有头浏览器扫码登录必需)
|
||||
"$DNF" install -y atk at-spi2-atk cups-libs libdrm libXcomposite libXdamage \
|
||||
libXrandr mesa-libgbm pango alsa-lib nss nspr libxkbcommon \
|
||||
libXScrnSaver gtk3 xorg-x11-server-Xvfb 2>/dev/null || true
|
||||
elif command -v apt-get &>/dev/null; then
|
||||
apt-get update
|
||||
apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 \
|
||||
libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 \
|
||||
libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2 \
|
||||
libatspi2.0-0 libxshmfence1 xvfb 2>/dev/null || true
|
||||
fi
|
||||
"$PY" -m playwright install-deps chromium || true
|
||||
else
|
||||
echo " 非 root,跳过 install-deps / Xvfb(若启动失败请 sudo 执行本脚本)"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "[2/4] 安装 pyvirtualdisplay(自动拉起 Xvfb 虚拟显示)..."
|
||||
# requirements.txt 已含 pyvirtualdisplay;这里兜底确保 venv 内已安装
|
||||
"$PY" -m pip show pyvirtualdisplay &>/dev/null \
|
||||
|| "$PY" -m pip install "pyvirtualdisplay==3.0" || true
|
||||
|
||||
echo "[3/4] 下载 Chromium..."
|
||||
"$PY" -m playwright install chromium
|
||||
|
||||
echo "[4/4] 校验..."
|
||||
CHROME=$(find "$PLAYWRIGHT_BROWSERS_PATH" -name chrome -type f 2>/dev/null | head -1)
|
||||
if [ -n "$CHROME" ] && [ -x "$CHROME" ]; then
|
||||
echo "[OK] 浏览器: $CHROME"
|
||||
chmod +x "$CHROME" 2>/dev/null || true
|
||||
chown -R www:www "$PLAYWRIGHT_BROWSERS_PATH" 2>/dev/null || true
|
||||
else
|
||||
echo "[错误] 未找到 chrome 可执行文件,请检查网络或磁盘空间"
|
||||
ls -la "$PLAYWRIGHT_BROWSERS_PATH" 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo
|
||||
# 校验虚拟显示链路:无桌面 Linux 上扫码登录依赖 Xvfb + pyvirtualdisplay
|
||||
if [ "$(uname -s)" = "Linux" ] && [ -z "${DISPLAY:-}" ]; then
|
||||
XVFB_OK=0
|
||||
command -v Xvfb &>/dev/null && XVFB_OK=1
|
||||
PVD_OK=0
|
||||
"$PY" -c "import pyvirtualdisplay" &>/dev/null && PVD_OK=1
|
||||
if [ "$XVFB_OK" = "1" ] && [ "$PVD_OK" = "1" ]; then
|
||||
echo "[OK] Xvfb + pyvirtualdisplay 就绪,云端可直接点「登录」用手机扫码"
|
||||
else
|
||||
echo "[警告] 扫码登录所需的虚拟显示未就绪:"
|
||||
[ "$XVFB_OK" = "0" ] && echo " - 缺少 Xvfb,请 root 执行: dnf install -y xorg-x11-server-Xvfb"
|
||||
[ "$PVD_OK" = "0" ] && echo " - 缺少 pyvirtualdisplay,请执行: $PY -m pip install pyvirtualdisplay==3.0"
|
||||
echo " 或改用: xvfb-run -a bash $ROOT/start_web.sh 启动服务"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "请将以下写入 .env 或宝塔环境变量:"
|
||||
echo " PLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH"
|
||||
echo
|
||||
echo "安装完成,重启项目: bash $BACKEND/baota_start.sh"
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# 宝塔 Python 项目 - 启动命令
|
||||
# 面板「启动命令」必须填这一行(不要用 .venv/bin/uvicorn):
|
||||
# bash /www/wwwroot/douyin/backend/baota_start.sh
|
||||
set -euo pipefail
|
||||
|
||||
BACKEND="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$BACKEND/.." && pwd)"
|
||||
# shellcheck source=lib/resolve_python.sh
|
||||
source "$BACKEND/lib/resolve_python.sh"
|
||||
|
||||
cd "$BACKEND"
|
||||
|
||||
PY="$(resolve_python "$BACKEND" "$ROOT")"
|
||||
ensure_uvicorn "$PY" "$BACKEND"
|
||||
|
||||
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$ROOT/playwright-browsers}"
|
||||
export KEFU_SERVE_WEB="${KEFU_SERVE_WEB:-true}"
|
||||
export KEFU_STATIC_DIR="${KEFU_STATIC_DIR:-$ROOT/frontend/dist}"
|
||||
|
||||
# 启动前检查 Chromium(缺失时给出明确提示)
|
||||
if ! find "$PLAYWRIGHT_BROWSERS_PATH" -name chrome -type f 2>/dev/null | grep -q .; then
|
||||
echo "[警告] 未找到 Chromium,请执行: bash $BACKEND/baota_playwright.sh" >&2
|
||||
fi
|
||||
|
||||
if [ -f "$ROOT/.env" ]; then
|
||||
set -a
|
||||
# shellcheck disable=SC1090
|
||||
source "$ROOT/.env"
|
||||
set +a
|
||||
fi
|
||||
|
||||
HOST="${KEFU_HOST:-0.0.0.0}"
|
||||
PORT="${KEFU_PORT:-8000}"
|
||||
|
||||
echo "[baota] Python: $PY"
|
||||
echo "[baota] Listen: http://${HOST}:${PORT}"
|
||||
|
||||
exec "$PY" -m uvicorn main:app --host "$HOST" --port "$PORT"
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env bash
|
||||
# 宝塔/服务器快速诊断:服务是否在跑、端口是否监听
|
||||
set -uo pipefail
|
||||
|
||||
BACKEND="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$BACKEND/.." && pwd)"
|
||||
PORT="${KEFU_PORT:-8000}"
|
||||
|
||||
echo "========================================"
|
||||
echo " 抖音客服 - 运行诊断"
|
||||
echo "========================================"
|
||||
echo
|
||||
|
||||
echo "[1] 端口 ${PORT} 监听状态:"
|
||||
if command -v ss &>/dev/null; then
|
||||
ss -tlnp | grep ":${PORT} " || echo " -> 未监听(服务未启动)"
|
||||
elif command -v netstat &>/dev/null; then
|
||||
netstat -tlnp 2>/dev/null | grep ":${PORT} " || echo " -> 未监听(服务未启动)"
|
||||
else
|
||||
echo " (无法检测,请安装 ss 或 netstat)"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "[2] Python 环境:"
|
||||
BROKEN_BT="/home/www/server/pyporject_evn/versions/3.11.15/bin/python"
|
||||
if [ -x "$BROKEN_BT" ] && ! "$BROKEN_BT" -c "import ssl" 2>/dev/null; then
|
||||
echo " [X] 宝塔 Python 无 SSL(不可用): $BROKEN_BT"
|
||||
echo " 修复: AUTO_BUILD_PYTHON=1 bash $ROOT/install.sh"
|
||||
fi
|
||||
if [ -f "$BACKEND/lib/resolve_python.sh" ]; then
|
||||
# shellcheck source=lib/resolve_python.sh
|
||||
source "$BACKEND/lib/resolve_python.sh"
|
||||
if PY="$(resolve_python "$BACKEND" "$ROOT" 2>/dev/null)"; then
|
||||
echo " Python: $PY"
|
||||
"$PY" -V 2>&1 | sed 's/^/ /'
|
||||
if "$PY" -c "import ssl" 2>/dev/null; then
|
||||
"$PY" -c "import ssl; print(' SSL:', ssl.OPENSSL_VERSION)"
|
||||
else
|
||||
echo " SSL: 无(不可用)"
|
||||
fi
|
||||
if "$PY" -c "import uvicorn" 2>/dev/null; then
|
||||
echo " uvicorn: 已安装"
|
||||
else
|
||||
echo " uvicorn: 未安装 -> 运行 bash $BACKEND/baota_fix.sh"
|
||||
fi
|
||||
else
|
||||
echo " 未找到可用 Python 3.10+"
|
||||
fi
|
||||
else
|
||||
command -v python && python -V || echo " 未找到 python"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "[3] 前端静态:"
|
||||
if [ -f "$ROOT/frontend/dist/index.html" ]; then
|
||||
echo " OK: $ROOT/frontend/dist/index.html"
|
||||
else
|
||||
echo " 缺失 -> 运行 bash $BACKEND/baota_init.sh"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "[4] 宝塔项目进程:"
|
||||
ps aux 2>/dev/null | grep -E "uvicorn|douyin" | grep -v grep || echo " 未发现 uvicorn 进程"
|
||||
echo
|
||||
|
||||
echo "[5] 防火墙(CentOS firewalld):"
|
||||
if command -v firewall-cmd &>/dev/null && systemctl is-active firewalld &>/dev/null; then
|
||||
firewall-cmd --list-ports 2>/dev/null | grep -q "${PORT}" \
|
||||
&& echo " 端口 ${PORT} 已放行" \
|
||||
|| echo " 端口 ${PORT} 未放行 -> firewall-cmd --add-port=${PORT}/tcp --permanent && firewall-cmd --reload"
|
||||
else
|
||||
echo " firewalld 未运行或未安装(可忽略,检查云服务器安全组)"
|
||||
fi
|
||||
echo
|
||||
|
||||
echo "========================================"
|
||||
echo " 若端口未监听,按顺序执行:"
|
||||
echo " 若宝塔 Python 无 SSL,必须先编译可用 Python:"
|
||||
echo " AUTO_BUILD_PYTHON=1 bash $ROOT/install.sh"
|
||||
echo " KEFU_PYTHON=/usr/local/python3.11/bin/python3.11 bash $BACKEND/baota_fix.sh"
|
||||
echo " bash $BACKEND/baota_init.sh"
|
||||
echo " 宝塔启动命令: bash $BACKEND/baota_start.sh"
|
||||
echo
|
||||
echo " 手动测试启动:"
|
||||
echo " bash $BACKEND/baota_start.sh"
|
||||
echo "========================================"
|
||||
@@ -0,0 +1,104 @@
|
||||
#!/usr/bin/env bash
|
||||
# 使用宝塔「Python环境管理」安装的 Python 完成项目配置
|
||||
# 用法: bash /www/wwwroot/douyin/backend/baota_use_panel_python.sh
|
||||
set -euo pipefail
|
||||
|
||||
BACKEND="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT="$(cd "$BACKEND/.." && pwd)"
|
||||
|
||||
echo "========================================"
|
||||
echo " 宝塔 Python 环境配置"
|
||||
echo "========================================"
|
||||
echo
|
||||
|
||||
# 扫描宝塔 Python(环境管理 / Python项目)
|
||||
PY=""
|
||||
for base in \
|
||||
/home/www/server/pyporject_evn/versions \
|
||||
/www/server/pyporject_evn/versions \
|
||||
/www/server/panel/pyenv/versions; do
|
||||
[ -d "$base" ] || continue
|
||||
for ver_dir in $(ls -d "$base"/* 2>/dev/null | sort -V -r); do
|
||||
for name in python3 python; do
|
||||
candidate="$ver_dir/bin/$name"
|
||||
[ -x "$candidate" ] || continue
|
||||
ver="$("$candidate" -V 2>&1 | awk '{print $2}')"
|
||||
echo "[发现] $candidate ($ver)"
|
||||
if ! "$candidate" -c "import sys; sys.exit(0 if sys.version_info>=(3,10) else 1)" 2>/dev/null; then
|
||||
echo " 跳过: 版本低于 3.10"
|
||||
continue
|
||||
fi
|
||||
if ! "$candidate" -c "import ssl" 2>/dev/null; then
|
||||
echo " 跳过: 无 SSL(需在宝塔重装 Python 前先 yum install openssl-devel)"
|
||||
continue
|
||||
fi
|
||||
PY="$candidate"
|
||||
echo " -> 选用"
|
||||
break 3
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
if [ -z "$PY" ]; then
|
||||
echo
|
||||
echo "[错误] 未找到可用的宝塔 Python 3.10+(需带 SSL)。"
|
||||
echo
|
||||
echo "请在宝塔面板操作:"
|
||||
echo " 1) 软件商店 -> Python项目管理器 -> 环境管理"
|
||||
echo " 2) SSH 先执行: yum install -y openssl openssl-devel"
|
||||
echo " 3) 环境管理 -> 安装 Python 3.11"
|
||||
echo " 4) 重新运行本脚本"
|
||||
echo
|
||||
echo "若已安装但仍无 SSL,卸载该版本后重装。"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo
|
||||
echo "[OK] 使用宝塔 Python: $PY"
|
||||
"$PY" -c "import ssl; print('[OK] OpenSSL:', ssl.OPENSSL_VERSION)"
|
||||
echo
|
||||
|
||||
export KEFU_PYTHON="$PY"
|
||||
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$ROOT/playwright-browsers}"
|
||||
|
||||
echo "[1/4] 安装 Python 依赖..."
|
||||
"$PY" -m pip install -U pip
|
||||
"$PY" -m pip install -r "$BACKEND/requirements.txt"
|
||||
echo
|
||||
|
||||
echo "[2/4] 创建项目 venv(可选,便于宝塔启动)..."
|
||||
"$PY" -m venv "$BACKEND/.venv" --clear
|
||||
"$BACKEND/.venv/bin/python" -m pip install -U pip -q
|
||||
"$BACKEND/.venv/bin/pip" install -r "$BACKEND/requirements.txt"
|
||||
chmod -R 755 "$BACKEND/.venv"
|
||||
chown -R www:www "$BACKEND/.venv" 2>/dev/null || true
|
||||
echo " venv: $BACKEND/.venv"
|
||||
echo
|
||||
|
||||
echo "[3/4] Playwright Chromium..."
|
||||
KEFU_PYTHON="$BACKEND/.venv/bin/python" bash "$BACKEND/baota_playwright.sh"
|
||||
echo
|
||||
|
||||
echo "[4/4] 前端与 IM 依赖..."
|
||||
if command -v npm &>/dev/null; then
|
||||
cd "$BACKEND/rpa_engine/douyin_im/static" && npm install --no-fund --no-audit
|
||||
cd "$ROOT/frontend" && npm install --no-fund --no-audit && npm run build
|
||||
else
|
||||
echo "[警告] 未安装 Node.js,请宝塔安装 Node 18+ 后手动 npm install"
|
||||
fi
|
||||
|
||||
chown -R www:www "$ROOT" 2>/dev/null || true
|
||||
|
||||
echo
|
||||
echo "========================================"
|
||||
echo " 配置完成"
|
||||
echo
|
||||
echo " 宝塔 Python项目 设置:"
|
||||
echo " 项目路径: $BACKEND"
|
||||
echo " Python环境: 选环境管理里的 3.11"
|
||||
echo " 启动命令: bash $BACKEND/baota_start.sh"
|
||||
echo " 依赖包: $BACKEND/requirements.txt"
|
||||
echo
|
||||
echo " .env 添加:"
|
||||
echo " PLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH"
|
||||
echo "========================================"
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 318 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 296 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.6 KiB |
@@ -0,0 +1,116 @@
|
||||
"""桌面客户端在线升级:发布配置(存 AppConfig 行)+ 安装包文件管理。
|
||||
|
||||
发布配置以 JSON 形式存放在 app_config 表的 id=DESKTOP_CONFIG_ROW_ID 行,
|
||||
安装包文件落在 backend/uploads/desktop/ 下。仅管理员可读写。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict, dataclass, fields
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import AppConfig
|
||||
|
||||
# app_config 表里给桌面发布单独用一行(系统设置用的是 id=1)
|
||||
DESKTOP_CONFIG_ROW_ID = 2
|
||||
|
||||
INSTALLER_DIR = os.path.join(os.path.dirname(__file__), "uploads", "desktop")
|
||||
INSTALLER_FILENAME = "DouyinHostedDesktop-Setup.exe"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DesktopReleaseData:
|
||||
version: str = "" # 已发布的最新版本号,如 1.0.1
|
||||
force: bool = False # true=强制升级(不可跳过)
|
||||
notes: str = "" # 更新说明
|
||||
installer_name: str = "" # 上传时的原始文件名
|
||||
installer_size: int = 0 # 安装包字节数
|
||||
installer_url: str = "" # 外部安装包直链(填写后优先于本地上传文件)
|
||||
updated_at: str = "" # 最近一次发布/上传时间
|
||||
|
||||
|
||||
def package_ready(data: "DesktopReleaseData") -> bool:
|
||||
"""是否具备可供客户端下载的安装包:外部直链 或 本地已上传文件。"""
|
||||
return bool((data.installer_url or "").strip()) or installer_exists()
|
||||
|
||||
|
||||
def installer_path() -> str:
|
||||
return os.path.join(INSTALLER_DIR, INSTALLER_FILENAME)
|
||||
|
||||
|
||||
def installer_exists() -> bool:
|
||||
p = installer_path()
|
||||
return os.path.isfile(p) and os.path.getsize(p) > 0
|
||||
|
||||
|
||||
def _coerce_bool(value: Any, default: bool) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.lower() in ("1", "true", "yes", "on")
|
||||
if value is None:
|
||||
return default
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _parse(raw: dict[str, Any]) -> DesktopReleaseData:
|
||||
base = DesktopReleaseData()
|
||||
allowed = {f.name for f in fields(DesktopReleaseData)}
|
||||
merged = {k: raw[k] for k in allowed if k in raw}
|
||||
if "force" in merged:
|
||||
merged["force"] = _coerce_bool(merged["force"], base.force)
|
||||
if "installer_size" in merged:
|
||||
try:
|
||||
merged["installer_size"] = int(merged["installer_size"])
|
||||
except (TypeError, ValueError):
|
||||
merged["installer_size"] = base.installer_size
|
||||
return DesktopReleaseData(**{**asdict(base), **merged})
|
||||
|
||||
|
||||
async def load_release(db: AsyncSession) -> DesktopReleaseData:
|
||||
row = (
|
||||
await db.execute(select(AppConfig).where(AppConfig.id == DESKTOP_CONFIG_ROW_ID))
|
||||
).scalar_one_or_none()
|
||||
if not row or not row.data:
|
||||
return DesktopReleaseData()
|
||||
try:
|
||||
payload = json.loads(row.data)
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
return _parse(payload if isinstance(payload, dict) else {})
|
||||
|
||||
|
||||
async def save_release(db: AsyncSession, updates: dict[str, Any]) -> DesktopReleaseData:
|
||||
current = await load_release(db)
|
||||
merged = asdict(current)
|
||||
for key, value in updates.items():
|
||||
if key not in merged or value is None:
|
||||
continue
|
||||
merged[key] = value
|
||||
data = _parse(merged)
|
||||
data.updated_at = datetime.utcnow().isoformat()
|
||||
|
||||
row = (
|
||||
await db.execute(select(AppConfig).where(AppConfig.id == DESKTOP_CONFIG_ROW_ID))
|
||||
).scalar_one_or_none()
|
||||
if not row:
|
||||
row = AppConfig(id=DESKTOP_CONFIG_ROW_ID, data="{}", updated_at=datetime.utcnow())
|
||||
db.add(row)
|
||||
row.data = json.dumps(asdict(data), ensure_ascii=False)
|
||||
row.updated_at = datetime.utcnow()
|
||||
await db.commit()
|
||||
return data
|
||||
|
||||
|
||||
def remove_installer() -> None:
|
||||
p = installer_path()
|
||||
try:
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
except OSError:
|
||||
pass
|
||||
@@ -0,0 +1,201 @@
|
||||
"""桌面客户端在线升级接口。
|
||||
|
||||
公开接口(桌面客户端启动时调用,无需登录):
|
||||
GET /api/desktop/latest 返回最新版本清单 {version,url,force,notes},无发布时返回 {}
|
||||
GET /api/desktop/download 下载安装包
|
||||
|
||||
管理员接口(仅 admin):
|
||||
GET /api/desktop/release 读取当前发布配置
|
||||
PUT /api/desktop/release 修改版本号/强制升级/更新说明
|
||||
POST /api/desktop/release/installer 上传安装包(.exe)
|
||||
DELETE /api/desktop/release/installer 删除安装包(即停用在线升级)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import FileResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from auth.dependencies import require_admin
|
||||
from auth.system_settings import get_cached_settings
|
||||
from desktop_release import (
|
||||
INSTALLER_DIR,
|
||||
installer_exists,
|
||||
installer_path,
|
||||
load_release,
|
||||
package_ready,
|
||||
remove_installer,
|
||||
save_release,
|
||||
)
|
||||
from models.database import get_db
|
||||
from models.models import User
|
||||
|
||||
router = APIRouter(prefix="/api/desktop", tags=["desktop-update"])
|
||||
|
||||
_MAX_INSTALLER_BYTES = 500 * 1024 * 1024 # 500MB 上限,足够覆盖安装包
|
||||
|
||||
|
||||
class DesktopReleaseResponse(BaseModel):
|
||||
version: str = ""
|
||||
force: bool = False
|
||||
notes: str = ""
|
||||
installer_name: str = ""
|
||||
installer_size: int = 0
|
||||
installer_url: str = ""
|
||||
updated_at: str = ""
|
||||
has_installer: bool = False # 本地是否已上传安装包文件
|
||||
package_ready: bool = False # 是否具备可下载安装包(外链或本地文件)
|
||||
download_url: str = "" # 本地上传文件的下载地址
|
||||
|
||||
|
||||
class DesktopReleaseUpdate(BaseModel):
|
||||
version: str | None = Field(default=None, max_length=40)
|
||||
force: bool | None = None
|
||||
notes: str | None = Field(default=None, max_length=4000)
|
||||
installer_url: str | None = Field(default=None, max_length=1000)
|
||||
|
||||
|
||||
def _public_base(request: Request) -> str:
|
||||
"""构造给客户端用的绝对地址:优先后台配置的 app_url,本地址兜底用请求来源。"""
|
||||
try:
|
||||
app_url = get_cached_settings().app_url_normalized()
|
||||
except Exception:
|
||||
app_url = ""
|
||||
if not app_url or "localhost" in app_url or "127.0.0.1" in app_url:
|
||||
app_url = str(request.base_url).rstrip("/")
|
||||
return app_url
|
||||
|
||||
|
||||
def _to_response(data, request: Request) -> DesktopReleaseResponse:
|
||||
return DesktopReleaseResponse(
|
||||
version=data.version,
|
||||
force=data.force,
|
||||
notes=data.notes,
|
||||
installer_name=data.installer_name,
|
||||
installer_size=data.installer_size,
|
||||
installer_url=(data.installer_url or "").strip(),
|
||||
updated_at=data.updated_at,
|
||||
has_installer=installer_exists(),
|
||||
package_ready=package_ready(data),
|
||||
download_url=f"{_public_base(request)}/api/desktop/download",
|
||||
)
|
||||
|
||||
|
||||
# ---------- 公开接口 ----------
|
||||
|
||||
@router.get("/latest")
|
||||
async def desktop_latest(request: Request, db: AsyncSession = Depends(get_db)):
|
||||
"""桌面客户端启动检查更新。未发布或安装包缺失时返回空对象。"""
|
||||
data = await load_release(db)
|
||||
if not data.version or not package_ready(data):
|
||||
return {}
|
||||
external = (data.installer_url or "").strip()
|
||||
download_url = external or f"{_public_base(request)}/api/desktop/download"
|
||||
return {
|
||||
"version": data.version,
|
||||
"url": download_url,
|
||||
"force": bool(data.force),
|
||||
"notes": data.notes or "",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/download")
|
||||
async def desktop_download(db: AsyncSession = Depends(get_db)):
|
||||
"""下载安装包。"""
|
||||
if not installer_exists():
|
||||
raise HTTPException(status_code=404, detail="安装包不存在")
|
||||
return FileResponse(
|
||||
installer_path(),
|
||||
media_type="application/octet-stream",
|
||||
filename="DouyinHostedDesktop-Setup.exe",
|
||||
)
|
||||
|
||||
|
||||
# ---------- 管理员接口 ----------
|
||||
|
||||
@router.get("/release", response_model=DesktopReleaseResponse)
|
||||
async def get_release(
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
data = await load_release(db)
|
||||
return _to_response(data, request)
|
||||
|
||||
|
||||
@router.put("/release", response_model=DesktopReleaseResponse)
|
||||
async def update_release(
|
||||
body: DesktopReleaseUpdate,
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
updates = body.model_dump(exclude_unset=True)
|
||||
if "version" in updates and updates["version"] is not None:
|
||||
updates["version"] = updates["version"].strip()
|
||||
if "notes" in updates and updates["notes"] is not None:
|
||||
updates["notes"] = updates["notes"].strip()
|
||||
if "installer_url" in updates and updates["installer_url"] is not None:
|
||||
url = updates["installer_url"].strip()
|
||||
if url and not url.lower().startswith(("http://", "https://")):
|
||||
raise HTTPException(status_code=400, detail="安装包网址需以 http:// 或 https:// 开头")
|
||||
updates["installer_url"] = url
|
||||
data = await save_release(db, updates)
|
||||
return _to_response(data, request)
|
||||
|
||||
|
||||
@router.post("/release/installer", response_model=DesktopReleaseResponse)
|
||||
async def upload_installer(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
filename = (file.filename or "").strip()
|
||||
if not filename.lower().endswith(".exe"):
|
||||
raise HTTPException(status_code=400, detail="请上传 .exe 安装包")
|
||||
|
||||
os.makedirs(INSTALLER_DIR, exist_ok=True)
|
||||
tmp_path = installer_path() + ".uploading"
|
||||
size = 0
|
||||
try:
|
||||
with open(tmp_path, "wb") as f:
|
||||
while True:
|
||||
chunk = await file.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > _MAX_INSTALLER_BYTES:
|
||||
raise HTTPException(status_code=400, detail="安装包超过 500MB 上限")
|
||||
f.write(chunk)
|
||||
except HTTPException:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
raise HTTPException(status_code=500, detail=f"保存安装包失败: {exc}") from exc
|
||||
|
||||
if size == 0:
|
||||
if os.path.exists(tmp_path):
|
||||
os.remove(tmp_path)
|
||||
raise HTTPException(status_code=400, detail="安装包为空")
|
||||
|
||||
os.replace(tmp_path, installer_path())
|
||||
data = await save_release(db, {"installer_name": filename, "installer_size": size})
|
||||
return _to_response(data, request)
|
||||
|
||||
|
||||
@router.delete("/release/installer", response_model=DesktopReleaseResponse)
|
||||
async def delete_installer(
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
remove_installer()
|
||||
data = await save_release(db, {"installer_name": "", "installer_size": 0})
|
||||
return _to_response(data, request)
|
||||
@@ -0,0 +1,29 @@
|
||||
"""帮助中心静态页面。"""
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from settings import BACKEND_DIR, PROJECT_ROOT
|
||||
|
||||
HELP_STATIC_DIR = BACKEND_DIR / "static" / "help"
|
||||
CREDENTIAL_TOOL_CANDIDATES = (
|
||||
PROJECT_ROOT / "凭证采集工具.html",
|
||||
HELP_STATIC_DIR / "credential-tool.html",
|
||||
)
|
||||
|
||||
|
||||
def _resolve_credential_tool_html() -> Path:
|
||||
for path in CREDENTIAL_TOOL_CANDIDATES:
|
||||
if path.is_file():
|
||||
return path
|
||||
raise HTTPException(status_code=404, detail="凭证采集工具页面不存在")
|
||||
|
||||
|
||||
def serve_credential_tool() -> FileResponse:
|
||||
html_path = _resolve_credential_tool_html()
|
||||
return FileResponse(
|
||||
html_path,
|
||||
media_type="text/html; charset=utf-8",
|
||||
content_disposition_type="inline",
|
||||
)
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 211 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 211 KiB |
@@ -0,0 +1,77 @@
|
||||
#!/usr/bin/env bash
|
||||
# 共用:解析可用 Python(必须 3.10+ 且带 SSL,跳过宝塔坏环境)
|
||||
_py_ok() {
|
||||
local py="$1"
|
||||
[ -n "$py" ] && [ -x "$py" ] || return 1
|
||||
"$py" -c "import sys, ssl; sys.exit(0 if sys.version_info>=(3,10) else 1)" 2>/dev/null
|
||||
}
|
||||
|
||||
resolve_python() {
|
||||
local backend="${1:?}"
|
||||
local root="${2:-$(cd "$backend/.." && pwd)}"
|
||||
local py="" candidate ver_dir
|
||||
|
||||
# 不 source 宝塔环境,避免 PATH 被坏 Python 污染
|
||||
local -a candidates=()
|
||||
|
||||
candidates+=("$backend/.venv/bin/python")
|
||||
candidates+=("/usr/local/python3.11/bin/python3.11")
|
||||
candidates+=("/usr/local/python3.10/bin/python3.10")
|
||||
candidates+=("/usr/bin/python3.11")
|
||||
candidates+=("/usr/bin/python3.10")
|
||||
candidates+=("/usr/local/bin/python3.11")
|
||||
candidates+=("/usr/local/bin/python3.10")
|
||||
|
||||
# 动态扫描宝塔 Python(不写死版本号)
|
||||
for base in \
|
||||
/home/www/server/pyporject_evn/versions \
|
||||
/www/server/pyporject_evn/versions \
|
||||
/www/server/panel/pyenv/versions; do
|
||||
[ -d "$base" ] || continue
|
||||
for ver_dir in $(ls -d "$base"/* 2>/dev/null | sort -V -r); do
|
||||
for name in python3.11 python3.10 python3 python; do
|
||||
candidates+=("$ver_dir/bin/$name")
|
||||
done
|
||||
done
|
||||
done
|
||||
|
||||
candidates+=("$(command -v python3.11 2>/dev/null || true)")
|
||||
candidates+=("$(command -v python3.10 2>/dev/null || true)")
|
||||
candidates+=("$(command -v python3 2>/dev/null || true)")
|
||||
candidates+=("$(command -v python 2>/dev/null || true)")
|
||||
|
||||
declare -A seen=()
|
||||
for candidate in "${candidates[@]}"; do
|
||||
[ -n "$candidate" ] || continue
|
||||
[ -n "${seen[$candidate]:-}" ] && continue
|
||||
seen[$candidate]=1
|
||||
if _py_ok "$candidate"; then
|
||||
py="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$py" ]; then
|
||||
echo "[错误] 未找到带 SSL 的 Python 3.10+。" >&2
|
||||
echo " 宝塔当前 Python 缺 _ssl,请执行:" >&2
|
||||
echo " AUTO_BUILD_PYTHON=1 bash $root/install.sh" >&2
|
||||
echo " 然后: KEFU_PYTHON=/usr/local/python3.11/bin/python3.11 bash $backend/baota_fix.sh" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "$py"
|
||||
}
|
||||
|
||||
ensure_uvicorn() {
|
||||
local py="${1:?}"
|
||||
local backend="${2:?}"
|
||||
if "$py" -c "import uvicorn" 2>/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
if ! "$py" -c "import ssl" 2>/dev/null; then
|
||||
echo "[错误] $py 无 SSL,无法 pip install" >&2
|
||||
return 1
|
||||
fi
|
||||
echo "[修复] 正在安装 Python 依赖..." >&2
|
||||
"$py" -m pip install -U pip -q
|
||||
"$py" -m pip install -r "$backend/requirements.txt"
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
"""链接卡片落地页:带 SEO meta 与自动跳转。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from io import BytesIO
|
||||
from typing import Optional, TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import Request
|
||||
|
||||
_SLUG_RE = re.compile(r"^[a-zA-Z0-9_-]{6,64}$")
|
||||
|
||||
FAVICON_SIZE = 32
|
||||
COVER_SIZE = 256
|
||||
FAVICON_MIME = "image/png"
|
||||
|
||||
|
||||
def generate_slug() -> str:
|
||||
return secrets.token_urlsafe(9).replace("-", "_").replace(".", "_")[:12]
|
||||
|
||||
|
||||
def is_valid_slug(slug: str) -> bool:
|
||||
return bool(_SLUG_RE.match(slug or ""))
|
||||
|
||||
|
||||
def public_base_url(request: Optional["Request"] = None) -> str:
|
||||
env = os.getenv("KEFU_PUBLIC_BASE_URL", "").strip().rstrip("/")
|
||||
if env:
|
||||
return env
|
||||
if request is not None:
|
||||
return str(request.base_url).rstrip("/")
|
||||
return ""
|
||||
|
||||
|
||||
def absolute_media_url(path: str, request: Optional["Request"] = None) -> str:
|
||||
path = (path or "").strip()
|
||||
if not path:
|
||||
return ""
|
||||
if path.startswith("http://") or path.startswith("https://"):
|
||||
return path
|
||||
base = public_base_url(request)
|
||||
if not base:
|
||||
return path
|
||||
if not path.startswith("/"):
|
||||
path = f"/{path}"
|
||||
return f"{base}{path}"
|
||||
|
||||
|
||||
def favicon_mime(image_path: str) -> str:
|
||||
ext = (image_path or "").rsplit(".", 1)[-1].lower()
|
||||
return {
|
||||
"svg": "image/svg+xml",
|
||||
"png": "image/png",
|
||||
"gif": "image/gif",
|
||||
"webp": "image/webp",
|
||||
"jpg": "image/jpeg",
|
||||
"jpeg": "image/jpeg",
|
||||
"ico": "image/x-icon",
|
||||
}.get(ext, FAVICON_MIME)
|
||||
|
||||
|
||||
def favicon_path_for_cover(cover_path: str) -> str:
|
||||
"""由封面路径推导同名 favicon 文件路径。"""
|
||||
path = (cover_path or "").strip()
|
||||
if not path:
|
||||
return ""
|
||||
if path.endswith(".favicon.png"):
|
||||
return path
|
||||
if path.endswith(".png"):
|
||||
return f"{path[:-4]}.favicon.png"
|
||||
base, _ = os.path.splitext(path)
|
||||
return f"{base}.favicon.png"
|
||||
|
||||
|
||||
def media_path_to_disk(upload_dir: str, media_path: str) -> str:
|
||||
prefix = "/api/media/link-cards/"
|
||||
path = (media_path or "").strip()
|
||||
if not path.startswith(prefix):
|
||||
return ""
|
||||
rel = path[len(prefix) :].replace("/", os.sep)
|
||||
return os.path.join(upload_dir, rel)
|
||||
|
||||
|
||||
def process_card_upload(raw: bytes) -> tuple[bytes, bytes]:
|
||||
"""将任意位图转为封面 PNG (256x256) 与 favicon PNG (32x32)。"""
|
||||
if not raw:
|
||||
raise ValueError("图片为空")
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError as exc:
|
||||
raise RuntimeError("服务器未安装 Pillow,无法转换 favicon") from exc
|
||||
|
||||
try:
|
||||
with Image.open(BytesIO(raw)) as img:
|
||||
if getattr(img, "is_animated", False):
|
||||
img.seek(0)
|
||||
if img.mode not in ("RGB", "RGBA"):
|
||||
img = img.convert("RGBA")
|
||||
else:
|
||||
img = img.copy()
|
||||
|
||||
width, height = img.size
|
||||
if width < 1 or height < 1:
|
||||
raise ValueError("图片尺寸无效")
|
||||
|
||||
side = min(width, height)
|
||||
left = (width - side) // 2
|
||||
top = (height - side) // 2
|
||||
square = img.crop((left, top, left + side, top + side))
|
||||
|
||||
resample = Image.Resampling.LANCZOS
|
||||
cover = square.resize((COVER_SIZE, COVER_SIZE), resample)
|
||||
favicon = square.resize((FAVICON_SIZE, FAVICON_SIZE), resample)
|
||||
|
||||
cover_buf = BytesIO()
|
||||
cover.save(cover_buf, format="PNG", optimize=True)
|
||||
favicon_buf = BytesIO()
|
||||
favicon.save(favicon_buf, format="PNG", optimize=True)
|
||||
return cover_buf.getvalue(), favicon_buf.getvalue()
|
||||
except ValueError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ValueError("无法识别图片格式,请上传 JPG、PNG、GIF 或 WebP") from exc
|
||||
|
||||
|
||||
def normalize_target_url(url: str) -> str:
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return ""
|
||||
if not url.startswith("http://") and not url.startswith("https://"):
|
||||
url = f"https://{url}"
|
||||
return url
|
||||
|
||||
|
||||
def build_keywords(title: str, content: str) -> str:
|
||||
parts = [p.strip() for p in (title, content) if p and p.strip()]
|
||||
return ", ".join(dict.fromkeys(parts))
|
||||
|
||||
|
||||
def render_link_card_page(
|
||||
*,
|
||||
title: str,
|
||||
content: str,
|
||||
keywords: str,
|
||||
cover_url: str,
|
||||
favicon_url: str,
|
||||
target_url: str,
|
||||
) -> str:
|
||||
safe_title = html.escape(title or "跳转中")
|
||||
safe_desc = html.escape(content or title or "")
|
||||
safe_keywords = html.escape(keywords or build_keywords(title, content))
|
||||
safe_cover = html.escape(cover_url or "")
|
||||
safe_favicon = html.escape(favicon_url or cover_url or "")
|
||||
redirect_url = normalize_target_url(target_url)
|
||||
safe_target = html.escape(redirect_url)
|
||||
js_target = json.dumps(redirect_url, ensure_ascii=False)
|
||||
|
||||
favicon_tag = ""
|
||||
og_image_tag = ""
|
||||
if safe_favicon:
|
||||
favicon_tag = f' <link rel="icon" type="{FAVICON_MIME}" href="{safe_favicon}" />\n'
|
||||
if safe_cover:
|
||||
og_image_tag = f' <meta property="og:image" content="{safe_cover}" />\n'
|
||||
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>{safe_title}</title>
|
||||
<meta name="description" content="{safe_desc}" />
|
||||
<meta name="keywords" content="{safe_keywords}" />
|
||||
<meta property="og:title" content="{safe_title}" />
|
||||
<meta property="og:description" content="{safe_desc}" />
|
||||
{og_image_tag}{favicon_tag} <meta http-equiv="refresh" content="0;url={safe_target}" />
|
||||
<script>location.replace({js_target});</script>
|
||||
</head>
|
||||
<body>
|
||||
<p>正在跳转到目标页面…</p>
|
||||
<p><a href="{safe_target}">若未自动跳转,请点击这里</a></p>
|
||||
</body>
|
||||
</html>"""
|
||||
@@ -0,0 +1,210 @@
|
||||
"""链接卡片 API 与公开落地页。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
|
||||
from fastapi.responses import HTMLResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from auth.dependencies import get_current_user, require_write
|
||||
from link_cards import (
|
||||
absolute_media_url,
|
||||
build_keywords,
|
||||
favicon_path_for_cover,
|
||||
generate_slug,
|
||||
is_valid_slug,
|
||||
media_path_to_disk,
|
||||
normalize_target_url,
|
||||
process_card_upload,
|
||||
public_base_url,
|
||||
render_link_card_page,
|
||||
)
|
||||
from models.database import get_db
|
||||
from models.models import LinkCardPage, User
|
||||
|
||||
router = APIRouter(tags=["link-cards"])
|
||||
|
||||
UPLOAD_DIR = os.path.join(os.path.dirname(__file__), "uploads", "link-cards")
|
||||
os.makedirs(UPLOAD_DIR, exist_ok=True)
|
||||
|
||||
|
||||
class LinkCardUpsert(BaseModel):
|
||||
id: Optional[int] = None
|
||||
title: str = Field(..., min_length=1, max_length=200)
|
||||
content: str = Field(default="", max_length=2000)
|
||||
target_url: str = Field(..., min_length=1, max_length=2000)
|
||||
image_path: str = Field(..., min_length=1, max_length=512)
|
||||
|
||||
|
||||
class LinkCardResponse(BaseModel):
|
||||
id: int
|
||||
slug: str
|
||||
title: str
|
||||
content: str
|
||||
target_url: str
|
||||
image_path: str
|
||||
page_url: str
|
||||
cover_url: str
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
|
||||
class UploadImageResponse(BaseModel):
|
||||
image_path: str
|
||||
cover_url: str
|
||||
|
||||
|
||||
async def _get_owned_card(
|
||||
db: AsyncSession, user: User, card_id: int, *, write: bool = False
|
||||
) -> LinkCardPage:
|
||||
stmt = select(LinkCardPage).where(LinkCardPage.id == card_id)
|
||||
card = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if not card or card.owner_id != user.id:
|
||||
raise HTTPException(status_code=404, detail="卡片不存在")
|
||||
if write and user.role == "viewer":
|
||||
raise HTTPException(status_code=403, detail="无写入权限")
|
||||
return card
|
||||
|
||||
|
||||
def _card_response(card: LinkCardPage, request: Request) -> LinkCardResponse:
|
||||
base = public_base_url(request)
|
||||
page_url = f"{base}/p/{card.slug}" if base else f"/p/{card.slug}"
|
||||
cover_url = absolute_media_url(card.image_path, request)
|
||||
return LinkCardResponse(
|
||||
id=card.id,
|
||||
slug=card.slug,
|
||||
title=card.title,
|
||||
content=card.content or "",
|
||||
target_url=card.target_url,
|
||||
image_path=card.image_path,
|
||||
page_url=page_url,
|
||||
cover_url=cover_url,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/link-cards/upload-image", response_model=UploadImageResponse)
|
||||
async def upload_link_card_image(
|
||||
request: Request,
|
||||
file: UploadFile = File(...),
|
||||
user: User = Depends(require_write),
|
||||
):
|
||||
if not file.content_type or not file.content_type.startswith("image/"):
|
||||
raise HTTPException(status_code=400, detail="仅支持上传图片文件")
|
||||
if file.content_type == "image/svg+xml":
|
||||
raise HTTPException(status_code=400, detail="请上传 JPG/PNG 等位图,系统将自动转为 favicon PNG")
|
||||
|
||||
raw = await file.read()
|
||||
if not raw:
|
||||
raise HTTPException(status_code=400, detail="图片为空")
|
||||
if len(raw) > 4 * 1024 * 1024:
|
||||
raise HTTPException(status_code=400, detail="图片大小不能超过 4MB")
|
||||
|
||||
try:
|
||||
cover_bytes, favicon_bytes = process_card_upload(raw)
|
||||
except RuntimeError as exc:
|
||||
raise HTTPException(status_code=500, detail=str(exc)) from exc
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
owner_dir = os.path.join(UPLOAD_DIR, str(user.id))
|
||||
os.makedirs(owner_dir, exist_ok=True)
|
||||
file_id = uuid.uuid4().hex
|
||||
cover_filename = f"{file_id}.png"
|
||||
favicon_filename = f"{file_id}.favicon.png"
|
||||
cover_path = os.path.join(owner_dir, cover_filename)
|
||||
favicon_path = os.path.join(owner_dir, favicon_filename)
|
||||
with open(cover_path, "wb") as f:
|
||||
f.write(cover_bytes)
|
||||
with open(favicon_path, "wb") as f:
|
||||
f.write(favicon_bytes)
|
||||
|
||||
image_path = f"/api/media/link-cards/{user.id}/{cover_filename}"
|
||||
return UploadImageResponse(
|
||||
image_path=image_path,
|
||||
cover_url=absolute_media_url(image_path, request),
|
||||
)
|
||||
|
||||
|
||||
@router.post("/api/link-cards", response_model=LinkCardResponse)
|
||||
async def upsert_link_card(
|
||||
body: LinkCardUpsert,
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: User = Depends(require_write),
|
||||
):
|
||||
title = body.title.strip()
|
||||
content = (body.content or "").strip()
|
||||
target_url = normalize_target_url(body.target_url)
|
||||
image_path = (body.image_path or "").strip()
|
||||
|
||||
if not title:
|
||||
raise HTTPException(status_code=400, detail="请填写卡片标题")
|
||||
if not target_url:
|
||||
raise HTTPException(status_code=400, detail="请填写跳转链接")
|
||||
if not image_path.startswith("/api/media/link-cards/"):
|
||||
raise HTTPException(status_code=400, detail="请先上传卡片封面图")
|
||||
|
||||
if body.id:
|
||||
card = await _get_owned_card(db, user, body.id, write=True)
|
||||
card.title = title
|
||||
card.content = content
|
||||
card.target_url = target_url
|
||||
card.image_path = image_path
|
||||
card.updated_at = datetime.utcnow()
|
||||
else:
|
||||
slug = generate_slug()
|
||||
for _ in range(5):
|
||||
exists = (
|
||||
await db.execute(select(LinkCardPage.id).where(LinkCardPage.slug == slug))
|
||||
).scalar_one_or_none()
|
||||
if not exists:
|
||||
break
|
||||
slug = generate_slug()
|
||||
else:
|
||||
raise HTTPException(status_code=500, detail="生成页面标识失败,请重试")
|
||||
|
||||
card = LinkCardPage(
|
||||
owner_id=user.id,
|
||||
slug=slug,
|
||||
title=title,
|
||||
content=content,
|
||||
target_url=target_url,
|
||||
image_path=image_path,
|
||||
)
|
||||
db.add(card)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(card)
|
||||
return _card_response(card, request)
|
||||
|
||||
|
||||
@router.get("/p/{slug}", response_class=HTMLResponse, include_in_schema=False)
|
||||
async def serve_link_card_page(slug: str, request: Request, db: AsyncSession = Depends(get_db)):
|
||||
if not is_valid_slug(slug):
|
||||
raise HTTPException(status_code=404, detail="页面不存在")
|
||||
|
||||
stmt = select(LinkCardPage).where(LinkCardPage.slug == slug)
|
||||
card = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if not card:
|
||||
raise HTTPException(status_code=404, detail="页面不存在")
|
||||
|
||||
cover_url = absolute_media_url(card.image_path, request)
|
||||
favicon_path = favicon_path_for_cover(card.image_path)
|
||||
favicon_disk = media_path_to_disk(UPLOAD_DIR, favicon_path)
|
||||
favicon_url = absolute_media_url(favicon_path, request) if favicon_disk and os.path.isfile(favicon_disk) else cover_url
|
||||
html = render_link_card_page(
|
||||
title=card.title,
|
||||
content=card.content or "",
|
||||
keywords=build_keywords(card.title, card.content or ""),
|
||||
cover_url=cover_url,
|
||||
favicon_url=favicon_url,
|
||||
target_url=card.target_url,
|
||||
)
|
||||
return HTMLResponse(content=html, media_type="text/html; charset=utf-8")
|
||||
+2425
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,28 @@
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import declarative_base, sessionmaker
|
||||
|
||||
from models.db_config import (
|
||||
build_database_url,
|
||||
create_database_engine,
|
||||
read_database_config,
|
||||
)
|
||||
|
||||
_config = read_database_config()
|
||||
DATABASE_URL = build_database_url(_config)
|
||||
engine = create_database_engine(_config)
|
||||
AsyncSessionLocal = sessionmaker(
|
||||
engine, class_=AsyncSession, expire_on_commit=False
|
||||
)
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
async def get_db():
|
||||
async with AsyncSessionLocal() as session:
|
||||
try:
|
||||
yield session
|
||||
await session.commit()
|
||||
except Exception:
|
||||
await session.rollback()
|
||||
raise
|
||||
finally:
|
||||
await session.close()
|
||||
@@ -0,0 +1,252 @@
|
||||
"""数据库连接配置:支持 SQLite / MySQL / PostgreSQL。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parent.parent
|
||||
PROJECT_ROOT = BACKEND_DIR.parent
|
||||
ENV_FILE = PROJECT_ROOT / ".env"
|
||||
|
||||
load_dotenv(ENV_FILE)
|
||||
|
||||
SUPPORTED_DB_TYPES = ("sqlite", "mysql", "postgresql")
|
||||
PASSWORD_PLACEHOLDER = "******"
|
||||
DEFAULT_SQLITE_PATH = BACKEND_DIR / "kefu.db"
|
||||
|
||||
|
||||
def _env(name: str, default: str = "") -> str:
|
||||
return (os.getenv(name) or default).strip()
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
raw = _env(name)
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return int(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
def normalize_db_type(value: str | None) -> str:
|
||||
raw = (value or "sqlite").strip().lower()
|
||||
if raw in ("postgres", "pgsql"):
|
||||
return "postgresql"
|
||||
if raw in SUPPORTED_DB_TYPES:
|
||||
return raw
|
||||
return "sqlite"
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatabaseConfig:
|
||||
db_type: str = "sqlite"
|
||||
db_host: str = "127.0.0.1"
|
||||
db_port: int = 0
|
||||
db_user: str = ""
|
||||
db_password: str = ""
|
||||
db_name: str = "kefu"
|
||||
db_path: str = ""
|
||||
database_url: str = ""
|
||||
|
||||
def normalized_type(self) -> str:
|
||||
return normalize_db_type(self.db_type)
|
||||
|
||||
def resolved_port(self) -> int:
|
||||
if self.db_port:
|
||||
return self.db_port
|
||||
if self.normalized_type() == "mysql":
|
||||
return 3306
|
||||
if self.normalized_type() == "postgresql":
|
||||
return 5432
|
||||
return 0
|
||||
|
||||
def sqlite_path(self) -> Path:
|
||||
if self.db_path:
|
||||
path = Path(self.db_path)
|
||||
if not path.is_absolute():
|
||||
path = BACKEND_DIR / path
|
||||
return path
|
||||
if self.database_url and self.database_url.startswith("sqlite"):
|
||||
# sqlite+aiosqlite:///path
|
||||
raw = self.database_url.split("///", 1)[-1]
|
||||
return Path(raw)
|
||||
return DEFAULT_SQLITE_PATH
|
||||
|
||||
|
||||
def read_database_config() -> DatabaseConfig:
|
||||
explicit_url = _env("KEFU_DATABASE_URL")
|
||||
db_type = normalize_db_type(_env("KEFU_DB_TYPE", "sqlite"))
|
||||
db_path = _env("KEFU_DB_PATH")
|
||||
if explicit_url and not _env("KEFU_DB_TYPE"):
|
||||
lowered = explicit_url.lower()
|
||||
if lowered.startswith("mysql"):
|
||||
db_type = "mysql"
|
||||
elif lowered.startswith("postgresql") or lowered.startswith("postgres"):
|
||||
db_type = "postgresql"
|
||||
elif lowered.startswith("sqlite"):
|
||||
db_type = "sqlite"
|
||||
return DatabaseConfig(
|
||||
db_type=db_type,
|
||||
db_host=_env("KEFU_DB_HOST", "127.0.0.1"),
|
||||
db_port=_env_int("KEFU_DB_PORT", 0),
|
||||
db_user=_env("KEFU_DB_USER"),
|
||||
db_password=_env("KEFU_DB_PASSWORD"),
|
||||
db_name=_env("KEFU_DB_NAME", "kefu"),
|
||||
db_path=db_path,
|
||||
database_url=explicit_url,
|
||||
)
|
||||
|
||||
|
||||
def build_database_url(config: DatabaseConfig | None = None) -> str:
|
||||
cfg = config or read_database_config()
|
||||
if cfg.database_url:
|
||||
return cfg.database_url
|
||||
|
||||
db_type = cfg.normalized_type()
|
||||
if db_type == "sqlite":
|
||||
path = cfg.sqlite_path()
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
return f"sqlite+aiosqlite:///{path.as_posix()}"
|
||||
|
||||
user = quote_plus(cfg.db_user or "")
|
||||
password = quote_plus(cfg.db_password or "")
|
||||
host = cfg.db_host or "127.0.0.1"
|
||||
port = cfg.resolved_port()
|
||||
db_name = cfg.db_name or "kefu"
|
||||
|
||||
if db_type == "mysql":
|
||||
auth = f"{user}:{password}@" if user else ""
|
||||
return (
|
||||
f"mysql+asyncmy://{auth}{host}:{port}/{db_name}"
|
||||
"?charset=utf8mb4"
|
||||
)
|
||||
|
||||
auth = f"{user}:{password}@" if user else ""
|
||||
return f"postgresql+asyncpg://{auth}{host}:{port}/{db_name}"
|
||||
|
||||
|
||||
def engine_kwargs_for_url(url: str) -> dict[str, Any]:
|
||||
kwargs: dict[str, Any] = {"echo": False}
|
||||
if not url.startswith("sqlite"):
|
||||
kwargs["pool_pre_ping"] = True
|
||||
kwargs["pool_recycle"] = 3600
|
||||
# 默认连接池仅 pool_size=5 + max_overflow=10。多账号托管 + 前端并发请求时
|
||||
# 容易耗尽连接导致请求阻塞/失败,这里放大连接池(可用环境变量覆盖)。
|
||||
try:
|
||||
_pool = int(os.getenv("KEFU_DB_POOL_SIZE", "20") or 20)
|
||||
except ValueError:
|
||||
_pool = 20
|
||||
try:
|
||||
_overflow = int(os.getenv("KEFU_DB_MAX_OVERFLOW", "40") or 40)
|
||||
except ValueError:
|
||||
_overflow = 40
|
||||
kwargs["pool_size"] = max(5, _pool)
|
||||
kwargs["max_overflow"] = max(0, _overflow)
|
||||
kwargs["pool_timeout"] = 30
|
||||
return kwargs
|
||||
|
||||
|
||||
def create_database_engine(config: DatabaseConfig | None = None) -> AsyncEngine:
|
||||
url = build_database_url(config)
|
||||
return create_async_engine(url, **engine_kwargs_for_url(url))
|
||||
|
||||
|
||||
def mask_database_url(url: str) -> str:
|
||||
if "@" not in url or "://" not in url:
|
||||
return url
|
||||
scheme, rest = url.split("://", 1)
|
||||
if "@" not in rest:
|
||||
return url
|
||||
creds, host_part = rest.rsplit("@", 1)
|
||||
if ":" in creds:
|
||||
user = creds.split(":", 1)[0]
|
||||
return f"{scheme}://{user}:{PASSWORD_PLACEHOLDER}@{host_part}"
|
||||
return f"{scheme}://{PASSWORD_PLACEHOLDER}@{host_part}"
|
||||
|
||||
|
||||
def database_config_to_response(cfg: DatabaseConfig | None = None) -> dict[str, Any]:
|
||||
cfg = cfg or read_database_config()
|
||||
url = build_database_url(cfg)
|
||||
return {
|
||||
"db_type": cfg.normalized_type(),
|
||||
"db_host": cfg.db_host,
|
||||
"db_port": cfg.resolved_port(),
|
||||
"db_user": cfg.db_user,
|
||||
"db_name": cfg.db_name,
|
||||
"db_path": str(cfg.sqlite_path()) if cfg.normalized_type() == "sqlite" else "",
|
||||
"db_password": PASSWORD_PLACEHOLDER if cfg.db_password else "",
|
||||
"db_password_configured": bool(cfg.db_password),
|
||||
"database_url_display": mask_database_url(url),
|
||||
"supported_types": list(SUPPORTED_DB_TYPES),
|
||||
}
|
||||
|
||||
|
||||
def persist_database_config(payload: dict[str, Any]) -> None:
|
||||
"""将数据库配置写入项目根 .env 文件。"""
|
||||
from dotenv import set_key
|
||||
|
||||
if not ENV_FILE.exists():
|
||||
ENV_FILE.write_text("", encoding="utf-8")
|
||||
|
||||
db_type = normalize_db_type(payload.get("db_type"))
|
||||
env_map = {
|
||||
"KEFU_DB_TYPE": db_type,
|
||||
"KEFU_DB_HOST": (payload.get("db_host") or "127.0.0.1").strip(),
|
||||
"KEFU_DB_PORT": str(payload.get("db_port") or ""),
|
||||
"KEFU_DB_USER": (payload.get("db_user") or "").strip(),
|
||||
"KEFU_DB_NAME": (payload.get("db_name") or "kefu").strip(),
|
||||
"KEFU_DB_PATH": (payload.get("db_path") or "").strip(),
|
||||
}
|
||||
|
||||
password = payload.get("db_password")
|
||||
if password and str(password).strip() not in ("", PASSWORD_PLACEHOLDER):
|
||||
env_map["KEFU_DB_PASSWORD"] = str(password).strip()
|
||||
|
||||
# 清除完整 URL,避免与分项配置冲突
|
||||
set_key(str(ENV_FILE), "KEFU_DATABASE_URL", "")
|
||||
|
||||
for key, value in env_map.items():
|
||||
set_key(str(ENV_FILE), key, value or "")
|
||||
|
||||
# 让当前进程也能读到新值(重启后才会重建 engine)
|
||||
for key, value in env_map.items():
|
||||
os.environ[key] = value or ""
|
||||
os.environ.pop("KEFU_DATABASE_URL", None)
|
||||
|
||||
|
||||
async def test_database_connection(payload: dict[str, Any]) -> None:
|
||||
cfg = DatabaseConfig(
|
||||
db_type=normalize_db_type(payload.get("db_type")),
|
||||
db_host=(payload.get("db_host") or "127.0.0.1").strip(),
|
||||
db_port=int(payload.get("db_port") or 0),
|
||||
db_user=(payload.get("db_user") or "").strip(),
|
||||
db_password=(payload.get("db_password") or "").strip(),
|
||||
db_name=(payload.get("db_name") or "kefu").strip(),
|
||||
db_path=(payload.get("db_path") or "").strip(),
|
||||
)
|
||||
|
||||
current = read_database_config()
|
||||
if cfg.db_password in ("", PASSWORD_PLACEHOLDER):
|
||||
cfg.db_password = current.db_password
|
||||
|
||||
url = build_database_url(cfg)
|
||||
engine = create_async_engine(url, **engine_kwargs_for_url(url))
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
await conn.execute(text("SELECT 1"))
|
||||
except ModuleNotFoundError as exc:
|
||||
driver = "asyncmy" if cfg.normalized_type() == "mysql" else "asyncpg"
|
||||
raise RuntimeError(
|
||||
f"缺少数据库驱动,请执行: pip install {driver}"
|
||||
) from exc
|
||||
finally:
|
||||
await engine.dispose()
|
||||
@@ -0,0 +1,223 @@
|
||||
"""跨数据库兼容的轻量 schema 迁移。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from sqlalchemy import inspect, text
|
||||
|
||||
|
||||
def _table_columns(conn, table: str) -> set[str]:
|
||||
try:
|
||||
insp = inspect(conn)
|
||||
return {col["name"] for col in insp.get_columns(table)}
|
||||
except Exception:
|
||||
return set()
|
||||
|
||||
|
||||
def _dialect(conn) -> str:
|
||||
return conn.dialect.name
|
||||
|
||||
|
||||
def _bool_default(conn, value: bool = True) -> str:
|
||||
if _dialect(conn) == "postgresql":
|
||||
return "TRUE" if value else "FALSE"
|
||||
return "1" if value else "0"
|
||||
|
||||
|
||||
def add_column_if_missing(conn, table: str, column: str, ddl_by_dialect: dict[str, str]) -> None:
|
||||
cols = _table_columns(conn, table)
|
||||
if not cols or column in cols:
|
||||
return
|
||||
dialect = _dialect(conn)
|
||||
ddl = ddl_by_dialect.get(dialect) or ddl_by_dialect.get("default")
|
||||
if ddl:
|
||||
conn.execute(text(ddl))
|
||||
|
||||
|
||||
def migrate_accounts_table(conn) -> None:
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"cookie_data",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN cookie_data TEXT"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"cookie_updated_at",
|
||||
{
|
||||
"default": "ALTER TABLE accounts ADD COLUMN cookie_updated_at DATETIME",
|
||||
"postgresql": "ALTER TABLE accounts ADD COLUMN cookie_updated_at TIMESTAMP",
|
||||
},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"im_session_data",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN im_session_data TEXT"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"reply_delay_seconds",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN reply_delay_seconds INTEGER DEFAULT 0"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"reply_cooldown_seconds",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN reply_cooldown_seconds INTEGER"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"follow_welcome_enabled",
|
||||
{
|
||||
"default": "ALTER TABLE accounts ADD COLUMN follow_welcome_enabled BOOLEAN DEFAULT 0",
|
||||
"postgresql": "ALTER TABLE accounts ADD COLUMN follow_welcome_enabled BOOLEAN DEFAULT FALSE",
|
||||
},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"follow_welcome_content",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN follow_welcome_content TEXT"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"owner_id",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN owner_id INTEGER"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"user_agent",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN user_agent TEXT"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"avatar_url",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN avatar_url TEXT"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"douyin_uid",
|
||||
{"default": "ALTER TABLE accounts ADD COLUMN douyin_uid VARCHAR(64)"},
|
||||
)
|
||||
|
||||
|
||||
def migrate_account_videos_table(conn) -> None:
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"account_videos",
|
||||
"media_type",
|
||||
{"default": "ALTER TABLE account_videos ADD COLUMN media_type VARCHAR(20)"},
|
||||
)
|
||||
|
||||
|
||||
def migrate_message_logs_table(conn) -> None:
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"message_logs",
|
||||
"sender_avatar",
|
||||
{"default": "ALTER TABLE message_logs ADD COLUMN sender_avatar TEXT"},
|
||||
)
|
||||
|
||||
|
||||
def migrate_rules_table(conn) -> None:
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"rules",
|
||||
"owner_id",
|
||||
{"default": "ALTER TABLE rules ADD COLUMN owner_id INTEGER"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"rules",
|
||||
"sort_order",
|
||||
{"default": "ALTER TABLE rules ADD COLUMN sort_order INTEGER DEFAULT 0"},
|
||||
)
|
||||
cols = _table_columns(conn, "rules")
|
||||
if cols and "sort_order" in cols:
|
||||
conn.execute(
|
||||
text("UPDATE rules SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0")
|
||||
)
|
||||
|
||||
|
||||
def migrate_users_table(conn) -> None:
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"users",
|
||||
"email",
|
||||
{"default": "ALTER TABLE users ADD COLUMN email VARCHAR(255)"},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"users",
|
||||
"email_verified",
|
||||
{
|
||||
"default": "ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT 0",
|
||||
"postgresql": "ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT FALSE",
|
||||
},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"users",
|
||||
"email_verified_at",
|
||||
{
|
||||
"default": "ALTER TABLE users ADD COLUMN email_verified_at DATETIME",
|
||||
"postgresql": "ALTER TABLE users ADD COLUMN email_verified_at TIMESTAMP",
|
||||
},
|
||||
)
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"users",
|
||||
"max_accounts",
|
||||
{"default": "ALTER TABLE users ADD COLUMN max_accounts INTEGER DEFAULT 3"},
|
||||
)
|
||||
cols = _table_columns(conn, "users")
|
||||
if not cols:
|
||||
return
|
||||
verified = _bool_default(conn, True)
|
||||
conn.execute(text(f"UPDATE users SET email_verified = {verified} WHERE email_verified IS NULL"))
|
||||
conn.execute(text("UPDATE users SET max_accounts = -1 WHERE role = 'admin'"))
|
||||
conn.execute(text("UPDATE users SET max_accounts = 3 WHERE max_accounts IS NULL"))
|
||||
|
||||
|
||||
def migrate_payment_orders_table(conn) -> None:
|
||||
cols = _table_columns(conn, "payment_orders")
|
||||
if not cols:
|
||||
return
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"payment_orders",
|
||||
"slots_applied",
|
||||
{
|
||||
"default": "ALTER TABLE payment_orders ADD COLUMN slots_applied BOOLEAN DEFAULT 0",
|
||||
"postgresql": "ALTER TABLE payment_orders ADD COLUMN slots_applied BOOLEAN DEFAULT FALSE",
|
||||
},
|
||||
)
|
||||
applied = _bool_default(conn, True)
|
||||
conn.execute(
|
||||
text(
|
||||
"UPDATE payment_orders SET slots_applied = "
|
||||
f"{applied} WHERE status = 'paid' AND (slots_applied IS NULL OR slots_applied = 0)"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def migrate_accounts_quota_disabled(conn) -> None:
|
||||
cols = _table_columns(conn, "accounts")
|
||||
if not cols:
|
||||
return
|
||||
add_column_if_missing(
|
||||
conn,
|
||||
"accounts",
|
||||
"quota_disabled",
|
||||
{
|
||||
"default": "ALTER TABLE accounts ADD COLUMN quota_disabled BOOLEAN DEFAULT 0",
|
||||
"postgresql": "ALTER TABLE accounts ADD COLUMN quota_disabled BOOLEAN DEFAULT FALSE",
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,323 @@
|
||||
"""从 SQLite 源库迁移数据到目标数据库。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import MetaData, insert, inspect, text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine
|
||||
|
||||
from models.database import Base
|
||||
import models.models # noqa: F401 — 注册 ORM 表结构
|
||||
from models.db_config import (
|
||||
PASSWORD_PLACEHOLDER,
|
||||
DatabaseConfig,
|
||||
build_database_url,
|
||||
create_database_engine,
|
||||
mask_database_url,
|
||||
read_database_config,
|
||||
)
|
||||
from models.db_migrate import (
|
||||
migrate_accounts_quota_disabled,
|
||||
migrate_accounts_table,
|
||||
migrate_message_logs_table,
|
||||
migrate_payment_orders_table,
|
||||
migrate_rules_table,
|
||||
migrate_users_table,
|
||||
)
|
||||
|
||||
# 按外键依赖顺序导入
|
||||
MIGRATION_TABLES: tuple[str, ...] = (
|
||||
"users",
|
||||
"email_verification_tokens",
|
||||
"password_reset_tokens",
|
||||
"app_config",
|
||||
"accounts",
|
||||
"rules",
|
||||
"message_logs",
|
||||
"received_message_logs",
|
||||
"system_logs",
|
||||
"payment_orders",
|
||||
)
|
||||
|
||||
BATCH_SIZE = 400
|
||||
|
||||
|
||||
def _parse_datetime(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (datetime, date)):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
text_value = value.strip()
|
||||
if not text_value:
|
||||
return None
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(text_value, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
try:
|
||||
return datetime.fromisoformat(text_value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return value
|
||||
return value
|
||||
|
||||
|
||||
def _normalize_rows(rows: list[dict[str, Any]], tbl) -> list[dict[str, Any]]:
|
||||
if not rows:
|
||||
return rows
|
||||
col_types = {column.name: column.type for column in tbl.columns}
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
for key, col_type in col_types.items():
|
||||
if key not in item or item[key] is None:
|
||||
continue
|
||||
type_name = col_type.__class__.__name__.lower()
|
||||
if "datetime" in type_name or "timestamp" in type_name:
|
||||
item[key] = _parse_datetime(item[key])
|
||||
elif "boolean" in type_name and not isinstance(item[key], bool):
|
||||
if isinstance(item[key], (int, float)):
|
||||
item[key] = bool(item[key])
|
||||
elif isinstance(item[key], str):
|
||||
item[key] = item[key].strip().lower() in ("1", "true", "t", "yes")
|
||||
normalized.append(item)
|
||||
return normalized
|
||||
|
||||
|
||||
def resolve_sqlite_source_path(source_path: str | None = None) -> Path:
|
||||
if source_path and str(source_path).strip():
|
||||
path = Path(source_path.strip())
|
||||
if not path.is_absolute():
|
||||
from models.db_config import BACKEND_DIR
|
||||
|
||||
path = BACKEND_DIR / path
|
||||
return path
|
||||
from models.db_config import DEFAULT_SQLITE_PATH
|
||||
|
||||
return DEFAULT_SQLITE_PATH
|
||||
|
||||
|
||||
def build_sqlite_config(path: Path) -> DatabaseConfig:
|
||||
return DatabaseConfig(db_type="sqlite", db_path=str(path))
|
||||
|
||||
|
||||
def _same_sqlite_target(source: Path, target: DatabaseConfig) -> bool:
|
||||
if target.normalized_type() != "sqlite":
|
||||
return False
|
||||
try:
|
||||
return source.resolve() == target.sqlite_path().resolve()
|
||||
except OSError:
|
||||
return str(source) == str(target.sqlite_path())
|
||||
|
||||
|
||||
async def inspect_sqlite_source(source_path: str | None = None) -> dict[str, Any]:
|
||||
path = resolve_sqlite_source_path(source_path)
|
||||
if not path.exists():
|
||||
return {
|
||||
"source_path": str(path),
|
||||
"exists": False,
|
||||
"tables": {},
|
||||
"total_rows": 0,
|
||||
}
|
||||
|
||||
cfg = build_sqlite_config(path)
|
||||
engine = create_database_engine(cfg)
|
||||
tables: dict[str, int] = {}
|
||||
total_rows = 0
|
||||
try:
|
||||
async with engine.connect() as conn:
|
||||
for table in MIGRATION_TABLES:
|
||||
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
|
||||
tables[table] = 0
|
||||
continue
|
||||
count = await conn.scalar(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
row_count = int(count or 0)
|
||||
tables[table] = row_count
|
||||
total_rows += row_count
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
return {
|
||||
"source_path": str(path),
|
||||
"exists": True,
|
||||
"tables": tables,
|
||||
"total_rows": total_rows,
|
||||
}
|
||||
|
||||
|
||||
async def _ensure_target_schema(engine: AsyncEngine) -> None:
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
await conn.run_sync(migrate_accounts_table)
|
||||
await conn.run_sync(migrate_rules_table)
|
||||
await conn.run_sync(migrate_message_logs_table)
|
||||
await conn.run_sync(migrate_users_table)
|
||||
await conn.run_sync(migrate_payment_orders_table)
|
||||
await conn.run_sync(migrate_accounts_quota_disabled)
|
||||
|
||||
|
||||
async def _count_target_rows(engine: AsyncEngine) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
async with engine.connect() as conn:
|
||||
for table in MIGRATION_TABLES:
|
||||
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
|
||||
counts[table] = 0
|
||||
continue
|
||||
count = await conn.scalar(text(f"SELECT COUNT(*) FROM {table}"))
|
||||
counts[table] = int(count or 0)
|
||||
return counts
|
||||
|
||||
|
||||
async def _fetch_table_rows(engine: AsyncEngine, table: str) -> list[dict[str, Any]]:
|
||||
async with engine.connect() as conn:
|
||||
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
|
||||
return []
|
||||
result = await conn.execute(text(f"SELECT * FROM {table} ORDER BY id"))
|
||||
return [dict(row) for row in result.mappings()]
|
||||
|
||||
|
||||
async def _clear_target_tables(engine: AsyncEngine) -> None:
|
||||
async with engine.begin() as conn:
|
||||
dialect = conn.dialect.name
|
||||
if dialect == "mysql":
|
||||
await conn.execute(text("SET FOREIGN_KEY_CHECKS=0"))
|
||||
for table in reversed(MIGRATION_TABLES):
|
||||
if await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
|
||||
await conn.execute(text(f"DELETE FROM {table}"))
|
||||
await conn.execute(text("SET FOREIGN_KEY_CHECKS=1"))
|
||||
elif dialect == "postgresql":
|
||||
existing = []
|
||||
for table in MIGRATION_TABLES:
|
||||
if await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
|
||||
existing.append(table)
|
||||
if existing:
|
||||
await conn.execute(
|
||||
text(f"TRUNCATE {', '.join(existing)} RESTART IDENTITY CASCADE")
|
||||
)
|
||||
else:
|
||||
await conn.execute(text("PRAGMA foreign_keys=OFF"))
|
||||
for table in reversed(MIGRATION_TABLES):
|
||||
if await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
|
||||
await conn.execute(text(f"DELETE FROM {table}"))
|
||||
await conn.execute(text("PRAGMA foreign_keys=ON"))
|
||||
|
||||
|
||||
async def _insert_rows(engine: AsyncEngine, table: str, rows: list[dict[str, Any]]) -> int:
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
tbl = Base.metadata.tables.get(table)
|
||||
if tbl is None:
|
||||
metadata = MetaData()
|
||||
|
||||
def _reflect(sync_conn) -> None:
|
||||
metadata.reflect(sync_conn, only=[table])
|
||||
|
||||
async with engine.connect() as conn:
|
||||
await conn.run_sync(_reflect)
|
||||
tbl = metadata.tables[table]
|
||||
|
||||
inserted = 0
|
||||
prepared_rows = _normalize_rows(rows, tbl)
|
||||
async with engine.begin() as conn:
|
||||
for offset in range(0, len(prepared_rows), BATCH_SIZE):
|
||||
batch = prepared_rows[offset : offset + BATCH_SIZE]
|
||||
await conn.execute(insert(tbl), batch)
|
||||
inserted += len(batch)
|
||||
return inserted
|
||||
|
||||
|
||||
async def _reset_auto_increment(engine: AsyncEngine) -> None:
|
||||
async with engine.begin() as conn:
|
||||
dialect = conn.dialect.name
|
||||
for table in MIGRATION_TABLES:
|
||||
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
|
||||
continue
|
||||
max_id = await conn.scalar(text(f"SELECT COALESCE(MAX(id), 0) FROM {table}"))
|
||||
if not max_id:
|
||||
continue
|
||||
next_id = int(max_id) + 1
|
||||
if dialect == "mysql":
|
||||
await conn.execute(text(f"ALTER TABLE {table} AUTO_INCREMENT = {next_id}"))
|
||||
elif dialect == "postgresql":
|
||||
await conn.execute(
|
||||
text(
|
||||
"SELECT setval("
|
||||
f"pg_get_serial_sequence('{table}', 'id'), "
|
||||
f"{next_id}, false)"
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _build_target_config(payload: dict[str, Any]) -> DatabaseConfig:
|
||||
current = read_database_config()
|
||||
password = (payload.get("db_password") or "").strip()
|
||||
if password in ("", PASSWORD_PLACEHOLDER):
|
||||
password = current.db_password
|
||||
return DatabaseConfig(
|
||||
db_type=payload.get("db_type") or current.db_type,
|
||||
db_host=(payload.get("db_host") or current.db_host or "127.0.0.1").strip(),
|
||||
db_port=int(payload.get("db_port") or current.resolved_port() or 0),
|
||||
db_user=(payload.get("db_user") or current.db_user).strip(),
|
||||
db_password=password,
|
||||
db_name=(payload.get("db_name") or current.db_name or "kefu").strip(),
|
||||
db_path=(payload.get("db_path") or current.db_path).strip(),
|
||||
)
|
||||
|
||||
|
||||
async def migrate_sqlite_to_target(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
clear_target: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
source_path = resolve_sqlite_source_path(payload.get("source_db_path"))
|
||||
if not source_path.exists():
|
||||
raise FileNotFoundError(f"SQLite 源文件不存在: {source_path}")
|
||||
|
||||
target_cfg = _build_target_config(payload)
|
||||
if _same_sqlite_target(source_path, target_cfg):
|
||||
raise ValueError("目标库不能与 SQLite 源文件相同,请指定不同的目标数据库")
|
||||
|
||||
source_cfg = build_sqlite_config(source_path)
|
||||
source_engine = create_database_engine(source_cfg)
|
||||
target_engine = create_database_engine(target_cfg)
|
||||
|
||||
try:
|
||||
preview = await inspect_sqlite_source(str(source_path))
|
||||
if preview["total_rows"] == 0:
|
||||
raise ValueError("SQLite 源库中没有可迁移的数据")
|
||||
|
||||
await _ensure_target_schema(target_engine)
|
||||
target_counts = await _count_target_rows(target_engine)
|
||||
target_total = sum(target_counts.values())
|
||||
if target_total > 0 and not clear_target:
|
||||
raise ValueError(
|
||||
f"目标库已有 {target_total} 条数据,请勾选「清空目标库后再导入」或先手动清空"
|
||||
)
|
||||
|
||||
if clear_target and target_total > 0:
|
||||
await _clear_target_tables(target_engine)
|
||||
|
||||
copied: dict[str, int] = {}
|
||||
total_rows = 0
|
||||
for table in MIGRATION_TABLES:
|
||||
rows = await _fetch_table_rows(source_engine, table)
|
||||
copied[table] = await _insert_rows(target_engine, table, rows)
|
||||
total_rows += copied[table]
|
||||
|
||||
await _reset_auto_increment(target_engine)
|
||||
finally:
|
||||
await source_engine.dispose()
|
||||
await target_engine.dispose()
|
||||
|
||||
target_type = target_cfg.normalized_type()
|
||||
return {
|
||||
"message": f"已从 SQLite 迁移 {total_rows} 条记录到 {target_type} 数据库",
|
||||
"source_path": str(source_path),
|
||||
"target_type": target_type,
|
||||
"target_url": mask_database_url(build_database_url(target_cfg)),
|
||||
"tables": copied,
|
||||
"total_rows": total_rows,
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
from datetime import datetime
|
||||
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import relationship
|
||||
from .database import Base
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
username = Column(String(50), unique=True, index=True, nullable=False)
|
||||
email = Column(String(255), unique=True, index=True, nullable=True)
|
||||
password_hash = Column(String(255), nullable=False)
|
||||
display_name = Column(String(100), nullable=True)
|
||||
role = Column(String(20), default="operator", index=True) # admin, operator, viewer
|
||||
is_active = Column(Boolean, default=True)
|
||||
email_verified = Column(Boolean, default=False)
|
||||
email_verified_at = Column(DateTime, nullable=True)
|
||||
max_accounts = Column(Integer, default=3)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
accounts = relationship("Account", back_populates="owner")
|
||||
verification_tokens = relationship(
|
||||
"EmailVerificationToken",
|
||||
back_populates="user",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
password_reset_tokens = relationship(
|
||||
"PasswordResetToken",
|
||||
back_populates="user",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
|
||||
|
||||
class EmailVerificationToken(Base):
|
||||
__tablename__ = "email_verification_tokens"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
token = Column(String(128), unique=True, index=True, nullable=False)
|
||||
expires_at = Column(DateTime, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
user = relationship("User", back_populates="verification_tokens")
|
||||
|
||||
|
||||
class PasswordResetToken(Base):
|
||||
__tablename__ = "password_reset_tokens"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
token = Column(String(128), unique=True, index=True, nullable=False)
|
||||
expires_at = Column(DateTime, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
user = relationship("User", back_populates="password_reset_tokens")
|
||||
|
||||
|
||||
class AppConfig(Base):
|
||||
"""系统功能配置(单例行 id=1,JSON 存储)。"""
|
||||
|
||||
__tablename__ = "app_config"
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
data = Column(Text, nullable=False, default="{}")
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class Account(Base):
|
||||
__tablename__ = "accounts"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
owner_id = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
username = Column(String(100), unique=True, index=True, nullable=True) # 登录后获取的用户名/ID
|
||||
avatar_url = Column(String(512), nullable=True) # 登录后获取的头像 URL
|
||||
douyin_uid = Column(String(64), nullable=True, index=True) # 抖音 UID
|
||||
phone = Column(String(20), nullable=True) # 绑定的手机号(可选)
|
||||
status = Column(String(50), default="offline") # offline, logging_in, online, error
|
||||
cookie_path = Column(String(255), nullable=True) # 存储 cookie/session 的路径
|
||||
cookie_data = Column(Text, nullable=True) # Playwright storage_state JSON
|
||||
cookie_updated_at = Column(DateTime, nullable=True) # Cookie 最近更新时间
|
||||
im_session_data = Column(Text, nullable=True) # IM 直连会话 (WS URL / device_id 等)
|
||||
reply_delay_seconds = Column(Integer, default=0) # 账号回复排队间隔;0/NULL=继承系统默认
|
||||
reply_cooldown_seconds = Column(Integer, nullable=True) # 自动回复冷却秒数;NULL=继承全局设置
|
||||
follow_welcome_enabled = Column(Boolean, default=False) # 新粉丝关注后自动发送欢迎语
|
||||
follow_welcome_content = Column(Text, nullable=True) # 关注欢迎语内容(空=不发)
|
||||
user_agent = Column(Text, nullable=True) # 伪装设备头(User-Agent),空=默认
|
||||
qr_code_base64 = Column(Text, nullable=True) # 当前登录二维码的 base64 字符串
|
||||
error_message = Column(Text, nullable=True) # 错误信息
|
||||
quota_disabled = Column(Boolean, default=False, index=True) # 额度不足被停用
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
# 关联
|
||||
owner = relationship("User", back_populates="accounts")
|
||||
rules = relationship("AutoReplyRule", back_populates="account", cascade="all, delete-orphan")
|
||||
logs = relationship("MessageLog", back_populates="account", cascade="all, delete-orphan")
|
||||
profile_detail = relationship(
|
||||
"AccountProfileDetail",
|
||||
back_populates="account",
|
||||
uselist=False,
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
videos = relationship(
|
||||
"AccountVideo",
|
||||
back_populates="account",
|
||||
cascade="all, delete-orphan",
|
||||
order_by="AccountVideo.sort_order",
|
||||
)
|
||||
|
||||
|
||||
class AccountProfileDetail(Base):
|
||||
"""托管账号抖音详细资料(本地缓存)。"""
|
||||
|
||||
__tablename__ = "account_profile_details"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), unique=True, nullable=False, index=True)
|
||||
uid = Column(String(64), nullable=True)
|
||||
nickname = Column(String(200), nullable=True)
|
||||
avatar_url = Column(String(512), nullable=True)
|
||||
unique_id = Column(String(100), nullable=True)
|
||||
signature = Column(Text, nullable=True)
|
||||
sec_user_id = Column(String(255), nullable=True)
|
||||
video_count = Column(Integer, nullable=True)
|
||||
follower_count = Column(Integer, nullable=True)
|
||||
following_count = Column(Integer, nullable=True)
|
||||
total_favorited = Column(Integer, nullable=True)
|
||||
favoriting_count = Column(Integer, nullable=True)
|
||||
sync_message = Column(Text, nullable=True)
|
||||
synced_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
account = relationship("Account", back_populates="profile_detail")
|
||||
|
||||
|
||||
class AccountVideo(Base):
|
||||
"""托管账号已发布作品(本地缓存)。"""
|
||||
|
||||
__tablename__ = "account_videos"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
aweme_id = Column(String(64), nullable=False, index=True)
|
||||
title = Column(Text, nullable=True)
|
||||
cover_url = Column(String(1024), nullable=True)
|
||||
video_url = Column(String(2048), nullable=True)
|
||||
share_url = Column(String(1024), nullable=True)
|
||||
create_time = Column(DateTime, nullable=True)
|
||||
digg_count = Column(Integer, nullable=True)
|
||||
comment_count = Column(Integer, nullable=True)
|
||||
play_count = Column(Integer, nullable=True)
|
||||
media_type = Column(String(20), nullable=True) # video, image, other
|
||||
sort_order = Column(Integer, default=0, index=True)
|
||||
synced_at = Column(DateTime, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
account = relationship("Account", back_populates="videos")
|
||||
|
||||
class LinkCardPage(Base):
|
||||
"""规则回复用的链接卡片落地页(带 SEO meta,打开后跳转至目标 URL)。"""
|
||||
|
||||
__tablename__ = "link_card_pages"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
owner_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
slug = Column(String(64), unique=True, index=True, nullable=False)
|
||||
title = Column(String(200), nullable=False)
|
||||
content = Column(Text, nullable=True)
|
||||
target_url = Column(String(2000), nullable=False)
|
||||
image_path = Column(String(512), nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
|
||||
class AutoReplyRule(Base):
|
||||
__tablename__ = "rules"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
owner_id = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
|
||||
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=True) # 为空代表全局规则
|
||||
keyword = Column(String(255), index=True) # 触发关键词,或者空代表兜底
|
||||
reply_content = Column(Text) # 回复内容
|
||||
match_type = Column(String(50), default="contains") # exact (精确), contains (包含), regex (正则), default (兜底)
|
||||
sort_order = Column(Integer, default=0, index=True) # 规则优先级,越小越优先
|
||||
is_active = Column(Boolean, default=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
account = relationship("Account", back_populates="rules")
|
||||
|
||||
class MessageLog(Base):
|
||||
__tablename__ = "message_logs"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"))
|
||||
sender_name = Column(String(100)) # 发送者名字
|
||||
sender_id = Column(String(100), nullable=True) # 发送者唯一ID
|
||||
sender_avatar = Column(String(512), nullable=True) # 发送者头像 URL
|
||||
message_content = Column(Text) # 接收到的消息
|
||||
reply_content = Column(Text, nullable=True) # 回复的消息
|
||||
status = Column(String(50), default="received") # received, replied, ignored, failed
|
||||
error_message = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
account = relationship("Account", back_populates="logs")
|
||||
|
||||
|
||||
class ReceivedMessageLog(Base):
|
||||
"""接收消息原始日志:仅记录收到的消息,内容原样保存。"""
|
||||
|
||||
__tablename__ = "received_message_logs"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
conversation_id = Column(String(128), nullable=True, index=True)
|
||||
sender_id = Column(String(100), nullable=True, index=True)
|
||||
sender_name = Column(String(100), nullable=True)
|
||||
sender_avatar = Column(String(512), nullable=True)
|
||||
message_type = Column(Integer, nullable=True)
|
||||
server_message_id = Column(String(64), nullable=True, index=True)
|
||||
raw_content = Column(Text, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow, index=True)
|
||||
|
||||
|
||||
class FollowWelcomeLog(Base):
|
||||
"""关注欢迎语去重表:每个账号对每个新粉丝只发送一次欢迎语(重启后仍生效)。"""
|
||||
|
||||
__tablename__ = "follow_welcome_logs"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
follower_uid = Column(String(64), nullable=False, index=True) # 新粉丝的抖音 UID
|
||||
status = Column(String(20), default="sent") # sent, failed
|
||||
detail = Column(Text, nullable=True)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("account_id", "follower_uid", name="uq_follow_welcome_account_follower"),
|
||||
)
|
||||
|
||||
|
||||
class SystemLog(Base):
|
||||
"""系统诊断日志:记录私信收发/连接/鉴权等链路事件,便于排查失败原因。"""
|
||||
__tablename__ = "system_logs"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
account_id = Column(Integer, nullable=True, index=True) # 关联账号(可空,全局事件)
|
||||
level = Column(String(20), default="info", index=True) # info, success, warning, error
|
||||
category = Column(String(40), default="system", index=True) # ws, send, recv, auth, poll, system
|
||||
event = Column(String(255)) # 事件标题
|
||||
detail = Column(Text, nullable=True) # 详细原因
|
||||
created_at = Column(DateTime, default=datetime.utcnow, index=True)
|
||||
|
||||
|
||||
class PaymentOrder(Base):
|
||||
"""账号额度购买订单。"""
|
||||
|
||||
__tablename__ = "payment_orders"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
order_no = Column(String(64), unique=True, index=True, nullable=False)
|
||||
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
channel = Column(String(20), nullable=False) # wechat, alipay
|
||||
slots = Column(Integer, nullable=False)
|
||||
amount_fen = Column(Integer, nullable=False)
|
||||
status = Column(String(20), default="pending", index=True) # pending, paid, expired, cancelled, refunded
|
||||
slots_applied = Column(Boolean, default=False)
|
||||
qr_code = Column(Text, nullable=True)
|
||||
pay_url = Column(Text, nullable=True)
|
||||
trade_no = Column(String(128), nullable=True)
|
||||
notify_payload = Column(Text, nullable=True)
|
||||
paid_at = Column(DateTime, nullable=True)
|
||||
expires_at = Column(DateTime, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
"""支付宝当面付扫码(precreate)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from base64 import b64decode, b64encode
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import quote_plus, urlencode
|
||||
|
||||
import httpx
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import padding
|
||||
|
||||
from auth.system_settings import SystemSettingsData
|
||||
|
||||
logger = logging.getLogger("payments.alipay")
|
||||
|
||||
ALIPAY_GATEWAY = "https://openapi.alipay.com/gateway.do"
|
||||
ALIPAY_SANDBOX_GATEWAY = "https://openapi-sandbox.dl.alipaydev.com/gateway.do"
|
||||
|
||||
|
||||
def _normalize_pem(text: str, label: str) -> str:
|
||||
body = text.strip()
|
||||
if "BEGIN" in body:
|
||||
return body
|
||||
return f"-----BEGIN {label}-----\n{body}\n-----END {label}-----"
|
||||
|
||||
|
||||
def _load_private_key(pem: str):
|
||||
return serialization.load_pem_private_key(
|
||||
_normalize_pem(pem, "RSA PRIVATE KEY").encode("utf-8"),
|
||||
password=None,
|
||||
)
|
||||
|
||||
|
||||
def _load_public_key(pem: str):
|
||||
return serialization.load_pem_public_key(
|
||||
_normalize_pem(pem, "PUBLIC KEY").encode("utf-8"),
|
||||
)
|
||||
|
||||
|
||||
def _sign_content(content: str, private_key) -> str:
|
||||
signature = private_key.sign(
|
||||
content.encode("utf-8"),
|
||||
padding.PKCS1v15(),
|
||||
hashes.SHA256(),
|
||||
)
|
||||
return b64encode(signature).decode("utf-8")
|
||||
|
||||
|
||||
def _gateway(settings: SystemSettingsData) -> str:
|
||||
return ALIPAY_SANDBOX_GATEWAY if settings.alipay_sandbox else ALIPAY_GATEWAY
|
||||
|
||||
|
||||
def create_precreate_order(
|
||||
settings: SystemSettingsData,
|
||||
order_no: str,
|
||||
subject: str,
|
||||
amount_yuan: str,
|
||||
notify_url: str,
|
||||
) -> str:
|
||||
"""创建当面付预下单,返回 qr_code 字符串。"""
|
||||
private_key = _load_private_key(settings.alipay_private_key)
|
||||
biz_content = {
|
||||
"out_trade_no": order_no,
|
||||
"total_amount": amount_yuan,
|
||||
"subject": subject[:256],
|
||||
}
|
||||
params = {
|
||||
"app_id": settings.alipay_app_id.strip(),
|
||||
"method": "alipay.trade.precreate",
|
||||
"format": "JSON",
|
||||
"charset": "utf-8",
|
||||
"sign_type": "RSA2",
|
||||
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"version": "1.0",
|
||||
"notify_url": notify_url,
|
||||
"biz_content": json.dumps(biz_content, ensure_ascii=False, separators=(",", ":")),
|
||||
}
|
||||
unsigned = "&".join(
|
||||
f"{k}={quote_plus(str(v), safe='')}"
|
||||
for k, v in sorted(params.items())
|
||||
if v is not None and str(v) != ""
|
||||
)
|
||||
params["sign"] = _sign_content(unsigned, private_key)
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
resp = client.post(_gateway(settings), data=params)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
key = "alipay_trade_precreate_response"
|
||||
result = data.get(key) or {}
|
||||
if result.get("code") != "10000":
|
||||
raise RuntimeError(result.get("sub_msg") or result.get("msg") or "支付宝下单失败")
|
||||
qr_code = result.get("qr_code")
|
||||
if not qr_code:
|
||||
raise RuntimeError("支付宝未返回 qr_code")
|
||||
return qr_code
|
||||
|
||||
|
||||
def verify_notify(params: dict[str, Any], settings: SystemSettingsData) -> bool:
|
||||
sign = params.get("sign")
|
||||
if not sign:
|
||||
return False
|
||||
verify_params = {k: v for k, v in params.items() if k not in ("sign", "sign_type") and v is not None}
|
||||
unsigned = "&".join(
|
||||
f"{k}={v}"
|
||||
for k, v in sorted(verify_params.items())
|
||||
if str(v) != ""
|
||||
)
|
||||
try:
|
||||
public_key = _load_public_key(settings.alipay_public_key)
|
||||
public_key.verify(
|
||||
b64decode(sign),
|
||||
unsigned.encode("utf-8"),
|
||||
padding.PKCS1v15(),
|
||||
hashes.SHA256(),
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Alipay notify verify failed: %s", exc)
|
||||
return False
|
||||
@@ -0,0 +1,125 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query, Request
|
||||
from fastapi.responses import PlainTextResponse
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from auth.dependencies import get_current_user, require_admin
|
||||
from auth.system_settings import load_settings
|
||||
from models.database import get_db
|
||||
from models.models import User
|
||||
from . import service
|
||||
from .schemas import (
|
||||
AdminUpdateOrderStatusRequest,
|
||||
CreatePaymentOrderRequest,
|
||||
MessageResponse,
|
||||
PaymentConfigResponse,
|
||||
PaymentOrderListItem,
|
||||
PaymentOrderListResponse,
|
||||
PaymentOrderResponse,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/payments", tags=["payments"])
|
||||
|
||||
|
||||
@router.get("/config", response_model=PaymentConfigResponse)
|
||||
async def get_payment_config(
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(get_current_user),
|
||||
):
|
||||
settings = await load_settings(db)
|
||||
return PaymentConfigResponse(**service.payment_config_payload(settings))
|
||||
|
||||
|
||||
@router.post("/orders", response_model=PaymentOrderResponse)
|
||||
async def create_payment_order(
|
||||
body: CreatePaymentOrderRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: User = Depends(get_current_user),
|
||||
):
|
||||
order, demo_mode = await service.create_order(db, user, body.slots, body.channel)
|
||||
return PaymentOrderResponse(**service.order_to_dict(order, demo_mode=demo_mode))
|
||||
|
||||
|
||||
@router.get("/orders", response_model=PaymentOrderListResponse)
|
||||
async def list_payment_orders(
|
||||
status: str | None = Query(default=None, description="pending/paid/expired/cancelled"),
|
||||
channel: str | None = Query(default=None, description="wechat/alipay"),
|
||||
page: int = Query(default=1, ge=1),
|
||||
page_size: int = Query(default=20, ge=1, le=100),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: User = Depends(get_current_user),
|
||||
):
|
||||
if status and status not in service.ORDER_STATUSES:
|
||||
raise HTTPException(status_code=400, detail="无效的订单状态")
|
||||
data = await service.list_orders(
|
||||
db,
|
||||
user,
|
||||
status=status,
|
||||
channel=channel,
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
)
|
||||
return PaymentOrderListResponse(**data)
|
||||
|
||||
|
||||
@router.get("/orders/{order_no}", response_model=PaymentOrderResponse)
|
||||
async def get_payment_order(
|
||||
order_no: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: User = Depends(get_current_user),
|
||||
):
|
||||
settings = await load_settings(db)
|
||||
order = await service.get_user_order(db, user, order_no)
|
||||
demo_mode = settings.payment_demo_mode and not settings.payment_channel_available(order.channel)
|
||||
return PaymentOrderResponse(**service.order_to_dict(order, demo_mode=demo_mode))
|
||||
|
||||
|
||||
@router.post("/orders/{order_no}/simulate", response_model=PaymentOrderResponse)
|
||||
async def simulate_payment_order(
|
||||
order_no: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
user: User = Depends(get_current_user),
|
||||
):
|
||||
settings = await load_settings(db)
|
||||
order = await service.simulate_pay(db, user, order_no)
|
||||
return PaymentOrderResponse(**service.order_to_dict(order, demo_mode=settings.payment_demo_mode))
|
||||
|
||||
|
||||
@router.put("/orders/{order_no}/status", response_model=PaymentOrderListItem)
|
||||
async def admin_update_payment_order_status(
|
||||
order_no: str,
|
||||
body: AdminUpdateOrderStatusRequest,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
order = await service.admin_update_order_status(db, order_no, body.status)
|
||||
return PaymentOrderListItem(**order)
|
||||
|
||||
|
||||
@router.delete("/orders/{order_no}", response_model=MessageResponse)
|
||||
async def admin_delete_payment_order(
|
||||
order_no: str,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_: User = Depends(require_admin),
|
||||
):
|
||||
await service.admin_delete_order(db, order_no)
|
||||
return MessageResponse(message="订单已删除")
|
||||
|
||||
|
||||
@router.post("/notify/wechat")
|
||||
async def wechat_payment_notify(request: Request, db: AsyncSession = Depends(get_db)):
|
||||
body = await request.body()
|
||||
headers = {k.lower(): v for k, v in request.headers.items()}
|
||||
ok = await service.handle_wechat_notify(db, body, headers)
|
||||
if not ok:
|
||||
return PlainTextResponse("FAIL", status_code=400)
|
||||
return PlainTextResponse('{"code": "SUCCESS", "message": "成功"}', media_type="application/json")
|
||||
|
||||
|
||||
@router.post("/notify/alipay")
|
||||
async def alipay_payment_notify(request: Request, db: AsyncSession = Depends(get_db)):
|
||||
form = await request.form()
|
||||
params = {k: v for k, v in form.items()}
|
||||
ok = await service.handle_alipay_notify(db, params)
|
||||
if not ok:
|
||||
return PlainTextResponse("fail")
|
||||
return PlainTextResponse("success")
|
||||
@@ -0,0 +1,69 @@
|
||||
from datetime import datetime
|
||||
from typing import Literal, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class PaymentConfigResponse(BaseModel):
|
||||
payment_enabled: bool
|
||||
demo_mode: bool
|
||||
unit_price: float
|
||||
min_slots: int
|
||||
max_slots: int
|
||||
channels: list[str]
|
||||
wechat_available: bool
|
||||
alipay_available: bool
|
||||
|
||||
|
||||
class CreatePaymentOrderRequest(BaseModel):
|
||||
slots: int = Field(ge=1, le=100)
|
||||
channel: Literal["wechat", "alipay"]
|
||||
|
||||
|
||||
class PaymentOrderResponse(BaseModel):
|
||||
order_no: str
|
||||
channel: str
|
||||
slots: int
|
||||
amount_fen: int
|
||||
amount_yuan: float
|
||||
status: str
|
||||
qr_code: Optional[str] = None
|
||||
pay_url: Optional[str] = None
|
||||
paid_at: Optional[datetime] = None
|
||||
expires_at: Optional[datetime] = None
|
||||
created_at: Optional[datetime] = None
|
||||
demo_mode: bool = False
|
||||
|
||||
|
||||
class PaymentOrderListItem(BaseModel):
|
||||
id: int
|
||||
order_no: str
|
||||
user_id: int
|
||||
username: Optional[str] = None
|
||||
display_name: Optional[str] = None
|
||||
channel: str
|
||||
slots: int
|
||||
amount_fen: int
|
||||
amount_yuan: float
|
||||
status: str
|
||||
trade_no: Optional[str] = None
|
||||
is_demo: bool = False
|
||||
paid_at: Optional[datetime] = None
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class PaymentOrderListResponse(BaseModel):
|
||||
items: list[PaymentOrderListItem]
|
||||
total: int
|
||||
page: int
|
||||
page_size: int
|
||||
paid_count: int = 0
|
||||
paid_amount_yuan: float = 0
|
||||
|
||||
|
||||
class AdminUpdateOrderStatusRequest(BaseModel):
|
||||
status: Literal["pending", "paid", "expired", "cancelled", "refunded"]
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
message: str
|
||||
@@ -0,0 +1,432 @@
|
||||
"""支付订单业务逻辑。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from auth.account_quota import default_stop_worker, sync_user_account_quota
|
||||
from auth.roles import is_admin
|
||||
from auth.system_settings import SystemSettingsData, load_settings
|
||||
from models.models import PaymentOrder, User
|
||||
from . import alipay, wechat
|
||||
|
||||
logger = logging.getLogger("payments.service")
|
||||
|
||||
ORDER_TTL_MINUTES = 30
|
||||
ORDER_STATUSES = ("pending", "paid", "expired", "cancelled", "refunded")
|
||||
|
||||
|
||||
def generate_order_no() -> str:
|
||||
return f"AC{datetime.utcnow().strftime('%Y%m%d%H%M%S')}{secrets.token_hex(4).upper()}"
|
||||
|
||||
|
||||
def yuan_to_fen(yuan: float) -> int:
|
||||
return int(round(yuan * 100))
|
||||
|
||||
|
||||
def fen_to_yuan(fen: int) -> float:
|
||||
return round(fen / 100, 2)
|
||||
|
||||
|
||||
def order_to_dict(order: PaymentOrder, demo_mode: bool = False) -> dict[str, Any]:
|
||||
return {
|
||||
"order_no": order.order_no,
|
||||
"channel": order.channel,
|
||||
"slots": order.slots,
|
||||
"amount_fen": order.amount_fen,
|
||||
"amount_yuan": fen_to_yuan(order.amount_fen),
|
||||
"status": order.status,
|
||||
"qr_code": order.qr_code,
|
||||
"pay_url": order.pay_url,
|
||||
"paid_at": order.paid_at.isoformat() if order.paid_at else None,
|
||||
"expires_at": order.expires_at.isoformat() if order.expires_at else None,
|
||||
"created_at": order.created_at.isoformat() if order.created_at else None,
|
||||
"demo_mode": demo_mode,
|
||||
}
|
||||
|
||||
|
||||
def payment_config_payload(settings: SystemSettingsData) -> dict[str, Any]:
|
||||
channels = settings.available_payment_channels()
|
||||
demo = settings.payment_demo_mode and settings.payment_enabled
|
||||
return {
|
||||
"payment_enabled": settings.payment_enabled,
|
||||
"demo_mode": demo,
|
||||
"unit_price": settings.account_slot_unit_price,
|
||||
"min_slots": settings.account_slot_purchase_min,
|
||||
"max_slots": settings.account_slot_purchase_max,
|
||||
"channels": channels,
|
||||
"wechat_available": "wechat" in channels,
|
||||
"alipay_available": "alipay" in channels,
|
||||
}
|
||||
|
||||
|
||||
def _validate_purchase_request(settings: SystemSettingsData, slots: int, channel: str) -> None:
|
||||
if not settings.payment_enabled:
|
||||
raise HTTPException(status_code=400, detail="在线购买功能未开启")
|
||||
if slots < settings.account_slot_purchase_min or slots > settings.account_slot_purchase_max:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"购买数量需在 {settings.account_slot_purchase_min}~{settings.account_slot_purchase_max} 之间",
|
||||
)
|
||||
if channel not in ("wechat", "alipay"):
|
||||
raise HTTPException(status_code=400, detail="不支持的支付方式")
|
||||
if channel == "wechat" and not settings.wechat_pay_enabled:
|
||||
raise HTTPException(status_code=400, detail="微信支付未开启")
|
||||
if channel == "alipay" and not settings.alipay_pay_enabled:
|
||||
raise HTTPException(status_code=400, detail="支付宝支付未开启")
|
||||
if not settings.payment_channel_selectable(channel):
|
||||
name = "微信" if channel == "wechat" else "支付宝"
|
||||
raise HTTPException(status_code=400, detail=f"{name}支付暂不可用,请联系管理员")
|
||||
if not settings.payment_channel_available(channel):
|
||||
if not settings.payment_demo_mode:
|
||||
name = "微信" if channel == "wechat" else "支付宝"
|
||||
raise HTTPException(status_code=400, detail=f"{name}支付尚未配置完成,请联系管理员")
|
||||
|
||||
|
||||
async def create_order(
|
||||
db: AsyncSession,
|
||||
user: User,
|
||||
slots: int,
|
||||
channel: str,
|
||||
) -> tuple[PaymentOrder, bool]:
|
||||
if is_admin(user.role):
|
||||
raise HTTPException(status_code=400, detail="管理员账号无需购买额度")
|
||||
settings = await load_settings(db)
|
||||
_validate_purchase_request(settings, slots, channel)
|
||||
|
||||
amount_fen = yuan_to_fen(settings.account_slot_unit_price * slots)
|
||||
if amount_fen < 1:
|
||||
raise HTTPException(status_code=400, detail="订单金额无效")
|
||||
|
||||
order = PaymentOrder(
|
||||
order_no=generate_order_no(),
|
||||
user_id=user.id,
|
||||
channel=channel,
|
||||
slots=slots,
|
||||
amount_fen=amount_fen,
|
||||
status="pending",
|
||||
expires_at=datetime.utcnow() + timedelta(minutes=ORDER_TTL_MINUTES),
|
||||
)
|
||||
db.add(order)
|
||||
await db.flush()
|
||||
|
||||
demo_mode = False
|
||||
notify_base = settings.app_url_normalized()
|
||||
description = f"抖音账号额度 x{slots}"
|
||||
|
||||
try:
|
||||
if channel == "wechat" and settings.wechat_pay_enabled and settings.wechat_pay_configured():
|
||||
notify_url = f"{notify_base}/api/payments/notify/wechat"
|
||||
order.qr_code = wechat.create_native_order(
|
||||
settings,
|
||||
order.order_no,
|
||||
description,
|
||||
amount_fen,
|
||||
notify_url,
|
||||
)
|
||||
elif channel == "alipay" and settings.alipay_pay_enabled and settings.alipay_configured():
|
||||
notify_url = f"{notify_base}/api/payments/notify/alipay"
|
||||
order.qr_code = alipay.create_precreate_order(
|
||||
settings,
|
||||
order.order_no,
|
||||
description,
|
||||
f"{fen_to_yuan(amount_fen):.2f}",
|
||||
notify_url,
|
||||
)
|
||||
elif settings.payment_demo_mode and settings.payment_channel_selectable(channel):
|
||||
demo_mode = True
|
||||
order.qr_code = f"DEMO-{order.order_no}"
|
||||
else:
|
||||
raise HTTPException(status_code=400, detail="支付渠道未配置")
|
||||
except HTTPException:
|
||||
await db.rollback()
|
||||
raise
|
||||
except Exception as exc:
|
||||
await db.rollback()
|
||||
logger.exception("Create payment order failed")
|
||||
raise HTTPException(status_code=400, detail=str(exc)) from exc
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(order)
|
||||
return order, demo_mode
|
||||
|
||||
|
||||
async def get_user_order(db: AsyncSession, user: User, order_no: str) -> PaymentOrder:
|
||||
result = await db.execute(
|
||||
select(PaymentOrder).where(
|
||||
PaymentOrder.order_no == order_no,
|
||||
PaymentOrder.user_id == user.id,
|
||||
)
|
||||
)
|
||||
order = result.scalar_one_or_none()
|
||||
if not order:
|
||||
raise HTTPException(status_code=404, detail="订单不存在")
|
||||
if order.status == "pending" and order.expires_at < datetime.utcnow():
|
||||
order.status = "expired"
|
||||
await db.commit()
|
||||
await db.refresh(order)
|
||||
return order
|
||||
|
||||
|
||||
async def _get_order_user(db: AsyncSession, order: PaymentOrder) -> User:
|
||||
result = await db.execute(select(User).where(User.id == order.user_id))
|
||||
user = result.scalar_one_or_none()
|
||||
if not user:
|
||||
raise HTTPException(status_code=404, detail="用户不存在")
|
||||
return user
|
||||
|
||||
|
||||
async def _apply_order_slots(db: AsyncSession, order: PaymentOrder, user: User) -> None:
|
||||
if order.slots_applied:
|
||||
return
|
||||
if is_admin(user.role) or (user.max_accounts is not None and user.max_accounts < 0):
|
||||
order.slots_applied = True
|
||||
return
|
||||
base = int(user.max_accounts) if user.max_accounts is not None else 0
|
||||
user.max_accounts = base + int(order.slots)
|
||||
order.slots_applied = True
|
||||
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
|
||||
|
||||
|
||||
async def _revoke_order_slots(db: AsyncSession, order: PaymentOrder, user: User) -> None:
|
||||
if not order.slots_applied:
|
||||
return
|
||||
if is_admin(user.role) or (user.max_accounts is not None and user.max_accounts < 0):
|
||||
order.slots_applied = False
|
||||
return
|
||||
base = int(user.max_accounts) if user.max_accounts is not None else 0
|
||||
new_limit = max(0, base - int(order.slots))
|
||||
user.max_accounts = new_limit
|
||||
order.slots_applied = False
|
||||
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
|
||||
|
||||
|
||||
async def _get_order_by_no(db: AsyncSession, order_no: str) -> PaymentOrder:
|
||||
result = await db.execute(select(PaymentOrder).where(PaymentOrder.order_no == order_no))
|
||||
order = result.scalar_one_or_none()
|
||||
if not order:
|
||||
raise HTTPException(status_code=404, detail="订单不存在")
|
||||
return order
|
||||
|
||||
|
||||
async def fulfill_order(db: AsyncSession, order: PaymentOrder, trade_no: str | None = None) -> None:
|
||||
if order.status == "paid" and order.slots_applied:
|
||||
return
|
||||
if order.status != "pending":
|
||||
raise HTTPException(status_code=400, detail="订单状态不可支付")
|
||||
|
||||
user = await _get_order_user(db, order)
|
||||
await _apply_order_slots(db, order, user)
|
||||
|
||||
order.status = "paid"
|
||||
order.paid_at = datetime.utcnow()
|
||||
if trade_no:
|
||||
order.trade_no = trade_no
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def simulate_pay(db: AsyncSession, user: User, order_no: str) -> PaymentOrder:
|
||||
settings = await load_settings(db)
|
||||
if not settings.payment_demo_mode:
|
||||
raise HTTPException(status_code=403, detail="演示支付未开启")
|
||||
order = await get_user_order(db, user, order_no)
|
||||
if order.status != "pending":
|
||||
raise HTTPException(status_code=400, detail="订单不可支付")
|
||||
await fulfill_order(db, order, trade_no=f"DEMO-{order.order_no}")
|
||||
await db.refresh(order)
|
||||
return order
|
||||
|
||||
|
||||
async def handle_wechat_notify(db: AsyncSession, body: bytes, headers: dict[str, str]) -> bool:
|
||||
settings = await load_settings(db)
|
||||
if not settings.wechat_pay_configured():
|
||||
return False
|
||||
if not wechat.verify_notify_signature(
|
||||
settings,
|
||||
body,
|
||||
headers.get("wechatpay-timestamp", ""),
|
||||
headers.get("wechatpay-nonce", ""),
|
||||
headers.get("wechatpay-signature", ""),
|
||||
headers.get("wechatpay-serial", ""),
|
||||
):
|
||||
return False
|
||||
|
||||
import json
|
||||
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
resource = payload.get("resource") or {}
|
||||
data = wechat.decrypt_notify_resource(settings.wechat_api_v3_key.strip(), resource)
|
||||
if data.get("trade_state") != "SUCCESS":
|
||||
return True
|
||||
|
||||
order_no = data.get("out_trade_no")
|
||||
trade_no = data.get("transaction_id")
|
||||
result = await db.execute(select(PaymentOrder).where(PaymentOrder.order_no == order_no))
|
||||
order = result.scalar_one_or_none()
|
||||
if not order:
|
||||
return False
|
||||
order.notify_payload = body.decode("utf-8", errors="replace")
|
||||
await fulfill_order(db, order, trade_no=trade_no)
|
||||
return True
|
||||
|
||||
|
||||
async def handle_alipay_notify(db: AsyncSession, params: dict[str, Any]) -> bool:
|
||||
settings = await load_settings(db)
|
||||
if not settings.alipay_configured():
|
||||
return False
|
||||
if not alipay.verify_notify(params, settings):
|
||||
return False
|
||||
if params.get("trade_status") not in ("TRADE_SUCCESS", "TRADE_FINISHED"):
|
||||
return True
|
||||
|
||||
order_no = params.get("out_trade_no")
|
||||
trade_no = params.get("trade_no")
|
||||
result = await db.execute(select(PaymentOrder).where(PaymentOrder.order_no == order_no))
|
||||
order = result.scalar_one_or_none()
|
||||
if not order:
|
||||
return False
|
||||
order.notify_payload = str(params)
|
||||
await fulfill_order(db, order, trade_no=trade_no)
|
||||
return True
|
||||
|
||||
|
||||
async def refresh_expired_orders(db: AsyncSession) -> None:
|
||||
now = datetime.utcnow()
|
||||
await db.execute(
|
||||
update(PaymentOrder)
|
||||
.where(PaymentOrder.status == "pending", PaymentOrder.expires_at < now)
|
||||
.values(status="expired")
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
def _order_is_demo(order: PaymentOrder) -> bool:
|
||||
trade_no = (order.trade_no or "").strip()
|
||||
return trade_no.startswith("DEMO-")
|
||||
|
||||
|
||||
def order_list_item(order: PaymentOrder, user: User | None = None) -> dict[str, Any]:
|
||||
payload = {
|
||||
"id": order.id,
|
||||
"order_no": order.order_no,
|
||||
"user_id": order.user_id,
|
||||
"username": user.username if user else None,
|
||||
"display_name": user.display_name if user else None,
|
||||
"channel": order.channel,
|
||||
"slots": order.slots,
|
||||
"amount_fen": order.amount_fen,
|
||||
"amount_yuan": fen_to_yuan(order.amount_fen),
|
||||
"status": order.status,
|
||||
"trade_no": order.trade_no,
|
||||
"is_demo": _order_is_demo(order),
|
||||
"paid_at": order.paid_at,
|
||||
"created_at": order.created_at,
|
||||
}
|
||||
return payload
|
||||
|
||||
|
||||
async def list_orders(
|
||||
db: AsyncSession,
|
||||
current_user: User,
|
||||
*,
|
||||
status: str | None = None,
|
||||
channel: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 20,
|
||||
) -> dict[str, Any]:
|
||||
await refresh_expired_orders(db)
|
||||
|
||||
page = max(1, page)
|
||||
page_size = max(1, min(100, page_size))
|
||||
|
||||
filters = []
|
||||
if not is_admin(current_user.role):
|
||||
filters.append(PaymentOrder.user_id == current_user.id)
|
||||
if status:
|
||||
filters.append(PaymentOrder.status == status)
|
||||
if channel in ("wechat", "alipay"):
|
||||
filters.append(PaymentOrder.channel == channel)
|
||||
|
||||
count_stmt = select(func.count()).select_from(PaymentOrder)
|
||||
for clause in filters:
|
||||
count_stmt = count_stmt.where(clause)
|
||||
total = int((await db.execute(count_stmt)).scalar() or 0)
|
||||
|
||||
paid_filters = list(filters) + [PaymentOrder.status == "paid"]
|
||||
paid_count_stmt = select(func.count()).select_from(PaymentOrder)
|
||||
for clause in paid_filters:
|
||||
paid_count_stmt = paid_count_stmt.where(clause)
|
||||
paid_count = int((await db.execute(paid_count_stmt)).scalar() or 0)
|
||||
|
||||
paid_amount_stmt = select(func.coalesce(func.sum(PaymentOrder.amount_fen), 0)).select_from(PaymentOrder)
|
||||
for clause in paid_filters:
|
||||
paid_amount_stmt = paid_amount_stmt.where(clause)
|
||||
paid_amount_fen = int((await db.execute(paid_amount_stmt)).scalar() or 0)
|
||||
|
||||
stmt = select(PaymentOrder, User).join(User, User.id == PaymentOrder.user_id)
|
||||
for clause in filters:
|
||||
stmt = stmt.where(clause)
|
||||
stmt = (
|
||||
stmt.order_by(PaymentOrder.created_at.desc())
|
||||
.offset((page - 1) * page_size)
|
||||
.limit(page_size)
|
||||
)
|
||||
rows = (await db.execute(stmt)).all()
|
||||
|
||||
items = [order_list_item(order, user) for order, user in rows]
|
||||
return {
|
||||
"items": items,
|
||||
"total": total,
|
||||
"page": page,
|
||||
"page_size": page_size,
|
||||
"paid_count": paid_count,
|
||||
"paid_amount_yuan": fen_to_yuan(paid_amount_fen),
|
||||
}
|
||||
|
||||
|
||||
async def admin_update_order_status(
|
||||
db: AsyncSession,
|
||||
order_no: str,
|
||||
new_status: str,
|
||||
) -> dict[str, Any]:
|
||||
if new_status not in ORDER_STATUSES:
|
||||
raise HTTPException(status_code=400, detail="无效的订单状态")
|
||||
|
||||
order = await _get_order_by_no(db, order_no)
|
||||
user = await _get_order_user(db, order)
|
||||
old_status = order.status
|
||||
|
||||
if new_status == old_status:
|
||||
return order_list_item(order, user)
|
||||
|
||||
if new_status == "paid":
|
||||
await _apply_order_slots(db, order, user)
|
||||
order.status = "paid"
|
||||
if not order.paid_at:
|
||||
order.paid_at = datetime.utcnow()
|
||||
else:
|
||||
if order.slots_applied:
|
||||
await _revoke_order_slots(db, order, user)
|
||||
order.status = new_status
|
||||
if new_status not in ("paid", "refunded"):
|
||||
order.paid_at = None
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(order)
|
||||
return order_list_item(order, user)
|
||||
|
||||
|
||||
async def admin_delete_order(db: AsyncSession, order_no: str) -> None:
|
||||
order = await _get_order_by_no(db, order_no)
|
||||
if order.slots_applied:
|
||||
user = await _get_order_user(db, order)
|
||||
await _revoke_order_slots(db, order, user)
|
||||
await db.delete(order)
|
||||
await db.commit()
|
||||
@@ -0,0 +1,139 @@
|
||||
"""微信支付 V3 Native 扫码。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from base64 import b64decode, b64encode
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import padding
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
from auth.system_settings import SystemSettingsData
|
||||
|
||||
logger = logging.getLogger("payments.wechat")
|
||||
|
||||
WECHAT_API = "https://api.mch.weixin.qq.com"
|
||||
|
||||
|
||||
def _load_private_key(pem: str):
|
||||
text = pem.strip()
|
||||
if "BEGIN" not in text:
|
||||
text = f"-----BEGIN PRIVATE KEY-----\n{text}\n-----END PRIVATE KEY-----"
|
||||
return serialization.load_pem_private_key(text.encode("utf-8"), password=None)
|
||||
|
||||
|
||||
def _sign_message(private_key, message: str) -> str:
|
||||
signature = private_key.sign(
|
||||
message.encode("utf-8"),
|
||||
padding.PKCS1v15(),
|
||||
hashes.SHA256(),
|
||||
)
|
||||
return b64encode(signature).decode("utf-8")
|
||||
|
||||
|
||||
def _build_auth_header(
|
||||
method: str,
|
||||
url_path: str,
|
||||
body: str,
|
||||
mch_id: str,
|
||||
serial_no: str,
|
||||
private_key,
|
||||
) -> str:
|
||||
timestamp = str(int(time.time()))
|
||||
nonce = uuid.uuid4().hex
|
||||
message = f"{method}\n{url_path}\n{timestamp}\n{nonce}\n{body}\n"
|
||||
sign = _sign_message(private_key, message)
|
||||
return (
|
||||
f'WECHATPAY2-SHA256-RSA2048 mchid="{mch_id}",'
|
||||
f'nonce_str="{nonce}",signature="{sign}",'
|
||||
f'timestamp="{timestamp}",serial_no="{serial_no}"'
|
||||
)
|
||||
|
||||
|
||||
def create_native_order(
|
||||
settings: SystemSettingsData,
|
||||
order_no: str,
|
||||
description: str,
|
||||
amount_fen: int,
|
||||
notify_url: str,
|
||||
) -> str:
|
||||
"""创建 Native 订单,返回 code_url。"""
|
||||
private_key = _load_private_key(settings.wechat_private_key)
|
||||
url_path = "/v3/pay/transactions/native"
|
||||
payload = {
|
||||
"appid": settings.wechat_app_id.strip(),
|
||||
"mchid": settings.wechat_mch_id.strip(),
|
||||
"description": description[:127],
|
||||
"out_trade_no": order_no,
|
||||
"notify_url": notify_url,
|
||||
"amount": {"total": amount_fen, "currency": "CNY"},
|
||||
}
|
||||
body = json.dumps(payload, ensure_ascii=False)
|
||||
headers = {
|
||||
"Authorization": _build_auth_header(
|
||||
"POST",
|
||||
url_path,
|
||||
body,
|
||||
settings.wechat_mch_id.strip(),
|
||||
settings.wechat_cert_serial.strip(),
|
||||
private_key,
|
||||
),
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
}
|
||||
with httpx.Client(timeout=30.0) as client:
|
||||
resp = client.post(f"{WECHAT_API}{url_path}", content=body.encode("utf-8"), headers=headers)
|
||||
if resp.status_code >= 400:
|
||||
detail = resp.text
|
||||
try:
|
||||
detail = resp.json().get("message") or detail
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError(f"微信支付下单失败: {detail}")
|
||||
data = resp.json()
|
||||
code_url = data.get("code_url")
|
||||
if not code_url:
|
||||
raise RuntimeError("微信支付未返回 code_url")
|
||||
return code_url
|
||||
|
||||
|
||||
def decrypt_notify_resource(api_v3_key: str, resource: dict[str, Any]) -> dict[str, Any]:
|
||||
nonce = resource.get("nonce", "")
|
||||
ciphertext = resource.get("ciphertext", "")
|
||||
associated_data = resource.get("associated_data", "")
|
||||
aesgcm = AESGCM(api_v3_key.encode("utf-8"))
|
||||
plain = aesgcm.decrypt(
|
||||
nonce.encode("utf-8"),
|
||||
b64decode(ciphertext),
|
||||
associated_data.encode("utf-8") if associated_data else None,
|
||||
)
|
||||
return json.loads(plain.decode("utf-8"))
|
||||
|
||||
|
||||
def verify_notify_signature(
|
||||
settings: SystemSettingsData,
|
||||
body: bytes,
|
||||
timestamp: str,
|
||||
nonce: str,
|
||||
signature: str,
|
||||
serial: str,
|
||||
) -> bool:
|
||||
"""简化验签:使用平台证书较复杂,此处用商户私钥对应逻辑 + 回调解密校验。"""
|
||||
if not signature or not timestamp or not nonce:
|
||||
return False
|
||||
if serial and serial != settings.wechat_cert_serial.strip():
|
||||
logger.warning("WeChat notify serial mismatch: %s", serial)
|
||||
try:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
resource = payload.get("resource") or {}
|
||||
decrypt_notify_resource(settings.wechat_api_v3_key.strip(), resource)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("WeChat notify verify failed: %s", exc)
|
||||
return False
|
||||
@@ -0,0 +1,134 @@
|
||||
# =============================================================================
|
||||
# 抖音多账号自动回复系统 - 完整 Python 依赖
|
||||
# 安装: pip install -r requirements.txt
|
||||
# 环境: Python 3.10 ~ 3.11
|
||||
# =============================================================================
|
||||
#
|
||||
# 【推荐】一键安装与 Web 部署:
|
||||
#
|
||||
# Linux/宝塔(自动探测 Python,兼容 pyporject_evn / 系统 Python):
|
||||
# chmod +x install.sh start_web.sh
|
||||
# ./install.sh --diagnose # 查看本机哪些 Python 可用
|
||||
# ./install.sh # 自动选择可用 Python 安装
|
||||
# AUTO_BUILD_PYTHON=1 ./install.sh # 无可用 Python 时自动编译 3.11
|
||||
# ./start_web.sh
|
||||
#
|
||||
# Windows:
|
||||
# install.bat # 安装 + 构建前端
|
||||
# start_web.bat # 单端口 Web 访问
|
||||
#
|
||||
# pip 安装完成后,还需在服务器执行(install.sh 已包含):
|
||||
#
|
||||
# 1) Playwright 浏览器(扫码登录需要)
|
||||
# python -m playwright install chromium
|
||||
# python -m playwright install-deps # 仅 Linux
|
||||
#
|
||||
# 2) IM 签名 Node 依赖(发送私信 a_bogus 必需,需系统已装 Node.js 18+)
|
||||
# cd rpa_engine/douyin_im/static && npm install
|
||||
#
|
||||
# 3) 前端构建(生产 Web 访问)
|
||||
# cd ../../frontend && npm install && npm run build
|
||||
#
|
||||
# 4) 环境变量(生产务必设置,见项目根 .env.example)
|
||||
# KEFU_SECRET_KEY=随机长字符串
|
||||
# KEFU_ADMIN_PASSWORD=强密码
|
||||
# PLAYWRIGHT_BROWSERS_PATH=/path/to/playwright-browsers
|
||||
#
|
||||
# 宝塔 Python 项目管理器启动命令:
|
||||
# /www/wwwroot/douyin/backend/.venv/bin/python -m uvicorn main:app --host 0.0.0.0 --port 8000
|
||||
# 工作目录: /www/wwwroot/douyin/backend
|
||||
# 运行用户: www(与 chown -R www:www 一致)
|
||||
# =============================================================================
|
||||
|
||||
# --- Web 框架 / ASGI ---
|
||||
fastapi==0.111.0
|
||||
starlette==0.37.2
|
||||
uvicorn==0.30.1
|
||||
httptools==0.8.0
|
||||
watchfiles==1.2.0
|
||||
websockets==12.0
|
||||
python-multipart==0.0.9
|
||||
|
||||
# --- 数据校验 / 配置 ---
|
||||
pydantic==2.7.4
|
||||
pydantic_core==2.18.4
|
||||
annotated-types==0.7.0
|
||||
typing_extensions==4.15.0
|
||||
email-validator==2.3.0
|
||||
dnspython==2.8.0
|
||||
python-dotenv==1.2.2
|
||||
|
||||
# --- 数据库 ---
|
||||
SQLAlchemy==2.0.30
|
||||
aiosqlite==0.20.0
|
||||
# MySQL / PostgreSQL 可选驱动(使用对应数据库类型时需安装)
|
||||
asyncmy==0.2.9
|
||||
asyncpg==0.29.0
|
||||
greenlet==3.0.3
|
||||
|
||||
# --- HTTP 客户端 ---
|
||||
httpx==0.27.0
|
||||
httpcore==1.0.9
|
||||
h11==0.16.0
|
||||
requests==2.34.2
|
||||
urllib3==2.7.0
|
||||
certifi==2026.5.20
|
||||
charset-normalizer==3.4.7
|
||||
idna==3.18
|
||||
sniffio==1.3.1
|
||||
anyio==4.13.0
|
||||
|
||||
# --- 鉴权 / 密码 ---
|
||||
python-jose==3.3.0
|
||||
passlib==1.7.4
|
||||
bcrypt==4.0.1
|
||||
cryptography==49.0.0
|
||||
cffi==2.0.0
|
||||
pycparser==3.0
|
||||
ecdsa==0.19.2
|
||||
pyasn1==0.6.3
|
||||
rsa==4.9.1
|
||||
|
||||
# --- 图片处理(卡片 favicon 转换 / 私信图片尺寸)---
|
||||
Pillow==10.4.0
|
||||
|
||||
# --- 浏览器自动化(登录 / 采集凭证)---
|
||||
playwright==1.44.0
|
||||
pyee==11.1.0
|
||||
# 无图形界面的 Linux 服务器跑有头浏览器需要虚拟显示(还需系统安装 Xvfb):
|
||||
# Debian/Ubuntu: apt install -y xvfb
|
||||
# CentOS/Rocky : yum install -y xorg-x11-server-Xvfb
|
||||
pyvirtualdisplay==3.0
|
||||
|
||||
# --- 抖音 IM(Protobuf / 签名 / WebSocket)---
|
||||
protobuf==5.27.1
|
||||
protobuf3-to-dict==0.1.5
|
||||
six==1.17.0
|
||||
PyExecJS==1.5.1
|
||||
websocket-client==1.9.0
|
||||
|
||||
# --- FastAPI CLI / 工具(随 fastapi 安装,显式锁定避免缺包)---
|
||||
fastapi-cli==0.0.24
|
||||
typer==0.26.7
|
||||
click==8.4.1
|
||||
shellingham==1.5.4
|
||||
rich==15.0.0
|
||||
rich-toolkit==0.20.1
|
||||
Pygments==2.20.0
|
||||
markdown-it-py==4.2.0
|
||||
mdurl==0.1.2
|
||||
Jinja2==3.1.6
|
||||
MarkupSafe==3.0.3
|
||||
annotated-doc==0.0.4
|
||||
orjson==3.11.9
|
||||
ujson==5.12.1
|
||||
PyYAML==6.0.3
|
||||
colorama==0.4.6
|
||||
|
||||
# =============================================================================
|
||||
# [npm] IM 签名 jsrsasign(在 rpa_engine/douyin_im/static/package.json)
|
||||
# jsrsasign@^11.1.0
|
||||
#
|
||||
# [npm] 前端(在 frontend/package.json)
|
||||
# vue, ant-design-vue, axios, pinia, vue-router, vite
|
||||
# =============================================================================
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,318 @@
|
||||
"""Bounded, deduplicated background queue for bulk account starts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
|
||||
logger = logging.getLogger("rpa.batch_start")
|
||||
|
||||
StartHandler = Callable[[int], Awaitable[dict[str, Any]]]
|
||||
JobToken = tuple[str, int]
|
||||
|
||||
|
||||
def _configured_concurrency() -> int:
|
||||
try:
|
||||
return max(1, min(8, int(os.getenv("KEFU_BATCH_START_CONCURRENCY", "2"))))
|
||||
except (TypeError, ValueError):
|
||||
return 2
|
||||
|
||||
|
||||
def _utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _BatchRecord:
|
||||
batch_id: str
|
||||
owner_id: int | None = None
|
||||
metadata: dict[str, Any] = field(default_factory=dict)
|
||||
items: dict[int, dict[str, Any]] = field(default_factory=dict)
|
||||
created_at: str = field(default_factory=_utc_now)
|
||||
updated_at: str = field(default_factory=_utc_now)
|
||||
|
||||
|
||||
class BatchStartQueue:
|
||||
"""Run account preparation with a small process-wide concurrency cap.
|
||||
|
||||
The HTTP endpoint only enqueues account ids. Long credential checks then
|
||||
run in these workers, so a batch of many accounts cannot block the request
|
||||
that submitted it. ``_pending_accounts`` makes overlapping clicks and
|
||||
overlapping batches idempotent for each account. Pending and active
|
||||
ownership is tied to a concrete job token so cleanup from a cancelled old
|
||||
job cannot release a newer submission for the same account.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
handler: StartHandler,
|
||||
concurrency: int | None = None,
|
||||
max_batches: int = 100,
|
||||
) -> None:
|
||||
self._handler = handler
|
||||
self.concurrency = max(1, int(concurrency or _configured_concurrency()))
|
||||
self.max_batches = max(10, int(max_batches or 100))
|
||||
self._queue: asyncio.Queue[JobToken] = asyncio.Queue()
|
||||
self._pending_jobs: dict[int, JobToken] = {}
|
||||
self._active_tasks: dict[int, tuple[JobToken, asyncio.Task]] = {}
|
||||
self._batches: dict[str, _BatchRecord] = {}
|
||||
self._workers: list[asyncio.Task] = []
|
||||
self._lock = asyncio.Lock()
|
||||
self._stopping = False
|
||||
|
||||
async def _ensure_workers(self) -> None:
|
||||
async with self._lock:
|
||||
self._workers = [task for task in self._workers if not task.done()]
|
||||
if self._workers or self._stopping:
|
||||
return
|
||||
for index in range(self.concurrency):
|
||||
self._workers.append(
|
||||
asyncio.create_task(
|
||||
self._worker(index + 1),
|
||||
name=f"account-batch-start-{index + 1}",
|
||||
)
|
||||
)
|
||||
|
||||
async def submit(
|
||||
self,
|
||||
account_ids: list[int],
|
||||
owner_id: int | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
await self._ensure_workers()
|
||||
unique_ids = list(dict.fromkeys(int(value) for value in account_ids if int(value) > 0))
|
||||
batch_id = uuid.uuid4().hex
|
||||
record = _BatchRecord(
|
||||
batch_id=batch_id,
|
||||
owner_id=owner_id,
|
||||
metadata=dict(metadata or {}),
|
||||
)
|
||||
|
||||
async with self._lock:
|
||||
if self._stopping:
|
||||
raise RuntimeError("账号启动队列正在停止")
|
||||
self._prune_locked()
|
||||
self._batches[batch_id] = record
|
||||
for account_id in unique_ids:
|
||||
if account_id in self._pending_jobs:
|
||||
record.items[account_id] = {
|
||||
"account_id": account_id,
|
||||
"status": "already_queued",
|
||||
"message": "账号已在启动队列中",
|
||||
}
|
||||
continue
|
||||
job_token = (batch_id, account_id)
|
||||
self._pending_jobs[account_id] = job_token
|
||||
record.items[account_id] = {
|
||||
"account_id": account_id,
|
||||
"status": "queued",
|
||||
"message": "等待启动",
|
||||
}
|
||||
self._queue.put_nowait(job_token)
|
||||
record.updated_at = _utc_now()
|
||||
return self._snapshot_locked(record)
|
||||
|
||||
async def get_batch(
|
||||
self,
|
||||
batch_id: str,
|
||||
owner_id: int | None = None,
|
||||
include_items: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
record = self._batches.get(str(batch_id or ""))
|
||||
if record and owner_id is not None and record.owner_id != int(owner_id):
|
||||
return None
|
||||
return self._snapshot_locked(record, include_items=include_items) if record else None
|
||||
|
||||
async def _worker(self, worker_number: int) -> None:
|
||||
while True:
|
||||
batch_id, account_id = await self._queue.get()
|
||||
job_token = (batch_id, account_id)
|
||||
started_at = time.monotonic()
|
||||
handler_task: asyncio.Task | None = None
|
||||
try:
|
||||
async with self._lock:
|
||||
record = self._batches.get(batch_id)
|
||||
if not record:
|
||||
if self._pending_jobs.get(account_id) == job_token:
|
||||
self._pending_jobs.pop(account_id, None)
|
||||
continue
|
||||
item = record.items[account_id]
|
||||
if item.get("status") == "cancelled":
|
||||
if self._pending_jobs.get(account_id) == job_token:
|
||||
self._pending_jobs.pop(account_id, None)
|
||||
continue
|
||||
item.update(status="processing", message="正在校验并启动")
|
||||
record.updated_at = _utc_now()
|
||||
handler_task = asyncio.create_task(
|
||||
self._handler(account_id),
|
||||
name=f"account-start-{account_id}",
|
||||
)
|
||||
self._active_tasks[account_id] = (job_token, handler_task)
|
||||
|
||||
result = await handler_task
|
||||
async with self._lock:
|
||||
record = self._batches.get(batch_id)
|
||||
if record:
|
||||
item = record.items[account_id]
|
||||
if item.get("status") != "cancelled":
|
||||
item.update(
|
||||
status="submitted",
|
||||
message=str(result.get("message") or "已提交启动"),
|
||||
login_mode=result.get("login_mode"),
|
||||
skip_browser=bool(result.get("skip_browser", False)),
|
||||
elapsed_seconds=round(time.monotonic() - started_at, 3),
|
||||
)
|
||||
record.updated_at = _utc_now()
|
||||
except asyncio.CancelledError:
|
||||
async with self._lock:
|
||||
record = self._batches.get(batch_id)
|
||||
if record:
|
||||
record.items[account_id].update(
|
||||
status="cancelled",
|
||||
message="服务停止,启动任务已取消",
|
||||
)
|
||||
record.updated_at = _utc_now()
|
||||
# Cancelling one account must not kill a long-lived queue
|
||||
# worker. Re-raise only when stop() cancelled the worker.
|
||||
if asyncio.current_task().cancelling():
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"Batch start failed account=%s worker=%s: %s",
|
||||
account_id,
|
||||
worker_number,
|
||||
exc,
|
||||
)
|
||||
async with self._lock:
|
||||
record = self._batches.get(batch_id)
|
||||
if record:
|
||||
detail = getattr(exc, "detail", None) or str(exc) or "启动失败"
|
||||
item = record.items[account_id]
|
||||
if item.get("status") != "cancelled":
|
||||
item.update(
|
||||
status="failed",
|
||||
message=str(detail),
|
||||
elapsed_seconds=round(time.monotonic() - started_at, 3),
|
||||
)
|
||||
record.updated_at = _utc_now()
|
||||
finally:
|
||||
async with self._lock:
|
||||
active_entry = self._active_tasks.get(account_id)
|
||||
if active_entry == (job_token, handler_task):
|
||||
self._active_tasks.pop(account_id, None)
|
||||
if self._pending_jobs.get(account_id) == job_token:
|
||||
self._pending_jobs.pop(account_id, None)
|
||||
self._queue.task_done()
|
||||
|
||||
async def cancel_account(self, account_id: int) -> int:
|
||||
"""Cancel queued/processing work so stop/delete cannot restart it."""
|
||||
account_key = int(account_id)
|
||||
cancelled = 0
|
||||
active_task: asyncio.Task | None = None
|
||||
async with self._lock:
|
||||
for record in self._batches.values():
|
||||
item = record.items.get(account_key)
|
||||
if not item or item.get("status") not in ("queued", "processing"):
|
||||
continue
|
||||
item.update(status="cancelled", message="启动任务已取消")
|
||||
record.updated_at = _utc_now()
|
||||
cancelled += 1
|
||||
self._pending_jobs.pop(account_key, None)
|
||||
active_entry = self._active_tasks.get(account_key)
|
||||
active_task = active_entry[1] if active_entry else None
|
||||
if active_task and not active_task.done():
|
||||
active_task.cancel()
|
||||
if active_task and not active_task.done():
|
||||
await asyncio.gather(active_task, return_exceptions=True)
|
||||
return cancelled
|
||||
|
||||
def _snapshot_locked(
|
||||
self,
|
||||
record: _BatchRecord,
|
||||
*,
|
||||
include_items: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
counts = {
|
||||
"queued": 0,
|
||||
"processing": 0,
|
||||
"submitted": 0,
|
||||
"failed": 0,
|
||||
"skipped": 0,
|
||||
"cancelled": 0,
|
||||
}
|
||||
browser_required = 0
|
||||
for item in record.items.values():
|
||||
status = item.get("status")
|
||||
if status == "submitted" and item.get("skip_browser") is False:
|
||||
browser_required += 1
|
||||
if status == "already_queued":
|
||||
counts["skipped"] += 1
|
||||
elif status in counts:
|
||||
counts[status] += 1
|
||||
active = counts["queued"] + counts["processing"]
|
||||
snapshot = {
|
||||
"batch_id": record.batch_id,
|
||||
"total_count": len(record.items),
|
||||
"accepted_count": len(record.items) - counts["skipped"],
|
||||
"queued_count": counts["queued"],
|
||||
"processing_count": counts["processing"],
|
||||
"submitted_count": counts["submitted"],
|
||||
"failed_count": counts["failed"],
|
||||
"skipped_count": counts["skipped"],
|
||||
"cancelled_count": counts["cancelled"],
|
||||
"browser_required_count": browser_required,
|
||||
"complete": active == 0,
|
||||
"created_at": record.created_at,
|
||||
"updated_at": record.updated_at,
|
||||
}
|
||||
snapshot.update(record.metadata)
|
||||
if include_items:
|
||||
snapshot["items"] = [dict(item) for item in record.items.values()]
|
||||
return snapshot
|
||||
|
||||
def _prune_locked(self) -> None:
|
||||
if len(self._batches) < self.max_batches:
|
||||
return
|
||||
removable = [
|
||||
batch_id
|
||||
for batch_id, record in self._batches.items()
|
||||
if self._snapshot_locked(record, include_items=False)["complete"]
|
||||
]
|
||||
for batch_id in removable[: max(1, len(self._batches) - self.max_batches + 1)]:
|
||||
self._batches.pop(batch_id, None)
|
||||
|
||||
async def stop(self) -> None:
|
||||
async with self._lock:
|
||||
self._stopping = True
|
||||
workers = list(self._workers)
|
||||
self._workers.clear()
|
||||
for task in workers:
|
||||
task.cancel()
|
||||
if workers:
|
||||
await asyncio.gather(*workers, return_exceptions=True)
|
||||
async with self._lock:
|
||||
self._active_tasks.clear()
|
||||
while True:
|
||||
try:
|
||||
batch_id, account_id = self._queue.get_nowait()
|
||||
except asyncio.QueueEmpty:
|
||||
break
|
||||
record = self._batches.get(batch_id)
|
||||
if record:
|
||||
record.items[account_id].update(
|
||||
status="cancelled",
|
||||
message="服务停止,启动任务已取消",
|
||||
)
|
||||
record.updated_at = _utc_now()
|
||||
job_token = (batch_id, account_id)
|
||||
if self._pending_jobs.get(account_id) == job_token:
|
||||
self._pending_jobs.pop(account_id, None)
|
||||
self._queue.task_done()
|
||||
@@ -0,0 +1,226 @@
|
||||
"""账号凭证检测:静态 Cookie 分析 + IM 运行时校验"""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from rpa_engine.douyin_im.auth import DouyinAuth
|
||||
from rpa_engine.douyin_im.frontier import ensure_frontier_ws
|
||||
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
|
||||
from rpa_engine.douyin_im.session import DouyinImSession
|
||||
from utils.cookie_store import analyze_cookie
|
||||
|
||||
logger = logging.getLogger("credential")
|
||||
|
||||
|
||||
def _should_reset_credentials(assessment: dict) -> bool:
|
||||
"""凭证全面失效时需清空 Cookie/IM 数据并重新登录。"""
|
||||
if not assessment.get("has_cookie"):
|
||||
return False
|
||||
if not assessment.get("cookie_valid"):
|
||||
return True
|
||||
message = assessment.get("message") or ""
|
||||
# 仅缺 ticket/签名/浏览器采集 — 保留 Cookie,走浏览器补全即可
|
||||
if any(
|
||||
token in message
|
||||
for token in ("ticket", "签名密钥", "浏览器模式", "web_protect")
|
||||
):
|
||||
return False
|
||||
if (
|
||||
not assessment.get("im_ready")
|
||||
and not assessment.get("can_skip_browser")
|
||||
and assessment.get("has_sessionid")
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build_im_session_from_storage(
|
||||
storage: dict,
|
||||
saved_im_data: Optional[str] = None,
|
||||
) -> DouyinImSession:
|
||||
session = DouyinImSession.from_storage_state(storage or {})
|
||||
if saved_im_data:
|
||||
try:
|
||||
saved = DouyinImSession.from_dict(json.loads(saved_im_data))
|
||||
# 新粘贴的 storage_state(含真实 frontier_ws_url)优先;只有它没带时才用缓存的。
|
||||
if saved.ws_urls and not session.ws_urls:
|
||||
session.ws_urls = saved.ws_urls
|
||||
if saved.sdk_cert and not session.sdk_cert:
|
||||
session.sdk_cert = saved.sdk_cert
|
||||
if saved.frontier_ts_sign and not session.frontier_ts_sign:
|
||||
session.frontier_ts_sign = saved.frontier_ts_sign
|
||||
if saved.keys_str and not session.keys_str:
|
||||
session.keys_str = saved.keys_str
|
||||
if saved.web_protect_str and not session.web_protect_str:
|
||||
session.web_protect_str = saved.web_protect_str
|
||||
if saved.my_uid and not session.my_uid:
|
||||
session.my_uid = saved.my_uid
|
||||
if saved.device_id and not session.device_id:
|
||||
session.device_id = saved.device_id
|
||||
if saved.web_id and not session.web_id:
|
||||
session.web_id = saved.web_id
|
||||
if saved.conv_meta:
|
||||
session.conv_meta = {**saved.conv_meta, **session.conv_meta}
|
||||
except Exception:
|
||||
pass
|
||||
session.sanitize_ws_urls()
|
||||
# Keep this builder pure and non-blocking. Frontier discovery can perform
|
||||
# a synchronous network request (up to 15 seconds); callers that need it
|
||||
# already do so from validate_im_session() through asyncio.to_thread() and
|
||||
# the shared background-traffic limiter. Running it here made a large
|
||||
# batch freeze the FastAPI event loop before any limiter was acquired.
|
||||
return session
|
||||
|
||||
|
||||
def has_im_session_token(session: DouyinImSession) -> bool:
|
||||
return bool(session.cookies.get("sessionid") or session.cookies.get("sessionid_ss"))
|
||||
|
||||
|
||||
def extract_sessionid_info(session: DouyinImSession) -> dict:
|
||||
sessionid = session.cookies.get("sessionid") or ""
|
||||
sessionid_ss = session.cookies.get("sessionid_ss") or ""
|
||||
return {
|
||||
"has_sessionid": bool(sessionid or sessionid_ss),
|
||||
"sessionid": sessionid,
|
||||
"sessionid_ss": sessionid_ss,
|
||||
}
|
||||
|
||||
|
||||
async def build_cookie_credential_detail(
|
||||
cookie_data: Optional[str],
|
||||
im_session_data: Optional[str] = None,
|
||||
runtime_check: bool = True,
|
||||
) -> dict:
|
||||
"""供编辑账号页展示 IM 凭证与 sessionid 信息"""
|
||||
sessionid_info = {
|
||||
"has_sessionid": False,
|
||||
"sessionid": "",
|
||||
"sessionid_ss": "",
|
||||
"im_ready": False,
|
||||
"im_status": "未保存 Cookie",
|
||||
"can_skip_browser": False,
|
||||
"should_reset": False,
|
||||
}
|
||||
if not cookie_data:
|
||||
return sessionid_info
|
||||
|
||||
try:
|
||||
storage = json.loads(cookie_data)
|
||||
session = build_im_session_from_storage(storage, im_session_data)
|
||||
sessionid_info.update(extract_sessionid_info(session))
|
||||
except Exception:
|
||||
sessionid_info["im_status"] = "Cookie 格式错误"
|
||||
return sessionid_info
|
||||
|
||||
if not runtime_check:
|
||||
if sessionid_info["has_sessionid"]:
|
||||
sessionid_info["im_status"] = "已检测到 sessionid(未做运行时验证)"
|
||||
else:
|
||||
sessionid_info["im_status"] = "缺少 sessionid,无法 IM 直连"
|
||||
return sessionid_info
|
||||
|
||||
assessment = await assess_account_credential(cookie_data, im_session_data)
|
||||
sessionid_info["im_ready"] = assessment["im_ready"]
|
||||
sessionid_info["im_status"] = assessment["message"]
|
||||
sessionid_info["can_skip_browser"] = assessment["can_skip_browser"]
|
||||
sessionid_info["should_reset"] = assessment["should_reset"]
|
||||
return sessionid_info
|
||||
|
||||
|
||||
async def validate_im_session(
|
||||
session: DouyinImSession,
|
||||
_bypass_global_limit: bool = False,
|
||||
) -> tuple[bool, str]:
|
||||
if not _bypass_global_limit:
|
||||
from rpa_engine.douyin_im.traffic_control import get_traffic_controller
|
||||
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(0, "credential validation"):
|
||||
return await validate_im_session(session, _bypass_global_limit=True)
|
||||
|
||||
if not session.can_direct_im():
|
||||
if not has_im_session_token(session):
|
||||
return False, "缺少 sessionid,无法直连 IM"
|
||||
return False, "Cookie 不满足 IM 直连条件"
|
||||
|
||||
await asyncio.to_thread(ensure_frontier_ws, session)
|
||||
try:
|
||||
auth = DouyinAuth.from_im_session(session)
|
||||
# 优先用已持久化的 my_uid,避免每次都发起网络 query_my_uid(uid_tt 是加密串,
|
||||
# int() 解析必然失败而回退到网络请求;该请求偶发失败会误判为“未就绪”)。
|
||||
uid = session.my_uid or auth.get_uid()
|
||||
if not uid:
|
||||
return False, "服务端未认可当前 Cookie(无法获取用户 UID)"
|
||||
if not auth.is_sign_ready():
|
||||
return False, "缺少 IM 签名密钥(web_protect/keys),请用浏览器登录补全"
|
||||
session.my_uid = int(uid)
|
||||
async with DouyinImHttpClient(session) as http:
|
||||
await http.get_unread_count()
|
||||
# 若已缓存到会话票据,优先校验其是否仍新鲜(最理想)。
|
||||
if session.conv_meta:
|
||||
ok, reason = await http.verify_messaging_capability(auth, session.my_uid)
|
||||
if ok:
|
||||
return True, reason
|
||||
# 没有缓存会话票据是首次登录的正常情况:会话 ticket 会在发送时即时
|
||||
# 创建/获取(resolve_conversation_meta),因此只要 Cookie + sessionid +
|
||||
# 签名密钥(web_protect/keys) + UID 齐全,就视为可 IM 直连托管,不必再开浏览器。
|
||||
return True, "IM 凭证就绪(Cookie 与签名密钥齐全,可直连托管)"
|
||||
except Exception as e:
|
||||
logger.warning(f"IM session validation failed: {e}")
|
||||
return False, f"IM 运行时验证失败: {e}"
|
||||
|
||||
|
||||
async def assess_account_credential(
|
||||
cookie_data: Optional[str],
|
||||
im_session_data: Optional[str] = None,
|
||||
) -> dict:
|
||||
cookie_info = analyze_cookie(cookie_data)
|
||||
result = {
|
||||
"has_cookie": cookie_info.get("has_cookie", False),
|
||||
"cookie_valid": cookie_info.get("cookie_valid", False),
|
||||
"cookie_status": cookie_info.get("reason", ""),
|
||||
"has_sessionid": False,
|
||||
"im_ready": False,
|
||||
"can_skip_browser": False,
|
||||
"login_mode": "browser",
|
||||
"message": "未保存 Cookie,需浏览器扫码登录",
|
||||
"should_reset": False,
|
||||
}
|
||||
|
||||
if not cookie_data:
|
||||
return result
|
||||
|
||||
try:
|
||||
storage = json.loads(cookie_data)
|
||||
except Exception:
|
||||
result["message"] = "Cookie 格式错误"
|
||||
result["should_reset"] = _should_reset_credentials(result)
|
||||
return result
|
||||
|
||||
session = build_im_session_from_storage(storage, im_session_data)
|
||||
result["has_sessionid"] = has_im_session_token(session)
|
||||
|
||||
if not cookie_info.get("cookie_valid"):
|
||||
result["message"] = cookie_info.get("reason") or "Cookie 无效,需重新登录"
|
||||
result["should_reset"] = _should_reset_credentials(result)
|
||||
return result
|
||||
|
||||
if not result["has_sessionid"]:
|
||||
result["login_mode"] = "browser"
|
||||
result["message"] = "Cookie 已保存但缺少 sessionid,需浏览器刷新登录态"
|
||||
result["should_reset"] = _should_reset_credentials(result)
|
||||
return result
|
||||
|
||||
im_ok, im_reason = await validate_im_session(session)
|
||||
result["im_ready"] = im_ok
|
||||
if im_ok:
|
||||
result["can_skip_browser"] = True
|
||||
result["login_mode"] = "im_direct"
|
||||
result["message"] = im_reason
|
||||
else:
|
||||
result["login_mode"] = "browser"
|
||||
result["message"] = im_reason
|
||||
|
||||
result["should_reset"] = _should_reset_credentials(result)
|
||||
return result
|
||||
@@ -0,0 +1,89 @@
|
||||
"""浏览器 / IM 伪装设备头(User-Agent)预设。
|
||||
|
||||
a_bogus 签名、Playwright 浏览器上下文、IM HTTP 请求头、Protobuf body 必须使用同一 UA,
|
||||
否则抖音会返回 7911 安全校验失败。
|
||||
"""
|
||||
|
||||
from rpa_engine.douyin_im.dy_util import DEFAULT_USER_AGENT
|
||||
|
||||
# id 用于前端下拉;user_agent 为完整字符串
|
||||
DEVICE_PROFILES: list[dict[str, str]] = [
|
||||
{
|
||||
"id": "chrome_win120",
|
||||
"label": "Chrome 120 · Windows(默认,推荐)",
|
||||
"platform": "Windows",
|
||||
"user_agent": DEFAULT_USER_AGENT,
|
||||
},
|
||||
{
|
||||
"id": "chrome_win131",
|
||||
"label": "Chrome 131 · Windows",
|
||||
"platform": "Windows",
|
||||
"user_agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "edge_win125",
|
||||
"label": "Edge 125 · Windows",
|
||||
"platform": "Windows",
|
||||
"user_agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0"
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "firefox_win117",
|
||||
"label": "Firefox 117 · Windows",
|
||||
"platform": "Windows",
|
||||
"user_agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) "
|
||||
"Gecko/20100101 Firefox/117.0"
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "chrome_mac120",
|
||||
"label": "Chrome 120 · macOS",
|
||||
"platform": "macOS",
|
||||
"user_agent": (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
),
|
||||
},
|
||||
{
|
||||
"id": "safari_mac17",
|
||||
"label": "Safari 17 · macOS",
|
||||
"platform": "macOS",
|
||||
"user_agent": (
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/17.0 Safari/605.1.15"
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
_PROFILE_BY_UA = {p["user_agent"]: p for p in DEVICE_PROFILES}
|
||||
|
||||
|
||||
def list_device_profiles() -> list[dict[str, str]]:
|
||||
return [
|
||||
{"id": p["id"], "label": p["label"], "platform": p["platform"], "user_agent": p["user_agent"]}
|
||||
for p in DEVICE_PROFILES
|
||||
]
|
||||
|
||||
|
||||
def resolve_user_agent(stored: str | None) -> str:
|
||||
"""账号保存的 UA;空则使用默认 Chrome 120。"""
|
||||
text = (stored or "").strip()
|
||||
return text or DEFAULT_USER_AGENT
|
||||
|
||||
|
||||
def profile_label_for_ua(ua: str | None) -> str:
|
||||
text = (ua or "").strip()
|
||||
if not text:
|
||||
return "Chrome 120 · Windows(默认)"
|
||||
hit = _PROFILE_BY_UA.get(text)
|
||||
if hit:
|
||||
return hit["label"]
|
||||
if len(text) > 48:
|
||||
return text[:48] + "…"
|
||||
return text
|
||||
@@ -0,0 +1,3 @@
|
||||
from .service import DouyinImService
|
||||
|
||||
__all__ = ["DouyinImService"]
|
||||
@@ -0,0 +1,163 @@
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
from .dy_util import (
|
||||
trans_cookies,
|
||||
generate_msToken,
|
||||
generate_a_bogus,
|
||||
splice_url,
|
||||
generate_webid,
|
||||
normalize_client_cert,
|
||||
resolve_proto_device_id,
|
||||
DEFAULT_USER_AGENT,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("douyin_im.auth")
|
||||
|
||||
|
||||
def _parse_storage_json(raw) -> dict | None:
|
||||
"""Parse localStorage JSON (supports nested data wrapper)."""
|
||||
if not raw:
|
||||
return None
|
||||
if isinstance(raw, dict):
|
||||
data = raw
|
||||
else:
|
||||
text = str(raw).strip()
|
||||
data = None
|
||||
for _ in range(4):
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except Exception:
|
||||
break
|
||||
if isinstance(parsed, str):
|
||||
text = parsed
|
||||
continue
|
||||
if isinstance(parsed, dict):
|
||||
data = parsed
|
||||
break
|
||||
break
|
||||
if not data:
|
||||
return None
|
||||
|
||||
inner = data.get("data")
|
||||
if isinstance(inner, str):
|
||||
try:
|
||||
inner = json.loads(inner)
|
||||
except Exception:
|
||||
inner = None
|
||||
if isinstance(inner, dict):
|
||||
return inner
|
||||
return data
|
||||
|
||||
|
||||
class DouyinAuth:
|
||||
def __init__(self):
|
||||
self.cookie = None
|
||||
self.cookie_str = None
|
||||
self.private_key = None
|
||||
self.ticket = None
|
||||
self.ts_sign = None
|
||||
self.client_cert = None
|
||||
self.ree_public_key = None
|
||||
self.uid = None
|
||||
self.msToken = None
|
||||
self.web_id = None
|
||||
|
||||
def perepare_auth(self, cookieStr: str, web_protect_: str = "", keys_: str = ""):
|
||||
self.cookie = trans_cookies(cookieStr)
|
||||
self.cookie_str = cookieStr
|
||||
self.msToken = self.cookie["msToken"] if "msToken" in self.cookie else generate_msToken()
|
||||
self.cookie["msToken"] = self.msToken
|
||||
self.cookie_str = "; ".join([f"{k}={v}" for k, v in self.cookie.items()])
|
||||
web_data = _parse_storage_json(web_protect_)
|
||||
if web_data:
|
||||
try:
|
||||
self.ticket = web_data.get("ticket") or ""
|
||||
self.ts_sign = web_data.get("ts_sign") or ""
|
||||
self.client_cert = web_data.get("client_cert") or ""
|
||||
except Exception as e:
|
||||
logger.debug(f"web_protect parse failed: {e}")
|
||||
|
||||
keys_data = _parse_storage_json(keys_)
|
||||
if keys_data:
|
||||
try:
|
||||
self.private_key = keys_data.get("ec_privateKey") or keys_data.get("privateKey") or ""
|
||||
if self.private_key:
|
||||
self.ree_public_key = base64.b64encode(self.private_key.encode()).decode()
|
||||
except Exception as e:
|
||||
logger.debug(f"keys parse failed: {e}")
|
||||
|
||||
def is_sign_ready(self) -> bool:
|
||||
return bool(
|
||||
self.private_key
|
||||
and self.ticket
|
||||
and self.ts_sign
|
||||
and self.client_cert
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_im_session(cls, session) -> "DouyinAuth":
|
||||
"""从 DouyinImSession 构建 HTTP 发送/拉会话用的签名上下文。
|
||||
|
||||
注意:frontier WS 的 sdk_cert/ts_sign 只用于长连接,不能覆盖 web_protect,
|
||||
否则 bd-ticket-guard 与 protobuf 签名会与 ticket 失配 -> 7911。
|
||||
"""
|
||||
auth = cls()
|
||||
auth.perepare_auth(
|
||||
session.cookie_header(),
|
||||
session.web_protect_str,
|
||||
session.keys_str,
|
||||
)
|
||||
auth.web_id = session.web_id or session.device_id or None
|
||||
auth.user_agent = session.user_agent or DEFAULT_USER_AGENT
|
||||
auth.device_id = resolve_proto_device_id(
|
||||
session.device_id, session.web_id, session.my_uid
|
||||
)
|
||||
# web_protect 缺 client_cert 时,才用 frontier 抓包证书兜底(不覆盖 ts_sign)
|
||||
if not auth.client_cert and getattr(session, "sdk_cert", ""):
|
||||
auth.client_cert = normalize_client_cert(session.sdk_cert)
|
||||
elif auth.client_cert:
|
||||
auth.client_cert = normalize_client_cert(auth.client_cert)
|
||||
return auth
|
||||
|
||||
def get_uid(self):
|
||||
if self.uid is None:
|
||||
# 优先从 cookie 尝试提取,否则请求接口
|
||||
for k in ("uid_tt", "uid_tt_ss"):
|
||||
if self.cookie and self.cookie.get(k):
|
||||
try:
|
||||
self.uid = int(self.cookie.get(k))
|
||||
return self.uid
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
self.uid = self.query_my_uid()
|
||||
except Exception:
|
||||
pass
|
||||
return self.uid
|
||||
|
||||
def query_my_uid(self) -> int:
|
||||
url = 'https://www.douyin.com/aweme/v1/web/query/user/'
|
||||
headers = {
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"publish_video_strategy_type": "2",
|
||||
"verifyFp": self.cookie.get('s_v_web_id', ''),
|
||||
"fp": self.cookie.get('s_v_web_id', ''),
|
||||
"webid": generate_webid(self, "https://www.douyin.com/"),
|
||||
"msToken": self.msToken
|
||||
}
|
||||
query = splice_url(params)
|
||||
abogus = generate_a_bogus(query, user_agent=DEFAULT_USER_AGENT)
|
||||
params['a_bogus'] = abogus
|
||||
|
||||
resp = requests.get(url, params=params, headers=headers, cookies=self.cookie, verify=False, timeout=10)
|
||||
resp_json = resp.json()
|
||||
return int(resp_json['user_uid'])
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Conversation id helpers for Douyin IM."""
|
||||
|
||||
|
||||
def parse_conversation_parts(conversation_id: str) -> tuple[int, int] | None:
|
||||
parts = (conversation_id or "").split(":")
|
||||
if len(parts) >= 4 and parts[0] == "0" and parts[1] == "1":
|
||||
try:
|
||||
return int(parts[2]), int(parts[3])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def resolve_peer_uid(conversation_id: str, my_uid: int) -> int | None:
|
||||
"""Resolve peer user id from conversation id or bare numeric id."""
|
||||
raw = (conversation_id or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
|
||||
parts = parse_conversation_parts(raw)
|
||||
if parts:
|
||||
uid1, uid2 = parts
|
||||
if my_uid and uid1 == my_uid:
|
||||
return uid2
|
||||
if my_uid and uid2 == my_uid:
|
||||
return uid1
|
||||
# 0:1:{my}:{peer} — 若 my_uid 与首段不一致,仍取末段为对方
|
||||
return uid2
|
||||
|
||||
if raw.isdigit():
|
||||
peer = int(raw)
|
||||
if my_uid and peer == my_uid:
|
||||
return None
|
||||
return peer
|
||||
return None
|
||||
|
||||
|
||||
def build_conversation_id(my_uid: int, peer_uid: int) -> str:
|
||||
return f"0:1:{int(my_uid)}:{int(peer_uid)}"
|
||||
|
||||
|
||||
def normalize_conversation_id(conversation_id: str, my_uid: int) -> str:
|
||||
"""Ensure conversation id uses current account uid as first participant."""
|
||||
peer_uid = resolve_peer_uid(conversation_id, my_uid)
|
||||
if peer_uid and my_uid:
|
||||
return build_conversation_id(my_uid, peer_uid)
|
||||
return (conversation_id or "").strip()
|
||||
@@ -0,0 +1,205 @@
|
||||
import hashlib
|
||||
import re
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
import base64
|
||||
import urllib.parse
|
||||
from os import path
|
||||
import subprocess
|
||||
original_popen = subprocess.Popen
|
||||
def patched_popen(*args, **kwargs):
|
||||
if kwargs.get('universal_newlines') or kwargs.get('text'):
|
||||
if 'encoding' not in kwargs:
|
||||
kwargs['encoding'] = 'utf-8'
|
||||
return original_popen(*args, **kwargs)
|
||||
subprocess.Popen = patched_popen
|
||||
|
||||
import execjs
|
||||
import requests
|
||||
|
||||
basedir = path.dirname(__file__)
|
||||
static_dir = path.join(basedir, 'static')
|
||||
node_modules = path.join(static_dir, 'node_modules')
|
||||
|
||||
# 全局唯一 User-Agent:a_bogus 签名、HTTP 请求头、protobuf body、webid 采集等
|
||||
# 必须全部使用同一个 UA,否则抖音服务端重算 a_bogus 时与请求头 UA 不一致 -> 7911。
|
||||
# 该值需与浏览器登录上下文(playwright new_context user_agent)保持一致。
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
# 动态编译 JS
|
||||
dy_path = path.join(static_dir, 'dy_ab.js')
|
||||
dy_js = execjs.compile(open(dy_path, 'r', encoding='utf-8').read(), cwd=node_modules)
|
||||
|
||||
sign_path = path.join(static_dir, 'dy_live_sign.js')
|
||||
sign_js = execjs.compile(open(sign_path, 'r', encoding='utf-8').read(), cwd=node_modules)
|
||||
|
||||
login_path = path.join(static_dir, 'login.js')
|
||||
login_js = execjs.compile(open(login_path, 'r', encoding='utf-8').read(), cwd=node_modules)
|
||||
|
||||
|
||||
def generateSecretPhoneNum(phone):
|
||||
return login_js.call('generateSecretPhoneNum', phone)
|
||||
|
||||
|
||||
def generateSecretCode(phone, code):
|
||||
return login_js.call('generateSecretCode', phone, code)
|
||||
|
||||
|
||||
def trans_cookies(cookies_str):
|
||||
cookies = {}
|
||||
for i in cookies_str.split("; "):
|
||||
try:
|
||||
parts = i.split('=')
|
||||
key = parts[0].strip()
|
||||
val = '='.join(parts[1:]).strip()
|
||||
# 防御性清洗:过滤掉因为误粘贴 fetch 等包含非法字符或换行的 Cookie 键
|
||||
if not key or any(c in key for c in "()[]{}'\"\n \t\\"):
|
||||
continue
|
||||
cookies[key] = val
|
||||
except:
|
||||
continue
|
||||
return cookies
|
||||
|
||||
|
||||
def generate_req_sign(e, priK):
|
||||
"""私信传 obj,其他的拼接"""
|
||||
return dy_js.call('get_req_sign', e, priK)
|
||||
|
||||
|
||||
def generate_a_bogus(query, data="", user_agent=None):
|
||||
"""query, data 都是拼接字符串。
|
||||
|
||||
user_agent 必须与实际发出请求所用的 User-Agent 完全一致(见 DEFAULT_USER_AGENT),
|
||||
否则抖音服务端用请求头 UA 重算 a_bogus 会对不上,导致 7911 安全校验失败。
|
||||
"""
|
||||
return dy_js.call('get_ab', query, data, user_agent or DEFAULT_USER_AGENT)
|
||||
|
||||
|
||||
def generate_signature(room_id, user_unique_id):
|
||||
raw_string = f"live_id=1,aid=6383,version_code=180800,webcast_sdk_version=1.0.15,room_id={room_id},sub_room_id=,sub_channel_id=,did_rule=3,user_unique_id={user_unique_id},device_platform=web,device_type=,ac=,identity=audience"
|
||||
x_ms_stub = hashlib.md5(raw_string.encode("utf-8")).hexdigest()
|
||||
result = sign_js.call("get_signature", x_ms_stub)
|
||||
return result.get("X-Bogus")
|
||||
|
||||
|
||||
def generate_ree_key(prik):
|
||||
"""传递私钥"""
|
||||
return dy_js.call('get_ree_key', prik)
|
||||
|
||||
|
||||
def generate_bd_ticket_client_data(api, ticket, ts_sign, priK):
|
||||
"""传递 query, ticket, ts_sign, priK"""
|
||||
timestamp = int(time.time())
|
||||
res_sign = f"ticket={ticket}&path={api}×tamp={timestamp}"
|
||||
p = {
|
||||
'ts_sign': ts_sign,
|
||||
'req_content': 'ticket,path,timestamp',
|
||||
'req_sign': generate_req_sign(res_sign, priK),
|
||||
'timestamp': timestamp,
|
||||
}
|
||||
p = json.dumps(p, ensure_ascii=False, separators=(',', ':'))
|
||||
return base64.urlsafe_b64encode(p.encode('utf-8')).decode('utf-8')
|
||||
|
||||
|
||||
def generate_msToken(randomlength=107):
|
||||
random_str = ''
|
||||
base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789='
|
||||
length = len(base_str) - 1
|
||||
for _ in range(randomlength):
|
||||
random_str += base_str[random.randint(0, length)]
|
||||
return random_str
|
||||
|
||||
|
||||
def generate_fake_webid(random_length=19):
|
||||
random_str = ''
|
||||
base_str = '0123456789'
|
||||
length = len(base_str) - 1
|
||||
for _ in range(random_length):
|
||||
random_str += base_str[random.randint(0, length)]
|
||||
return random_str
|
||||
|
||||
|
||||
def generate_webid(auth=None, url=""):
|
||||
# 优先用已采集到的 web_id(避免每次发送都发起一次阻塞的 HTTP 请求,导致事件循环卡顿)
|
||||
cached = getattr(auth, "web_id", None) if auth is not None else None
|
||||
if cached:
|
||||
return str(cached)
|
||||
if url == "":
|
||||
url = "https://www.douyin.com/discover?modal_id=7376449060384935209"
|
||||
try:
|
||||
from .auth import DouyinAuth
|
||||
headers = {
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"upgrade-insecure-requests": "1"
|
||||
}
|
||||
if auth and auth.cookie_str:
|
||||
headers['cookie'] = auth.cookie_str
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
proxies = requests_proxies()
|
||||
except Exception:
|
||||
proxies = None
|
||||
response = requests.get(
|
||||
url, headers=headers, verify=False, timeout=10, proxies=proxies
|
||||
)
|
||||
res_text = response.text
|
||||
user_unique_id = re.findall(r'\\"user_unique_id\\":\\"(.*?)\\"', res_text)[0]
|
||||
# 把发现的 web_id 回写到 auth 上,避免同一次同步里反复发起阻塞的 HTTP 请求
|
||||
if auth is not None and user_unique_id:
|
||||
try:
|
||||
auth.web_id = user_unique_id
|
||||
except Exception:
|
||||
pass
|
||||
return user_unique_id
|
||||
except Exception:
|
||||
# 失败时同样缓存一个伪 web_id,避免后续调用重复走 10s 超时的网络请求
|
||||
fake = generate_fake_webid()
|
||||
if auth is not None:
|
||||
try:
|
||||
if not getattr(auth, "web_id", None):
|
||||
auth.web_id = fake
|
||||
except Exception:
|
||||
pass
|
||||
return fake
|
||||
|
||||
|
||||
def generate_millisecond():
|
||||
return int(round(time.time() * 1000))
|
||||
|
||||
|
||||
def normalize_client_cert(cert: str) -> str:
|
||||
"""统一 client_cert / sdk_cert 格式为「base64 证书体」。
|
||||
|
||||
web_protect.client_cert 与 frontier WS 的 sdk_cert 通常已是 base64(PEM);
|
||||
若误把 PEM 原文或 frontier 证书二次 base64,会导致 7911。
|
||||
"""
|
||||
cert = (cert or "").strip()
|
||||
if not cert:
|
||||
return ""
|
||||
if cert.startswith("-----BEGIN"):
|
||||
return base64.b64encode(cert.encode("utf-8")).decode("utf-8")
|
||||
return cert
|
||||
|
||||
|
||||
def resolve_proto_device_id(device_id: str = "", web_id: str = "", my_uid: int = 0) -> str:
|
||||
"""protobuf / frontier 更倾向使用数字 device_id(通常等于 my_uid)。"""
|
||||
for candidate in (device_id, web_id, str(my_uid or "")):
|
||||
c = str(candidate or "").strip()
|
||||
if c.isdigit():
|
||||
return c
|
||||
return str(device_id or web_id or "0")
|
||||
|
||||
|
||||
def splice_url(params):
|
||||
splice_url_str = ''
|
||||
for key, value in params.items():
|
||||
if value is None:
|
||||
value = ''
|
||||
splice_url_str += key + '=' + urllib.parse.quote(str(value)) + '&'
|
||||
return splice_url_str[:-1]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""抖音标准表情(评论区 emoji)名称 -> 图片 URL 映射。
|
||||
|
||||
抖音文字表情如 [酷拽]/[微笑] 通过 WS 以 message_type=7 的纯文本下发,
|
||||
content 形如 {"text":"[酷拽]","aweType":700},不带图片地址;
|
||||
浏览器端靠本地表情表把 [name] 渲染成小图。这里拉取官方表情列表接口,
|
||||
建立 name->url 映射,收到文字表情时补成可显示的贴纸。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("douyin_im.emoji")
|
||||
|
||||
_EMOJI_MAP: dict[str, str] = {}
|
||||
_FETCHED_AT: float = 0.0
|
||||
_TTL = 6 * 3600 # 6 小时刷新一次
|
||||
_lock = threading.Lock()
|
||||
_fetch_lock = threading.Lock()
|
||||
_LAST_FETCH_ATTEMPT: float = 0.0
|
||||
_FAILURE_RETRY_SECONDS = 60.0
|
||||
|
||||
_BRACKET_RE = re.compile(r"^\[[^\[\]]{1,24}\]$")
|
||||
|
||||
|
||||
def has_emoji_map() -> bool:
|
||||
return bool(_EMOJI_MAP)
|
||||
|
||||
|
||||
def is_fresh() -> bool:
|
||||
return bool(_EMOJI_MAP) and (time.time() - _FETCHED_AT) < _TTL
|
||||
|
||||
|
||||
def set_emoji_map(mapping: dict[str, str]) -> None:
|
||||
global _EMOJI_MAP, _FETCHED_AT
|
||||
if mapping:
|
||||
with _lock:
|
||||
_EMOJI_MAP = dict(mapping)
|
||||
_FETCHED_AT = time.time()
|
||||
|
||||
|
||||
def lookup_emoji_url(name: str) -> str:
|
||||
"""name 可带或不带中括号,返回标准表情图片 URL(无则空串)。"""
|
||||
if not name:
|
||||
return ""
|
||||
key = name.strip()
|
||||
if not key:
|
||||
return ""
|
||||
if not key.startswith("["):
|
||||
key = f"[{key}]"
|
||||
return _EMOJI_MAP.get(key, "")
|
||||
|
||||
|
||||
def looks_like_emoji_token(text: str) -> bool:
|
||||
return bool(_BRACKET_RE.match((text or "").strip()))
|
||||
|
||||
|
||||
def fetch_emoji_map(session) -> dict[str, str]:
|
||||
"""用账号会话拉取官方表情列表,返回 {display_name: url}。失败返回 {}。"""
|
||||
try:
|
||||
import requests
|
||||
|
||||
from .auth import DouyinAuth
|
||||
from .dy_util import generate_a_bogus, generate_msToken, splice_url
|
||||
|
||||
auth = DouyinAuth.from_im_session(session)
|
||||
ua = session.user_agent
|
||||
s_v_web_id = session.cookies.get("s_v_web_id", "") if session.cookies else ""
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"pc_client_type": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"cookie_enabled": "true",
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Mozilla",
|
||||
"browser_online": "true",
|
||||
"verifyFp": s_v_web_id,
|
||||
"fp": s_v_web_id,
|
||||
"webid": session.web_id or session.device_id or "",
|
||||
"msToken": generate_msToken(),
|
||||
}
|
||||
params["a_bogus"] = generate_a_bogus(splice_url(params), user_agent=ua)
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Cookie": session.cookie_header(),
|
||||
}
|
||||
resp = requests.get(
|
||||
"https://www.douyin.com/aweme/v1/web/emoji/list",
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie if getattr(auth, "cookie", None) else None,
|
||||
verify=False,
|
||||
timeout=20,
|
||||
)
|
||||
data = resp.json()
|
||||
mapping: dict[str, str] = {}
|
||||
for item in data.get("emoji_list") or []:
|
||||
name = item.get("display_name")
|
||||
urls = (item.get("emoji_url") or {}).get("url_list") or []
|
||||
if name and urls:
|
||||
mapping[name] = urls[0]
|
||||
logger.info("Fetched %d douyin emoji", len(mapping))
|
||||
return mapping
|
||||
except Exception as e:
|
||||
logger.warning("fetch_emoji_map failed: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def ensure_emoji_map(session) -> None:
|
||||
"""若缓存为空/过期则拉取(同步阻塞,调用方建议放线程)。"""
|
||||
global _LAST_FETCH_ATTEMPT
|
||||
if is_fresh():
|
||||
return
|
||||
# Batch-started accounts used to all observe an empty cache and fetch the
|
||||
# same emoji list concurrently. Keep the network request itself inside a
|
||||
# separate single-flight lock (set_emoji_map uses _lock).
|
||||
with _fetch_lock:
|
||||
if is_fresh():
|
||||
return
|
||||
now = time.time()
|
||||
if now - _LAST_FETCH_ATTEMPT < _FAILURE_RETRY_SECONDS:
|
||||
return
|
||||
_LAST_FETCH_ATTEMPT = now
|
||||
mapping = fetch_emoji_map(session)
|
||||
if mapping:
|
||||
set_emoji_map(mapping)
|
||||
@@ -0,0 +1,183 @@
|
||||
"""抖音网页版「粉丝列表」拉取,用于检测新粉丝(关注欢迎语功能)。
|
||||
|
||||
复用与 peer_profile / account_profile 相同的 a_bogus + msToken + cookie 签名方式,
|
||||
调用 https://www.douyin.com/aweme/v1/web/user/follower/list/ 拉取本账号最近的粉丝。
|
||||
|
||||
返回的每个粉丝含:uid / sec_uid / nickname / follow_status / follower_status。
|
||||
其中 follow_status 表示「我」与对方的关系:0=未关注 1=我已关注 2=互相关注(互关)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from .dy_util import (
|
||||
DEFAULT_USER_AGENT,
|
||||
generate_a_bogus,
|
||||
generate_msToken,
|
||||
generate_webid,
|
||||
splice_url,
|
||||
)
|
||||
from .auth import DouyinAuth
|
||||
|
||||
logger = logging.getLogger("douyin_im.follower_poll")
|
||||
|
||||
FOLLOWER_LIST_URL = "https://www.douyin.com/aweme/v1/web/user/follower/list/"
|
||||
|
||||
|
||||
def _requests_proxies() -> dict | None:
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
|
||||
return requests_proxies()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _to_int(value: Any) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_followers(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw = data.get("followers")
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
uid = str(item.get("uid") or item.get("user_id") or "").strip()
|
||||
if not uid:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"uid": uid,
|
||||
"sec_uid": str(item.get("sec_uid") or item.get("sec_user_id") or "").strip(),
|
||||
"nickname": str(item.get("nickname") or item.get("nick_name") or "").strip(),
|
||||
# follow_status:我对对方的关系(2=互关);follower_status:对方对我的关系
|
||||
"follow_status": _to_int(item.get("follow_status")),
|
||||
"follower_status": _to_int(item.get("follower_status")),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def fetch_recent_followers_sync(
|
||||
session,
|
||||
sec_user_id: str,
|
||||
count: int = 20,
|
||||
max_time: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""同步拉取最近粉丝(第一页)。失败返回 [],并在日志里写明原因。"""
|
||||
import requests
|
||||
|
||||
sec_user_id = (sec_user_id or "").strip()
|
||||
if not sec_user_id:
|
||||
logger.warning("fetch followers skipped: 缺少本账号 sec_user_id")
|
||||
return []
|
||||
|
||||
try:
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||||
except Exception as exc:
|
||||
logger.warning("fetch followers: build auth failed: %s", exc)
|
||||
return []
|
||||
|
||||
ua = session.user_agent or DEFAULT_USER_AGENT
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"sec_user_id": sec_user_id,
|
||||
"count": str(count),
|
||||
"max_time": str(max_time),
|
||||
"min_time": "0",
|
||||
"offset": "0",
|
||||
"source_type": "1",
|
||||
"gps_access": "0",
|
||||
"address_book_access": "0",
|
||||
"is_top": "1",
|
||||
"update_version_code": "170400",
|
||||
"pc_client_type": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"cookie_enabled": "true",
|
||||
"screen_width": "1536",
|
||||
"screen_height": "960",
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Chrome",
|
||||
"browser_version": "120.0.0.0",
|
||||
"browser_online": "true",
|
||||
"os_name": "Windows",
|
||||
"os_version": "10",
|
||||
"platform": "PC",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
}
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
try:
|
||||
resp = requests.get(
|
||||
FOLLOWER_LIST_URL,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
timeout=15,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
snippet = (resp.text or "")[:200].replace("\n", " ")
|
||||
logger.warning(
|
||||
"fetch followers: 非 JSON 响应 (HTTP %s): %s", resp.status_code, snippet
|
||||
)
|
||||
return []
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("fetch followers: 响应不是 JSON 对象")
|
||||
return []
|
||||
status_code = data.get("status_code")
|
||||
if status_code not in (None, 0):
|
||||
logger.warning(
|
||||
"fetch followers: status_code=%s msg=%s",
|
||||
status_code,
|
||||
data.get("status_msg") or data.get("message") or "",
|
||||
)
|
||||
return []
|
||||
followers = _extract_followers(data)
|
||||
logger.info(
|
||||
"fetch followers ok: 拿到 %s 个粉丝 (has_more=%s total=%s)",
|
||||
len(followers),
|
||||
data.get("has_more"),
|
||||
data.get("total"),
|
||||
)
|
||||
return followers
|
||||
except Exception as exc:
|
||||
logger.warning("fetch followers failed: %s", exc)
|
||||
return []
|
||||
|
||||
|
||||
async def fetch_recent_followers(
|
||||
session,
|
||||
sec_user_id: str,
|
||||
count: int = 20,
|
||||
max_time: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
return await asyncio.to_thread(
|
||||
fetch_recent_followers_sync, session, sec_user_id, count, max_time
|
||||
)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Build frontier-im WebSocket URL (DouYin_Spider douyin_recv_msg logic)."""
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from typing import Optional
|
||||
from urllib.parse import unquote
|
||||
|
||||
import requests
|
||||
|
||||
from .auth import DouyinAuth
|
||||
from .dy_util import generate_a_bogus, generate_msToken, generate_webid, splice_url
|
||||
from .session import DouyinImSession, is_frontier_ws_url
|
||||
|
||||
logger = logging.getLogger("douyin_im.frontier")
|
||||
|
||||
APP_KEY = "e1bd35ec9db7b8d846de66ed140b1ad9"
|
||||
FP_ID = "9"
|
||||
|
||||
|
||||
def build_frontier_ws_url(session: DouyinImSession, device_id: str) -> Optional[str]:
|
||||
token = session.cookies.get("sessionid") or session.cookies.get("sessionid_ss") or ""
|
||||
if not token or not device_id:
|
||||
return None
|
||||
access_key_raw = f"{FP_ID}{APP_KEY}{device_id}f8a69f1719916z"
|
||||
access_key = hashlib.md5(access_key_raw.encode("utf-8")).hexdigest()
|
||||
params = {
|
||||
"aid": "6383",
|
||||
"device_platform": "douyin_pc",
|
||||
"fpid": FP_ID,
|
||||
"device_id": device_id,
|
||||
"token": token,
|
||||
"access_key": access_key,
|
||||
}
|
||||
query = "&".join(f"{k}={v}" for k, v in params.items())
|
||||
return f"wss://frontier-im.douyin.com/ws/v2?{query}"
|
||||
|
||||
|
||||
def fetch_device_id(session: DouyinImSession) -> str:
|
||||
"""Call Douyin user query API to obtain device/web id."""
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(
|
||||
session.cookie_header(),
|
||||
session.web_protect_str,
|
||||
session.keys_str,
|
||||
)
|
||||
url = "https://www.douyin.com/aweme/v1/web/query/user"
|
||||
headers = {
|
||||
"User-Agent": session.user_agent,
|
||||
"Referer": "https://www.douyin.com/discover",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Cookie": session.cookie_header(),
|
||||
}
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"publish_video_strategy_type": "2",
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/discover"),
|
||||
"msToken": generate_msToken(),
|
||||
}
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=session.user_agent)
|
||||
try:
|
||||
resp = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
verify=False,
|
||||
timeout=15,
|
||||
)
|
||||
data = resp.json()
|
||||
device_id = str(data.get("id") or data.get("device_id") or "")
|
||||
if device_id.isdigit():
|
||||
logger.info(f"Fetched device_id: {device_id[:20]}...")
|
||||
return device_id
|
||||
logger.warning(f"query/user returned non-numeric id: {device_id[:32]!r}")
|
||||
except Exception as e:
|
||||
logger.warning(f"fetch_device_id failed: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_frontier_device_id(session: DouyinImSession) -> str:
|
||||
"""Frontier WS requires numeric device_id from Douyin query/user API."""
|
||||
current = str(session.device_id or session.web_id or "")
|
||||
if current.isdigit():
|
||||
return current
|
||||
|
||||
fetched = fetch_device_id(session)
|
||||
if fetched and str(fetched).isdigit():
|
||||
session.device_id = str(fetched)
|
||||
logger.info(f"Using numeric device_id for frontier WS: {fetched[:16]}...")
|
||||
return str(fetched)
|
||||
|
||||
logger.warning(
|
||||
f"Invalid frontier device_id={current[:24]!r}; "
|
||||
"expected numeric id from query/user API"
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _ws_device_id(url: str) -> str:
|
||||
m = re.search(r"[?&]device_id=([^&\s]+)", url or "")
|
||||
return unquote(m.group(1)) if m else ""
|
||||
|
||||
|
||||
def _ws_device_matches_session(session: DouyinImSession, url: str) -> bool:
|
||||
ws_dev = _ws_device_id(url)
|
||||
if not ws_dev or not ws_dev.isdigit():
|
||||
return True
|
||||
for candidate in (session.my_uid, session.web_id, session.device_id):
|
||||
if candidate and str(candidate) == ws_dev:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _ws_token_looks_encoded(url: str) -> bool:
|
||||
m = re.search(r"[?&]token=([^&\s]+)", url or "")
|
||||
if not m:
|
||||
return False
|
||||
token = unquote(m.group(1))
|
||||
return len(token) >= 40 or not token.replace("_", "").replace("-", "").isalnum()
|
||||
|
||||
|
||||
def ensure_frontier_ws(session: DouyinImSession) -> Optional[str]:
|
||||
"""Ensure session has a usable frontier WebSocket URL."""
|
||||
session.sanitize_ws_urls()
|
||||
|
||||
for url in session.ws_urls:
|
||||
if is_frontier_ws_url(url) and "sdk_cert=" in url and _ws_token_looks_encoded(url):
|
||||
session.ws_urls = [url]
|
||||
logger.info("Using captured real frontier WS URL (with sdk_cert)")
|
||||
return url
|
||||
|
||||
for url in session.ws_urls:
|
||||
if is_frontier_ws_url(url) and _ws_token_looks_encoded(url):
|
||||
session.ws_urls = [url]
|
||||
logger.info("Using captured frontier WS URL")
|
||||
return url
|
||||
|
||||
device_id = resolve_frontier_device_id(session)
|
||||
if not device_id:
|
||||
session.ws_urls = []
|
||||
return None
|
||||
|
||||
built = build_frontier_ws_url(session, device_id)
|
||||
if built:
|
||||
session.ws_urls = [built]
|
||||
logger.info("Built frontier WS URL from cookie session")
|
||||
return built
|
||||
return None
|
||||
@@ -0,0 +1,38 @@
|
||||
"""本系统当前托管中的账号 UID 注册表。
|
||||
|
||||
用途:避免两个都在本系统托管的账号互相自动回复,形成无限回环——
|
||||
这种高频来回发送是触发抖音风控(7911)/业务拒绝(8004)的常见根因。
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("douyin_im.hosted_registry")
|
||||
|
||||
_HOSTED_UIDS: set[int] = set()
|
||||
|
||||
|
||||
def register(uid) -> None:
|
||||
try:
|
||||
u = int(uid)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if u:
|
||||
_HOSTED_UIDS.add(u)
|
||||
logger.info(f"Registered hosted uid {u} (total={len(_HOSTED_UIDS)})")
|
||||
|
||||
|
||||
def unregister(uid) -> None:
|
||||
try:
|
||||
u = int(uid)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
_HOSTED_UIDS.discard(u)
|
||||
logger.info(f"Unregistered hosted uid {u} (total={len(_HOSTED_UIDS)})")
|
||||
|
||||
|
||||
def is_hosted(uid) -> bool:
|
||||
try:
|
||||
u = int(uid)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return u in _HOSTED_UIDS
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
"""Parse Douyin IM Response protobuf for conversation metadata."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def extract_conv_meta_from_response_bytes(raw: bytes) -> dict[str, dict]:
|
||||
"""Return {conversation_id: {conversation_short_id, ticket}} from IM API protobuf body."""
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
from .static import Response_pb2 as ResponseProto
|
||||
|
||||
response_proto = ResponseProto.Response()
|
||||
response_proto.ParseFromString(raw)
|
||||
body = response_proto.body
|
||||
out: dict[str, dict] = {}
|
||||
for field in (
|
||||
"create_conversation_v2_body",
|
||||
"get_conversation_info_list_v2_response_body",
|
||||
):
|
||||
if not body.HasField(field):
|
||||
continue
|
||||
conv_body = getattr(body, field)
|
||||
for conv in conv_body.conversation_info_list:
|
||||
if not conv.conversation_id:
|
||||
continue
|
||||
out[conv.conversation_id] = {
|
||||
"conversation_short_id": str(conv.conversation_short_id),
|
||||
"ticket": conv.ticket,
|
||||
}
|
||||
return out
|
||||
except Exception:
|
||||
return {}
|
||||
@@ -0,0 +1,720 @@
|
||||
"""抖音 IM 私信图片上传。
|
||||
|
||||
复刻抖音 PC 网页版私信发图的真实链路(与"创作者中心 ImageX 图文图床"是两套,
|
||||
IM 发送只认这条链路产出的 tos-cn-o-* 资源):
|
||||
|
||||
1. GET www.douyin.com/aweme/v1/web/im/upload/config/v2 (a_bogus 签名)
|
||||
→ 返回内含 STS2 凭证(AccessKeyID + SignedSecretAccessKey)与 SpaceName(=zhenzhen)
|
||||
2. GET vod.bytedanceapi.com/?Action=ApplyUploadInner&SpaceName=zhenzhen&FileType=image
|
||||
(AWS4-HMAC-SHA256,service=vod,带 x-amz-security-token=STS2…)
|
||||
→ 返回 UploadHost / StoreUri(tos-cn-o-*) / Auth(SpaceKey JWT) / SessionKey
|
||||
3. POST https://{UploadHost}/upload/v1/{StoreUri}
|
||||
(Authorization: SpaceKey/zhenzhen/…JWT,Content-CRC32,X-Storage-U=my_uid)
|
||||
4. POST vod.bytedanceapi.com/?Action=CommitUploadInner&SpaceName=zhenzhen (AWS4 签名)
|
||||
→ 确认上传,最终 uri 即 StoreUri(tos-cn-o-*)
|
||||
|
||||
之后用该 tos-cn-o-* uri 构造 type=27 消息体即可被 IM 后端校验通过。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import zlib
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
logger = logging.getLogger("douyin_im.image_upload")
|
||||
|
||||
_LOCAL_URL_RE = re.compile(
|
||||
r"^(/api/media/messages/|https?://(?:localhost|127\.0\.0\.1)(?::\d+)?/api/media/messages/)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
# 抖音 web 私信图片上传专用通道(VOD/ImageX inner 接口 + IM 上传配置)
|
||||
VOD_HOST = "https://vod.bytedanceapi.com/"
|
||||
VOD_REGION = "cn-north-1"
|
||||
VOD_SERVICE = "vod"
|
||||
IM_UPLOAD_CONFIG_URL = "https://www.douyin.com/aweme/v1/web/im/upload/config/v2"
|
||||
DEFAULT_SPACE_NAME = "zhenzhen"
|
||||
|
||||
|
||||
def is_local_media_url(url: str) -> bool:
|
||||
raw = (url or "").strip()
|
||||
if not raw:
|
||||
return False
|
||||
if raw.startswith("/api/media/messages/"):
|
||||
return True
|
||||
if raw.startswith("/api/media/link-cards/"):
|
||||
return True
|
||||
return bool(_LOCAL_URL_RE.match(raw))
|
||||
|
||||
|
||||
def is_douyin_cdn_url(url: str) -> bool:
|
||||
raw = (url or "").strip().lower()
|
||||
if not raw.startswith("http"):
|
||||
return False
|
||||
return any(
|
||||
host in raw
|
||||
for host in (
|
||||
"douyinpic.com",
|
||||
"byteimg.com",
|
||||
"ibyteimg.com",
|
||||
"douyinstatic.com",
|
||||
"snssdk.com",
|
||||
"vodupload.com",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def parse_local_message_media_path(url: str) -> tuple[int, str] | None:
|
||||
"""从 /api/media/messages/{account_id}/{filename} 解析 account_id 与文件名。"""
|
||||
raw = (url or "").strip()
|
||||
m = re.search(r"/api/media/messages/(\d+)/([^/?#]+)", raw)
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)), m.group(2)
|
||||
|
||||
|
||||
def parse_link_card_media_path(url: str) -> tuple[int, str] | None:
|
||||
"""从 /api/media/link-cards/{owner_id}/{filename} 解析 owner_id 与文件名。"""
|
||||
raw = (url or "").strip()
|
||||
m = re.search(r"/api/media/link-cards/(\d+)/([^/?#]+)", raw)
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)), m.group(2)
|
||||
|
||||
|
||||
|
||||
|
||||
def _requests_proxies() -> dict | None:
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
|
||||
return requests_proxies()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _random_s() -> str:
|
||||
chars = string.digits + string.ascii_lowercase
|
||||
return "".join(random.choice(chars) for _ in range(11))
|
||||
|
||||
|
||||
def _signing_key(secret: str, date_stamp: str, region: str, service: str) -> bytes:
|
||||
k_date = hmac.new(("AWS4" + secret).encode(), date_stamp.encode(), hashlib.sha256).digest()
|
||||
k_region = hmac.new(k_date, region.encode(), hashlib.sha256).digest()
|
||||
k_service = hmac.new(k_region, service.encode(), hashlib.sha256).digest()
|
||||
return hmac.new(k_service, b"aws4_request", hashlib.sha256).digest()
|
||||
|
||||
|
||||
def _aws4_authorization(
|
||||
*,
|
||||
method: str,
|
||||
canonical_querystring: str,
|
||||
amz_date: str,
|
||||
date_stamp: str,
|
||||
session_token: str,
|
||||
access_key_id: str,
|
||||
secret_access_key: str,
|
||||
payload_hash: str | None = None,
|
||||
signed_headers: list[str] | None = None,
|
||||
service: str = VOD_SERVICE,
|
||||
region: str = VOD_REGION,
|
||||
) -> str:
|
||||
if signed_headers is None:
|
||||
signed_headers = ["x-amz-date", "x-amz-security-token"]
|
||||
if payload_hash is None:
|
||||
payload_hash = hashlib.sha256(b"").hexdigest()
|
||||
|
||||
header_lines = []
|
||||
for name in signed_headers:
|
||||
if name == "x-amz-date":
|
||||
header_lines.append(f"x-amz-date:{amz_date}\n")
|
||||
elif name == "x-amz-security-token":
|
||||
header_lines.append(f"x-amz-security-token:{session_token}\n")
|
||||
elif name == "x-amz-content-sha256":
|
||||
header_lines.append(f"x-amz-content-sha256:{payload_hash}\n")
|
||||
canonical_headers = "".join(header_lines)
|
||||
signed = ";".join(signed_headers)
|
||||
canonical_request = (
|
||||
f"{method}\n/\n{canonical_querystring}\n{canonical_headers}\n{signed}\n{payload_hash}"
|
||||
)
|
||||
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
|
||||
string_to_sign = (
|
||||
"AWS4-HMAC-SHA256\n"
|
||||
f"{amz_date}\n"
|
||||
f"{credential_scope}\n"
|
||||
f"{hashlib.sha256(canonical_request.encode()).hexdigest()}"
|
||||
)
|
||||
signature = hmac.new(
|
||||
_signing_key(secret_access_key, date_stamp, region, service),
|
||||
string_to_sign.encode(),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return (
|
||||
f"AWS4-HMAC-SHA256 Credential={access_key_id}/{credential_scope}, "
|
||||
f"SignedHeaders={signed}, Signature={signature}"
|
||||
)
|
||||
|
||||
|
||||
def _safe_json(resp) -> dict[str, Any]:
|
||||
text = (getattr(resp, "text", None) or "").strip()
|
||||
if not text:
|
||||
return {"error": f"空响应 (HTTP {getattr(resp, 'status_code', '?')})"}
|
||||
try:
|
||||
data = resp.json()
|
||||
return data if isinstance(data, dict) else {"error": "响应不是 JSON 对象"}
|
||||
except Exception:
|
||||
snippet = text[:200].replace("\n", " ")
|
||||
return {"error": f"非 JSON 响应 (HTTP {getattr(resp, 'status_code', '?')}): {snippet}"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 1 步:拉取 IM 上传配置,提取 STS 凭证 + SpaceName
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _sanitize_for_log(obj: Any, depth: int = 0) -> Any:
|
||||
"""结构化脱敏:保留字段名与层级,长字符串只留前 12 位 + 长度,便于排查凭证字段。"""
|
||||
if depth > 6:
|
||||
return "…"
|
||||
if isinstance(obj, dict):
|
||||
return {str(k): _sanitize_for_log(v, depth + 1) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_sanitize_for_log(v, depth + 1) for v in obj[:3]]
|
||||
if isinstance(obj, str):
|
||||
return f"{obj[:12]}…(len={len(obj)})" if len(obj) > 24 else obj
|
||||
return obj
|
||||
|
||||
|
||||
def _find_auth_object(obj: Any, depth: int = 0) -> dict | None:
|
||||
"""找到「直接含有 STS2 会话凭证字符串」的那个 dict(即临时凭证对象)。
|
||||
|
||||
该对象里通常同时含有 AccessKeyId / SecretAccessKey / SessionToken(=STS2…)。
|
||||
关键:签名要用同级的 SecretAccessKey 字段,而**不是** STS2 令牌内部解出的
|
||||
SignedSecretAccessKey(那是服务端校验用的,拿来当签名密钥会 SignatureDoesNotMatch)。
|
||||
"""
|
||||
if depth > 8 or not isinstance(obj, (dict, list)):
|
||||
return None
|
||||
if isinstance(obj, dict):
|
||||
for value in obj.values():
|
||||
if isinstance(value, str) and value.startswith("STS2"):
|
||||
return obj
|
||||
for value in obj.values():
|
||||
found = _find_auth_object(value, depth + 1)
|
||||
if found is not None:
|
||||
return found
|
||||
else:
|
||||
for value in obj:
|
||||
found = _find_auth_object(value, depth + 1)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _find_sts_token(obj: Any, depth: int = 0) -> str:
|
||||
"""递归在响应里找到形如 'STS2...' 的会话凭证字符串。"""
|
||||
if depth > 8:
|
||||
return ""
|
||||
if isinstance(obj, str):
|
||||
return obj if obj.startswith("STS2") else ""
|
||||
if isinstance(obj, dict):
|
||||
for value in obj.values():
|
||||
found = _find_sts_token(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
found = _find_sts_token(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
return ""
|
||||
|
||||
|
||||
def _find_space_name(obj: Any, depth: int = 0) -> str:
|
||||
if depth > 8:
|
||||
return ""
|
||||
if isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
if str(key).lower() in ("space_name", "spacename") and isinstance(value, str) and value:
|
||||
return value
|
||||
found = _find_space_name(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
found = _find_space_name(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
return ""
|
||||
|
||||
|
||||
def _decode_sts(sts_token: str) -> tuple[str, str]:
|
||||
"""STS2<base64(JSON)>,解出 AccessKeyID 与 SignedSecretAccessKey。"""
|
||||
try:
|
||||
b64 = sts_token[4:] if sts_token.startswith("STS2") else sts_token
|
||||
b64 += "=" * (-len(b64) % 4)
|
||||
data = json.loads(base64.b64decode(b64).decode("utf-8", "ignore"))
|
||||
ak = data.get("AccessKeyID") or data.get("AccessKeyId") or ""
|
||||
sk = data.get("SignedSecretAccessKey") or data.get("SecretAccessKey") or ""
|
||||
return ak, sk
|
||||
except Exception:
|
||||
return "", ""
|
||||
|
||||
|
||||
def _fetch_im_upload_sts(session) -> tuple[str, str, str, str]:
|
||||
"""返回 (access_key_id, secret_access_key, sts_token, space_name)。"""
|
||||
import requests
|
||||
|
||||
from .auth import DouyinAuth
|
||||
from .dy_util import (
|
||||
DEFAULT_USER_AGENT,
|
||||
generate_a_bogus,
|
||||
generate_msToken,
|
||||
generate_webid,
|
||||
splice_url,
|
||||
)
|
||||
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||||
ua = session.user_agent or DEFAULT_USER_AGENT
|
||||
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"update_version_code": "170400",
|
||||
"pc_client_type": "1",
|
||||
"pc_libra_divert": "Windows",
|
||||
"support_h265": "1",
|
||||
"support_dash": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"cookie_enabled": "true",
|
||||
"screen_width": "1536",
|
||||
"screen_height": "960",
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Chrome",
|
||||
"browser_version": "120.0.0.0",
|
||||
"browser_online": "true",
|
||||
"engine_name": "Blink",
|
||||
"engine_version": "120.0.0.0",
|
||||
"os_name": "Windows",
|
||||
"os_version": "10",
|
||||
"cpu_core_num": "8",
|
||||
"device_memory": "8",
|
||||
"platform": "PC",
|
||||
"downlink": "10",
|
||||
"effective_type": "4g",
|
||||
"round_trip_time": "50",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
}
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
resp = requests.get(
|
||||
IM_UPLOAD_CONFIG_URL,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
timeout=20,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"获取 IM 上传配置失败:{data['error']}")
|
||||
if data.get("status_code") not in (None, 0):
|
||||
raise RuntimeError(
|
||||
f"获取 IM 上传配置失败:status_code={data.get('status_code')} "
|
||||
f"{data.get('status_msg') or ''}"
|
||||
)
|
||||
|
||||
# 诊断:把 config/v2 响应结构(字段名保留、长字符串脱敏)打到日志,便于核对凭证字段。
|
||||
try:
|
||||
logger.info(
|
||||
"im/upload/config/v2 结构: %s",
|
||||
json.dumps(_sanitize_for_log(data), ensure_ascii=False),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 凭证块选择(决定 8003 是否发生的关键):
|
||||
# - inner_image_config / public_image_config / public_file_config 共用同一套
|
||||
# 可用于 VOD 上传的 STS 令牌,仅 space 不同;
|
||||
# - public_image_config_v2 的 token 不同,不兼容 VOD 内部上传(会报
|
||||
# "session token sequence is broken"),必须避开。
|
||||
# 空间含义:maya_review = 审核空间,图片上传后处于待审核态,作为消息发送会被拒(8003);
|
||||
# zhenzhen = 公开可发送空间,IM 图片消息应使用它。
|
||||
# 因此优先 public_image_config(zhenzhen),其凭证同样能完成 VOD 上传。
|
||||
def _block_has_sts(block: Any) -> bool:
|
||||
return isinstance(block, dict) and any(
|
||||
isinstance(v, str) and v.startswith("STS2") for v in block.values()
|
||||
)
|
||||
|
||||
auth_obj = None
|
||||
for _key in ("public_image_config", "inner_image_config"):
|
||||
if isinstance(data, dict) and _block_has_sts(data.get(_key)):
|
||||
auth_obj = data[_key]
|
||||
break
|
||||
if auth_obj is None:
|
||||
auth_obj = _find_auth_object(data)
|
||||
if not auth_obj:
|
||||
raise RuntimeError(
|
||||
"IM 上传配置响应里未找到 STS 凭证(STS2 token);可能 cookie/签名失效,请用浏览器模式重新登录"
|
||||
)
|
||||
|
||||
sts = next(
|
||||
(v for v in auth_obj.values() if isinstance(v, str) and v.startswith("STS2")),
|
||||
"",
|
||||
)
|
||||
# 优先用凭证对象同级的 AccessKeyId / SecretAccessKey(用于 SigV4 签名的真实密钥)。
|
||||
ak = (
|
||||
auth_obj.get("AccessKeyID")
|
||||
or auth_obj.get("AccessKeyId")
|
||||
or auth_obj.get("access_key_id")
|
||||
or ""
|
||||
)
|
||||
sk = (
|
||||
auth_obj.get("SecretAccessKey")
|
||||
or auth_obj.get("SecretAccesskey")
|
||||
or auth_obj.get("secret_access_key")
|
||||
or ""
|
||||
)
|
||||
# 兜底:若响应没给独立的 ak/sk,再尝试从 STS2 令牌解码(SignedSecretAccessKey 一般不可用,仅最后兜底)。
|
||||
if not ak or not sk:
|
||||
dec_ak, dec_sk = _decode_sts(sts)
|
||||
ak = ak or dec_ak
|
||||
sk = sk or dec_sk
|
||||
if not ak or not sk:
|
||||
raise RuntimeError("解析 STS 凭证失败(缺少 AccessKeyId / SecretAccessKey)")
|
||||
# space 必须取自与凭证同一块,避免凭证用 inner_image_config 而 space 误取到别处。
|
||||
space = (
|
||||
auth_obj.get("space_name")
|
||||
or auth_obj.get("SpaceName")
|
||||
or auth_obj.get("spaceName")
|
||||
or _find_space_name(data)
|
||||
or DEFAULT_SPACE_NAME
|
||||
)
|
||||
return ak, sk, sts, space
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 2 步:ApplyUploadInner(VOD),申请上传地址
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_apply_inner(data: dict[str, Any]) -> tuple[str, str, str, str]:
|
||||
"""返回 (upload_host, store_uri, jwt_auth, session_key)。"""
|
||||
result = data.get("Result") or {}
|
||||
addr = result.get("InnerUploadAddress") or result.get("UploadAddress") or {}
|
||||
|
||||
nodes = addr.get("UploadNodes") or []
|
||||
if nodes:
|
||||
node = nodes[0]
|
||||
stores = node.get("StoreInfos") or []
|
||||
store = stores[0] if stores else {}
|
||||
host = node.get("UploadHost") or ""
|
||||
if not host:
|
||||
hosts = node.get("UploadHosts") or addr.get("UploadHosts") or []
|
||||
host = hosts[0] if hosts else ""
|
||||
return (
|
||||
str(host or ""),
|
||||
str(store.get("StoreUri") or ""),
|
||||
str(store.get("Auth") or ""),
|
||||
str(node.get("SessionKey") or addr.get("SessionKey") or ""),
|
||||
)
|
||||
|
||||
hosts = addr.get("UploadHosts") or []
|
||||
host = hosts[0] if hosts else ""
|
||||
stores = addr.get("StoreInfos") or []
|
||||
store = stores[0] if stores else {}
|
||||
return (
|
||||
str(host or ""),
|
||||
str(store.get("StoreUri") or ""),
|
||||
str(store.get("Auth") or ""),
|
||||
str(addr.get("SessionKey") or result.get("SessionKey") or ""),
|
||||
)
|
||||
|
||||
|
||||
def _vod_apply_upload_inner(
|
||||
ak: str, sk: str, token: str, space: str, file_size: int
|
||||
) -> tuple[str, str, str, str]:
|
||||
import requests
|
||||
|
||||
from .dy_util import DEFAULT_USER_AGENT
|
||||
|
||||
now = datetime.datetime.utcnow()
|
||||
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
|
||||
date_stamp = now.strftime("%Y%m%d")
|
||||
params = {
|
||||
"Action": "ApplyUploadInner",
|
||||
"Version": "2020-11-19",
|
||||
"SpaceName": space,
|
||||
"FileType": "image",
|
||||
"IsInner": "1",
|
||||
"NeedFallback": "true",
|
||||
"FileSize": str(file_size),
|
||||
"s": _random_s(),
|
||||
}
|
||||
qs = urlencode(sorted(params.items()))
|
||||
authorization = _aws4_authorization(
|
||||
method="GET",
|
||||
canonical_querystring=qs,
|
||||
amz_date=amz_date,
|
||||
date_stamp=date_stamp,
|
||||
session_token=token,
|
||||
access_key_id=ak,
|
||||
secret_access_key=sk,
|
||||
service=VOD_SERVICE,
|
||||
)
|
||||
resp = requests.get(
|
||||
f"{VOD_HOST}?{qs}",
|
||||
headers={
|
||||
"accept": "*/*",
|
||||
"authorization": authorization,
|
||||
"user-agent": DEFAULT_USER_AGENT,
|
||||
"x-amz-date": amz_date,
|
||||
"x-amz-security-token": token,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
},
|
||||
timeout=30,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"申请上传地址失败:{data['error']}")
|
||||
meta = data.get("ResponseMetadata") or {}
|
||||
err = meta.get("Error")
|
||||
if err:
|
||||
raise RuntimeError(f"申请上传地址失败:{err.get('Message') or err.get('Code')}")
|
||||
return _extract_apply_inner(data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 3 步:上传二进制(SpaceKey JWT 鉴权)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _vod_upload_binary(
|
||||
host: str, store_uri: str, jwt_auth: str, user_id: str, raw: bytes, session=None
|
||||
) -> None:
|
||||
import requests
|
||||
|
||||
from .dy_util import DEFAULT_USER_AGENT
|
||||
|
||||
crc32 = format(zlib.crc32(raw) & 0xFFFFFFFF, "08x")
|
||||
url = f"https://{host}/upload/v1/{store_uri}"
|
||||
ua = (getattr(session, "user_agent", None) or DEFAULT_USER_AGENT)
|
||||
headers = {
|
||||
"Authorization": jwt_auth,
|
||||
"Content-CRC32": crc32,
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Disposition": 'attachment; filename="undefined"',
|
||||
"User-Agent": ua,
|
||||
}
|
||||
if user_id:
|
||||
headers["X-Storage-U"] = str(user_id)
|
||||
resp = requests.post(
|
||||
url,
|
||||
headers=headers,
|
||||
data=raw,
|
||||
timeout=60,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"上传图片数据失败:{data['error']}")
|
||||
if data.get("code") not in (2000, 0, None):
|
||||
raise RuntimeError(f"上传图片数据失败:{data.get('message') or data}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 4 步:CommitUploadInner(VOD),确认上传
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _vod_commit_upload_inner(
|
||||
ak: str, sk: str, token: str, space: str, session_key: str
|
||||
) -> dict[str, Any]:
|
||||
import requests
|
||||
|
||||
from .dy_util import DEFAULT_USER_AGENT
|
||||
|
||||
now = datetime.datetime.utcnow()
|
||||
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
|
||||
date_stamp = now.strftime("%Y%m%d")
|
||||
params = {
|
||||
"Action": "CommitUploadInner",
|
||||
"Version": "2020-11-19",
|
||||
"SpaceName": space,
|
||||
}
|
||||
qs = urlencode(sorted(params.items()))
|
||||
body = json.dumps({"SessionKey": session_key, "Functions": []}, separators=(",", ":"))
|
||||
payload_hash = hashlib.sha256(body.encode()).hexdigest()
|
||||
signed_headers = ["x-amz-content-sha256", "x-amz-date", "x-amz-security-token"]
|
||||
authorization = _aws4_authorization(
|
||||
method="POST",
|
||||
canonical_querystring=qs,
|
||||
amz_date=amz_date,
|
||||
date_stamp=date_stamp,
|
||||
session_token=token,
|
||||
access_key_id=ak,
|
||||
secret_access_key=sk,
|
||||
payload_hash=payload_hash,
|
||||
signed_headers=signed_headers,
|
||||
service=VOD_SERVICE,
|
||||
)
|
||||
resp = requests.post(
|
||||
f"{VOD_HOST}?{qs}",
|
||||
data=body,
|
||||
headers={
|
||||
"accept": "*/*",
|
||||
"authorization": authorization,
|
||||
"content-type": "application/json",
|
||||
"user-agent": DEFAULT_USER_AGENT,
|
||||
"x-amz-content-sha256": payload_hash,
|
||||
"x-amz-date": amz_date,
|
||||
"x-amz-security-token": token,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
},
|
||||
timeout=30,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"确认上传失败:{data['error']}")
|
||||
meta = data.get("ResponseMetadata") or {}
|
||||
err = meta.get("Error")
|
||||
if err:
|
||||
raise RuntimeError(f"确认上传失败:{err.get('Message') or err.get('Code')}")
|
||||
return data.get("Result") or {}
|
||||
|
||||
|
||||
def upload_im_image(
|
||||
session,
|
||||
raw: bytes,
|
||||
*,
|
||||
filename: str = "image.jpg",
|
||||
content_type: str = "image/jpeg",
|
||||
) -> dict[str, Any]:
|
||||
"""上传图片到抖音 IM 私信图床(VOD/zhenzhen 空间)。
|
||||
|
||||
成功返回 {uri(tos-cn-o-*), url, url_list, md5};失败返回 {"error": "..."}。
|
||||
"""
|
||||
del filename, content_type # VOD 按二进制上传,文件名仅用于本地存储
|
||||
if not raw:
|
||||
return {"error": "图片为空"}
|
||||
try:
|
||||
ak, sk, token, space = _fetch_im_upload_sts(session)
|
||||
host, store_uri, jwt_auth, session_key = _vod_apply_upload_inner(
|
||||
ak, sk, token, space, len(raw)
|
||||
)
|
||||
if not host or not store_uri or not jwt_auth:
|
||||
return {"error": "申请上传地址失败:缺少 UploadHost/StoreUri/Auth"}
|
||||
|
||||
user_id = str(getattr(session, "my_uid", "") or "")
|
||||
_vod_upload_binary(host, store_uri, jwt_auth, user_id, raw, session)
|
||||
_vod_commit_upload_inner(ak, sk, token, space, session_key)
|
||||
|
||||
uri = store_uri.lstrip("/")
|
||||
out: dict[str, Any] = {"uri": uri, "md5": hashlib.md5(raw).hexdigest()}
|
||||
|
||||
from .message_content import uri_to_cdn_urls
|
||||
|
||||
urls = uri_to_cdn_urls(uri)
|
||||
if urls:
|
||||
out["url_list"] = urls
|
||||
out["url"] = urls[0]
|
||||
|
||||
logger.info("Uploaded IM image via VOD uri=%s host=%s space=%s", uri, host, space)
|
||||
return out
|
||||
except Exception as exc:
|
||||
logger.warning("upload_im_image (VOD) failed: %s", exc)
|
||||
return {"error": str(exc)}
|
||||
|
||||
|
||||
def prepare_image_reply_spec(spec: dict[str, Any], session, upload_dir: str) -> tuple[dict[str, Any], str]:
|
||||
"""若图片仍是本地地址,则上传到抖音 CDN 并补全 uri。返回 (spec, error)。"""
|
||||
if spec.get("type") != "image":
|
||||
return spec, ""
|
||||
|
||||
uri = str(spec.get("uri") or "").strip()
|
||||
url = str(spec.get("url") or "").strip()
|
||||
|
||||
# 已经持有抖音 CDN 的 uri,说明图片早已上传完成(且经历过风控/转码),直接复用即可。
|
||||
# 关键修复:不要因为 url 仍是本机预览地址(/api/media/...)而再次上传——
|
||||
# 重复上传会拿到一个“刚提交、尚未完成风控/转码”的新 uri,发送时常被抖音以
|
||||
# raw_check_code=1 / status_code=8003 拒绝(与是否互关无关)。
|
||||
# 发送链路(build_msg_payload)只用 uri / url_list,从不使用这个本机 url,故本机 url 无害。
|
||||
if uri and not is_local_media_url(uri):
|
||||
return spec, ""
|
||||
|
||||
if url and is_douyin_cdn_url(url) and not uri:
|
||||
return spec, ""
|
||||
|
||||
raw: bytes | None = None
|
||||
filename = "image.jpg"
|
||||
content_type = "image/jpeg"
|
||||
|
||||
if is_local_media_url(url):
|
||||
import os
|
||||
|
||||
card_parsed = parse_link_card_media_path(url)
|
||||
if card_parsed:
|
||||
# 卡片封面在 uploads/link-cards/{owner}/ 下,与消息图片目录(uploads/messages)不同。
|
||||
owner_id, fname = card_parsed
|
||||
link_cards_dir = os.path.join(os.path.dirname(upload_dir), "link-cards")
|
||||
path = os.path.join(link_cards_dir, str(owner_id), fname)
|
||||
else:
|
||||
parsed = parse_local_message_media_path(url)
|
||||
if not parsed:
|
||||
return spec, "无法解析本地图片路径"
|
||||
_, fname = parsed
|
||||
path = os.path.join(upload_dir, str(parsed[0]), fname)
|
||||
if not os.path.isfile(path):
|
||||
return spec, f"本地图片不存在:{fname}"
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
filename = fname
|
||||
if fname.lower().endswith(".png"):
|
||||
content_type = "image/png"
|
||||
elif fname.lower().endswith(".webp"):
|
||||
content_type = "image/webp"
|
||||
elif fname.lower().endswith(".gif"):
|
||||
content_type = "image/gif"
|
||||
|
||||
if raw is None:
|
||||
if url.startswith("http") and not is_douyin_cdn_url(url):
|
||||
return spec, "图片地址必须是抖音 CDN 或本地上传后的地址,外部 URL 无法用于 IM 发送"
|
||||
return spec, "缺少可上传的图片数据"
|
||||
|
||||
uploaded = upload_im_image(session, raw, filename=filename, content_type=content_type)
|
||||
if uploaded.get("error"):
|
||||
return spec, uploaded["error"]
|
||||
if not uploaded.get("uri"):
|
||||
return spec, "抖音图片上传失败:未返回 uri"
|
||||
|
||||
merged = {
|
||||
**spec,
|
||||
**uploaded,
|
||||
"type": "image",
|
||||
"text": spec.get("text") or "[图片]",
|
||||
}
|
||||
return merged, ""
|
||||
@@ -0,0 +1,746 @@
|
||||
"""私信消息内容解析、存储与展示(文本 / 图片 / 表情 / 语音 / 视频)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
MSG_TYPE_TEXT = 7
|
||||
MSG_TYPE_STICKER = 5
|
||||
MSG_TYPE_VOICE = 17
|
||||
MSG_TYPE_IMAGE = 27
|
||||
MSG_TYPE_VIDEO = 8
|
||||
MSG_TYPE_LINK_CARD = 70
|
||||
|
||||
_TYPE_LABELS = {
|
||||
"text": "文本",
|
||||
"image": "图片",
|
||||
"sticker": "表情",
|
||||
"voice": "语音",
|
||||
"video": "视频",
|
||||
"link": "链接",
|
||||
"link_card": "链接卡片",
|
||||
}
|
||||
|
||||
_PLACEHOLDER_MARKERS = {
|
||||
"[表情包]",
|
||||
"[语音]",
|
||||
"[图片]",
|
||||
"[视频]",
|
||||
"[未读消息]",
|
||||
}
|
||||
|
||||
_URI_HINT_RE = re.compile(
|
||||
r"(tos-cn|aweme-|voice/|ies-music|\.mp3|\.m4a|\.aac|\.mpeg|\.webp|\.jpeg|\.jpg|\.png|\.gif)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
_AUDIO_URL_RE = re.compile(
|
||||
r"(douyin-user-audio|/audio/|sc=audio|voice/|ies-music|\.mp3|\.m4a|\.aac|\.mpeg|\.wav|\.ogg)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
_VIDEO_URL_RE = re.compile(
|
||||
r"(sc=video|/video/|\.mp4|\.mov|\.webm|\.m3u8)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
_MEDIA_STRING_KEYS = (
|
||||
"url",
|
||||
"uri",
|
||||
"main_url",
|
||||
"download_url",
|
||||
"remote_url",
|
||||
"encrypt_url",
|
||||
"play_url",
|
||||
"secret_url",
|
||||
"audio_url",
|
||||
"video_url",
|
||||
"cover_url",
|
||||
"local_path",
|
||||
)
|
||||
|
||||
_MEDIA_NESTED_KEYS = (
|
||||
"resource_url",
|
||||
"static_url",
|
||||
"animate_url",
|
||||
"cover_url",
|
||||
"thumb_url",
|
||||
"origin_url",
|
||||
"image",
|
||||
"picture",
|
||||
"pic",
|
||||
"sticker",
|
||||
"emoji",
|
||||
"audio",
|
||||
"voice",
|
||||
"video",
|
||||
"media",
|
||||
"large_url",
|
||||
"medium_url",
|
||||
"thumb",
|
||||
"avatar_thumb",
|
||||
"play_url",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_uri_path(raw: str) -> str:
|
||||
path = (raw or "").strip().lstrip("/")
|
||||
if path.startswith("obj/"):
|
||||
path = path[4:]
|
||||
return path
|
||||
|
||||
|
||||
def uri_to_cdn_urls(uri: str, *, prefer_voice: bool = False) -> list[str]:
|
||||
"""将抖音 IM 中的 uri / tos 路径转为可访问的 CDN URL 候选列表。"""
|
||||
raw = (uri or "").strip()
|
||||
if not raw:
|
||||
return []
|
||||
if raw.startswith("//"):
|
||||
return [f"https:{raw}"]
|
||||
if raw.startswith("http://") or raw.startswith("https://"):
|
||||
return [raw]
|
||||
|
||||
path = _normalize_uri_path(raw)
|
||||
if not path:
|
||||
return []
|
||||
|
||||
candidates: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(url: str) -> None:
|
||||
url = (url or "").strip()
|
||||
if url and url not in seen:
|
||||
seen.add(url)
|
||||
candidates.append(url)
|
||||
|
||||
lower = path.lower()
|
||||
is_voice = prefer_voice or lower.startswith("voice/") or lower.endswith((".mp3", ".m4a", ".aac"))
|
||||
is_image = (
|
||||
not is_voice
|
||||
and (
|
||||
"tos-cn-i" in lower
|
||||
or "aweme-" in lower
|
||||
or lower.endswith((".jpeg", ".jpg", ".png", ".webp", ".gif"))
|
||||
)
|
||||
)
|
||||
|
||||
if is_voice:
|
||||
for host in (
|
||||
"sf6-cdn-tos.douyinstatic.com",
|
||||
"sf3-cdn-tos.douyinstatic.com",
|
||||
"lf3-static.bytednsdoc.com",
|
||||
):
|
||||
add(f"https://{host}/obj/{path}")
|
||||
add(f"https://p3.douyinpic.com/obj/{path}")
|
||||
|
||||
if is_image or "tos-cn" in lower or "aweme" in lower:
|
||||
add(f"https://p3.douyinpic.com/obj/{path}")
|
||||
for size in ("720x720", "480x480", "300x300", "200x200", "100x100"):
|
||||
add(f"https://p3.douyinpic.com/aweme/{size}/{path}")
|
||||
add(f"https://p9-dy.byteimg.com/img/{path}")
|
||||
add(f"https://p6-dy.byteimg.com/img/{path}")
|
||||
|
||||
add(f"https://p3.douyinpic.com/obj/{path}")
|
||||
add(f"https://p3-sign.douyinpic.com/obj/{path}".replace("-sign", ""))
|
||||
return candidates
|
||||
|
||||
|
||||
def resolve_media_uri(uri: str, *, prefer_voice: bool = False) -> str:
|
||||
urls = uri_to_cdn_urls(uri, prefer_voice=prefer_voice)
|
||||
return urls[0] if urls else ""
|
||||
|
||||
|
||||
def _looks_like_audio_url(value: str) -> bool:
|
||||
raw = (value or "").strip()
|
||||
return bool(raw and _AUDIO_URL_RE.search(raw))
|
||||
|
||||
|
||||
def _looks_like_video_url(value: str) -> bool:
|
||||
raw = (value or "").strip()
|
||||
return bool(raw and _VIDEO_URL_RE.search(raw))
|
||||
|
||||
|
||||
def _infer_media_type_from_url(url: str) -> str:
|
||||
"""根据 URL 特征推断媒体类型(语音/视频/图片)。"""
|
||||
if _looks_like_audio_url(url):
|
||||
return "voice"
|
||||
if _looks_like_video_url(url):
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
|
||||
def _looks_like_media_uri(value: str) -> bool:
|
||||
raw = (value or "").strip()
|
||||
if not raw or raw.startswith("{"):
|
||||
return False
|
||||
if raw.startswith("http://") or raw.startswith("https://") or raw.startswith("//"):
|
||||
return True
|
||||
return bool(_URI_HINT_RE.search(raw))
|
||||
|
||||
|
||||
def _resolve_string_media(value: str, *, prefer_voice: bool = False) -> tuple[str, str]:
|
||||
raw = (value or "").strip()
|
||||
if not raw:
|
||||
return "", ""
|
||||
if raw.startswith("//"):
|
||||
return f"https:{raw}", raw
|
||||
if raw.startswith("http://") or raw.startswith("https://"):
|
||||
return raw, ""
|
||||
if _looks_like_media_uri(raw):
|
||||
return resolve_media_uri(raw, prefer_voice=prefer_voice), raw
|
||||
return "", ""
|
||||
|
||||
|
||||
def _valid_sticker_id(value: Any) -> str:
|
||||
if value is None or value == "":
|
||||
return ""
|
||||
try:
|
||||
if int(value) == 0:
|
||||
return ""
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
sticker_id = str(value).strip()
|
||||
return "" if sticker_id in ("0", "null", "None") else sticker_id
|
||||
|
||||
|
||||
def _collect_media_candidates(value: Any, out: list[str], *, prefer_voice: bool = False, depth: int = 0) -> None:
|
||||
if depth > 12:
|
||||
return
|
||||
if isinstance(value, str):
|
||||
raw = value.strip()
|
||||
if raw.startswith("http://") or raw.startswith("https://") or raw.startswith("//"):
|
||||
url, _ = _resolve_string_media(raw, prefer_voice=prefer_voice)
|
||||
if url:
|
||||
out.append(url)
|
||||
return
|
||||
if isinstance(value, dict):
|
||||
for list_key in ("url_list", "urls", "urlList"):
|
||||
urls = value.get(list_key)
|
||||
if isinstance(urls, list):
|
||||
for item in urls:
|
||||
_collect_media_candidates(item, out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
for key in _MEDIA_STRING_KEYS:
|
||||
direct = value.get(key)
|
||||
if isinstance(direct, str):
|
||||
url, _ = _resolve_string_media(direct, prefer_voice=prefer_voice)
|
||||
if url:
|
||||
out.append(url)
|
||||
for nested_key in _MEDIA_NESTED_KEYS:
|
||||
_collect_media_candidates(value.get(nested_key), out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
for nested in value.values():
|
||||
if isinstance(nested, (dict, list)):
|
||||
_collect_media_candidates(nested, out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
_collect_media_candidates(item, out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
|
||||
|
||||
def _pick_http_url(value: Any, *, prefer_voice: bool = False) -> str:
|
||||
candidates: list[str] = []
|
||||
_collect_media_candidates(value, candidates, prefer_voice=prefer_voice)
|
||||
return candidates[0] if candidates else ""
|
||||
|
||||
|
||||
def _pick_media_uri(value: Any) -> str:
|
||||
if isinstance(value, str) and _looks_like_media_uri(value) and not value.strip().startswith("http"):
|
||||
return _normalize_uri_path(value)
|
||||
if isinstance(value, dict):
|
||||
for key in ("uri", "local_path", "remote_url"):
|
||||
direct = value.get(key)
|
||||
if isinstance(direct, str) and _looks_like_media_uri(direct) and not direct.strip().startswith("http"):
|
||||
return _normalize_uri_path(direct)
|
||||
for nested_key in _MEDIA_NESTED_KEYS:
|
||||
uri = _pick_media_uri(value.get(nested_key))
|
||||
if uri:
|
||||
return uri
|
||||
for nested in value.values():
|
||||
if isinstance(nested, (dict, list)):
|
||||
uri = _pick_media_uri(nested)
|
||||
if uri:
|
||||
return uri
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
uri = _pick_media_uri(item)
|
||||
if uri:
|
||||
return uri
|
||||
return ""
|
||||
|
||||
|
||||
def _coerce_message_type(value: Any, default: int = MSG_TYPE_TEXT) -> int:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _parse_content_json(content_raw: str | dict | None) -> dict[str, Any]:
|
||||
if isinstance(content_raw, dict):
|
||||
data = content_raw
|
||||
else:
|
||||
raw = str(content_raw or "").strip()
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, dict):
|
||||
data = parsed
|
||||
else:
|
||||
return {}
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
else:
|
||||
return {}
|
||||
|
||||
for nested_key in ("ext", "ai_ext", "extra", "payload", "data"):
|
||||
nested = data.get(nested_key)
|
||||
if isinstance(nested, str) and nested.strip().startswith("{"):
|
||||
try:
|
||||
nested_data = json.loads(nested)
|
||||
if isinstance(nested_data, dict):
|
||||
merged = {**nested_data, **data}
|
||||
data = merged
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def format_im_message(content_raw: str | dict | None, message_type: int = MSG_TYPE_TEXT) -> dict[str, Any]:
|
||||
"""将 IM 原始 content 解析为统一结构 {type, text, url, ...}。"""
|
||||
content_json = _parse_content_json(content_raw)
|
||||
embedded_type = _coerce_message_type(
|
||||
content_json.get("message_type")
|
||||
or content_json.get("messageType")
|
||||
or content_json.get("msg_type"),
|
||||
message_type,
|
||||
)
|
||||
if embedded_type != MSG_TYPE_TEXT:
|
||||
message_type = embedded_type
|
||||
|
||||
_EMPTY_MEDIA_DEFAULTS = {
|
||||
MSG_TYPE_IMAGE: ("image", "[图片]"),
|
||||
MSG_TYPE_STICKER: ("sticker", "[表情包]"),
|
||||
MSG_TYPE_VOICE: ("voice", "[语音]"),
|
||||
MSG_TYPE_VIDEO: ("video", "[视频]"),
|
||||
MSG_TYPE_LINK_CARD: ("link_card", "[链接卡片]"),
|
||||
}
|
||||
|
||||
if not content_json:
|
||||
raw = str(content_raw or "").strip()
|
||||
if not raw:
|
||||
# 抖音相册图片(type 27)/部分语音等会以「空 content」推送,URL 不随推送下发。
|
||||
# 不能直接丢弃,否则消息「收不到」;这里按类型返回占位,URL 留空待后续补取。
|
||||
if message_type in _EMPTY_MEDIA_DEFAULTS:
|
||||
t, txt = _EMPTY_MEDIA_DEFAULTS[message_type]
|
||||
return {"type": t, "text": txt}
|
||||
return {"type": "text", "text": ""}
|
||||
if raw in _PLACEHOLDER_MARKERS:
|
||||
mapping = {
|
||||
"[图片]": "image",
|
||||
"[表情包]": "sticker",
|
||||
"[语音]": "voice",
|
||||
"[视频]": "video",
|
||||
}
|
||||
return {"type": mapping.get(raw, "text"), "text": raw}
|
||||
if message_type == MSG_TYPE_TEXT:
|
||||
return {"type": "text", "text": raw}
|
||||
content_json = {"text": raw}
|
||||
|
||||
prefer_voice = message_type == MSG_TYPE_VOICE or bool(content_json.get("audio") or content_json.get("voice"))
|
||||
url = _pick_http_url(content_json, prefer_voice=prefer_voice)
|
||||
media_uri = _pick_media_uri(content_json)
|
||||
if not url and media_uri:
|
||||
url = resolve_media_uri(media_uri, prefer_voice=prefer_voice)
|
||||
duration = _safe_int(
|
||||
content_json.get("duration")
|
||||
or content_json.get("audio_duration")
|
||||
or content_json.get("video_duration")
|
||||
)
|
||||
width = _safe_int(content_json.get("width") or content_json.get("w"))
|
||||
height = _safe_int(content_json.get("height") or content_json.get("h"))
|
||||
|
||||
def _media_payload(msg_type: str, text: str, **extra: Any) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {"type": msg_type, "text": text}
|
||||
if url:
|
||||
payload["url"] = url
|
||||
elif media_uri:
|
||||
payload["uri"] = media_uri
|
||||
payload.update({k: v for k, v in extra.items() if v not in (None, "", 0)})
|
||||
return payload
|
||||
|
||||
# 语音/视频也会带 resource_url,不能仅凭该字段判为图片;优先按类型与 URL 特征识别。
|
||||
is_voice = (
|
||||
message_type == MSG_TYPE_VOICE
|
||||
or bool(content_json.get("audio") or content_json.get("voice"))
|
||||
or (url and _looks_like_audio_url(url))
|
||||
)
|
||||
is_video = (
|
||||
message_type == MSG_TYPE_VIDEO
|
||||
or bool(content_json.get("video"))
|
||||
or (url and _looks_like_video_url(url))
|
||||
)
|
||||
if is_voice and not is_video:
|
||||
return _media_payload("voice", "[语音]", duration=duration)
|
||||
if is_video:
|
||||
return _media_payload("video", "[视频]", duration=duration, width=width, height=height)
|
||||
|
||||
has_image_hint = (
|
||||
message_type == MSG_TYPE_IMAGE
|
||||
or content_json.get("image")
|
||||
or content_json.get("inline_pic")
|
||||
or (
|
||||
content_json.get("resource_url")
|
||||
and not (url and (_looks_like_audio_url(url) or _looks_like_video_url(url)))
|
||||
)
|
||||
)
|
||||
if has_image_hint:
|
||||
# 抖音相册私图(biz_tag=aweme_im)的大图 URL 是加密内容,浏览器无法直接渲染;
|
||||
# 但 content 内嵌 inline_pic(base64 WEBP 缩略图),直接转 data URI 即可显示。
|
||||
inline = content_json.get("inline_pic")
|
||||
if isinstance(inline, str) and inline.strip():
|
||||
b64 = re.sub(r"\s+", "", inline)
|
||||
data_uri = f"data:image/webp;base64,{b64}"
|
||||
payload = {"type": "image", "text": "[图片]", "url": data_uri}
|
||||
if width:
|
||||
payload["width"] = width
|
||||
if height:
|
||||
payload["height"] = height
|
||||
return payload
|
||||
return _media_payload("image", "[图片]", width=width, height=height)
|
||||
if message_type == MSG_TYPE_STICKER or content_json.get("static_url") or content_json.get("animate_url") or _valid_sticker_id(
|
||||
content_json.get("sticker_id") or content_json.get("id")
|
||||
):
|
||||
sticker_id = _valid_sticker_id(content_json.get("sticker_id") or content_json.get("id"))
|
||||
return _media_payload(
|
||||
"sticker",
|
||||
"[表情包]",
|
||||
sticker_id=sticker_id,
|
||||
name=str(content_json.get("display_name") or content_json.get("name") or ""),
|
||||
)
|
||||
|
||||
link_card = _parse_link_card_payload(content_json, message_type)
|
||||
if link_card:
|
||||
return link_card
|
||||
|
||||
rich_link = _parse_rich_text_link(content_json)
|
||||
if rich_link:
|
||||
return rich_link
|
||||
|
||||
text = (
|
||||
str(content_json.get("text") or content_json.get("content") or content_json.get("message") or "")
|
||||
).strip()
|
||||
if not text and url:
|
||||
inferred = _infer_media_type_from_url(url)
|
||||
if inferred == "voice":
|
||||
return {"type": "voice", "text": "[语音]", "url": url, "duration": duration}
|
||||
if inferred == "video":
|
||||
return {"type": "video", "text": "[视频]", "url": url, "duration": duration}
|
||||
if message_type == MSG_TYPE_IMAGE:
|
||||
return {"type": "image", "text": "[图片]", "url": url, "width": width, "height": height}
|
||||
if message_type == MSG_TYPE_STICKER:
|
||||
return {"type": "sticker", "text": "[表情包]", "url": url}
|
||||
if message_type == MSG_TYPE_VOICE:
|
||||
return {"type": "voice", "text": "[语音]", "url": url, "duration": duration}
|
||||
if message_type == MSG_TYPE_VIDEO:
|
||||
return {"type": "video", "text": "[视频]", "url": url, "duration": duration}
|
||||
|
||||
final_text = text or str(content_raw or "").strip()
|
||||
emoji = _resolve_text_emoji(final_text)
|
||||
if emoji:
|
||||
return emoji
|
||||
return {"type": "text", "text": final_text}
|
||||
|
||||
|
||||
def _parse_link_card_payload(content_json: dict[str, Any], message_type: int) -> dict[str, Any] | None:
|
||||
link_info = content_json.get("link_info")
|
||||
if not isinstance(link_info, dict):
|
||||
link_info = {}
|
||||
has_link = (
|
||||
message_type == MSG_TYPE_LINK_CARD
|
||||
or link_info
|
||||
or content_json.get("link_url")
|
||||
or content_json.get("cover_url")
|
||||
)
|
||||
if not has_link:
|
||||
return None
|
||||
title = str(content_json.get("title") or link_info.get("title") or "").strip()
|
||||
desc = str(
|
||||
content_json.get("desc")
|
||||
or content_json.get("description")
|
||||
or link_info.get("desc")
|
||||
or link_info.get("description")
|
||||
or ""
|
||||
).strip()
|
||||
url = str(
|
||||
content_json.get("link_url")
|
||||
or content_json.get("url")
|
||||
or link_info.get("url")
|
||||
or link_info.get("link_url")
|
||||
or ""
|
||||
).strip()
|
||||
cover = str(content_json.get("cover_url") or link_info.get("cover_url") or "").strip()
|
||||
text = title or desc or url or "[链接卡片]"
|
||||
payload: dict[str, Any] = {
|
||||
"type": "link_card",
|
||||
"text": text,
|
||||
"title": title,
|
||||
"desc": desc,
|
||||
}
|
||||
if url:
|
||||
payload["url"] = url
|
||||
if cover:
|
||||
payload["cover_url"] = cover
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_rich_text_link(content_json: dict[str, Any]) -> dict[str, Any] | None:
|
||||
rich = content_json.get("richTextInfos")
|
||||
if not isinstance(rich, list):
|
||||
return None
|
||||
for item in rich:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
link = str(item.get("link") or item.get("url") or "").strip()
|
||||
if not link:
|
||||
continue
|
||||
text = str(item.get("text") or item.get("display_text") or link).strip()
|
||||
return {"type": "link", "text": text or link, "url": link}
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_text_emoji(text: str) -> dict[str, Any] | None:
|
||||
"""文字表情 [酷拽] 等:查标准表情表,命中则转成可显示的贴纸。"""
|
||||
stripped = (text or "").strip()
|
||||
if not stripped or not (stripped.startswith("[") and stripped.endswith("]")):
|
||||
return None
|
||||
try:
|
||||
from .emoji_pack import looks_like_emoji_token, lookup_emoji_url
|
||||
|
||||
if not looks_like_emoji_token(stripped):
|
||||
return None
|
||||
url = lookup_emoji_url(stripped)
|
||||
if url:
|
||||
return {
|
||||
"type": "sticker",
|
||||
"text": stripped,
|
||||
"url": url,
|
||||
"name": stripped.strip("[]"),
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def parse_incoming_message(data: dict[str, Any]) -> str:
|
||||
"""从 IM API / WebSocket 消息 dict 提取并序列化展示内容。"""
|
||||
if not isinstance(data, dict):
|
||||
return str(data or "").strip()
|
||||
|
||||
msg_type = _coerce_message_type(
|
||||
data.get("message_type") or data.get("messageType") or data.get("msg_type"),
|
||||
MSG_TYPE_TEXT,
|
||||
)
|
||||
content_raw = (
|
||||
data.get("content")
|
||||
or data.get("message")
|
||||
or data.get("msg")
|
||||
or data.get("lastMessage")
|
||||
or data.get("last_msg")
|
||||
or data.get("preview")
|
||||
or data.get("brief")
|
||||
or ""
|
||||
)
|
||||
if isinstance(content_raw, dict):
|
||||
if not msg_type or msg_type == MSG_TYPE_TEXT:
|
||||
msg_type = _coerce_message_type(
|
||||
content_raw.get("message_type")
|
||||
or content_raw.get("messageType")
|
||||
or content_raw.get("msg_type"),
|
||||
msg_type,
|
||||
)
|
||||
parsed = format_im_message(content_raw, msg_type)
|
||||
return serialize_message_content(parsed)
|
||||
|
||||
if isinstance(content_raw, str):
|
||||
raw = content_raw.strip()
|
||||
if raw.startswith("{"):
|
||||
parsed = format_im_message(raw, msg_type)
|
||||
if parsed.get("type") != "text" or parsed.get("url") or msg_type != MSG_TYPE_TEXT:
|
||||
return serialize_message_content(parsed)
|
||||
if raw in _PLACEHOLDER_MARKERS and msg_type != MSG_TYPE_TEXT:
|
||||
parsed = format_im_message(raw, msg_type)
|
||||
return serialize_message_content(parsed)
|
||||
if msg_type != MSG_TYPE_TEXT:
|
||||
parsed = format_im_message(raw, msg_type)
|
||||
return serialize_message_content(parsed)
|
||||
return raw
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def serialize_message_content(msg: dict[str, Any]) -> str:
|
||||
"""序列化写入 message_logs / reply_content。"""
|
||||
msg_type = (msg.get("type") or "text").strip()
|
||||
if msg_type == "text":
|
||||
text = str(msg.get("text") or "").strip()
|
||||
return text
|
||||
cleaned = {k: v for k, v in msg.items() if v not in (None, "", [], {})}
|
||||
return json.dumps(cleaned, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def parse_stored_content(raw: str | None) -> dict[str, Any]:
|
||||
"""解析数据库中的 message_content / reply_content。"""
|
||||
text = (raw or "").strip()
|
||||
if not text:
|
||||
return {"type": "text", "text": ""}
|
||||
if text.startswith("{"):
|
||||
try:
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict) and data.get("type"):
|
||||
if not data.get("url") and data.get("uri"):
|
||||
prefer_voice = data.get("type") == "voice"
|
||||
resolved = resolve_media_uri(str(data["uri"]), prefer_voice=prefer_voice)
|
||||
if resolved:
|
||||
data = {**data, "url": resolved}
|
||||
url = str(data.get("url") or "").strip()
|
||||
if url:
|
||||
inferred = _infer_media_type_from_url(url)
|
||||
current = data.get("type")
|
||||
if current == "image" and inferred in ("voice", "video"):
|
||||
data = {
|
||||
**data,
|
||||
"type": inferred,
|
||||
"text": "[语音]" if inferred == "voice" else "[视频]",
|
||||
}
|
||||
elif current not in ("voice", "video", "image", "sticker") and inferred:
|
||||
data = {**data, "type": inferred}
|
||||
return data
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if text in _PLACEHOLDER_MARKERS:
|
||||
mapping = {
|
||||
"[图片]": "image",
|
||||
"[表情包]": "sticker",
|
||||
"[语音]": "voice",
|
||||
"[视频]": "video",
|
||||
}
|
||||
return {"type": mapping.get(text, "text"), "text": text}
|
||||
return {"type": "text", "text": text}
|
||||
|
||||
|
||||
def message_preview(raw: str | None) -> str:
|
||||
"""会话列表/日志摘要。"""
|
||||
msg = parse_stored_content(raw)
|
||||
msg_type = msg.get("type") or "text"
|
||||
if msg_type == "text":
|
||||
return str(msg.get("text") or "")
|
||||
label = _TYPE_LABELS.get(msg_type, msg.get("text") or "[消息]")
|
||||
extra = str(msg.get("name") or "").strip()
|
||||
if extra and msg_type == "sticker":
|
||||
return f"[表情] {extra}"
|
||||
return str(msg.get("text") or label)
|
||||
|
||||
|
||||
def normalize_outgoing_content(
|
||||
content: str = "",
|
||||
message_type: str | None = None,
|
||||
media_url: str | None = None,
|
||||
sticker_url: str | None = None,
|
||||
width: int | None = None,
|
||||
height: int | None = None,
|
||||
sticker_id: str | None = None,
|
||||
) -> str:
|
||||
"""构造可发送/可落库的 content 字符串。"""
|
||||
raw = (content or "").strip()
|
||||
parsed_json: dict[str, Any] | None = None
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and data.get("type"):
|
||||
parsed_json = data
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if parsed_json:
|
||||
merged = dict(parsed_json)
|
||||
if width and not merged.get("width"):
|
||||
merged["width"] = width
|
||||
if height and not merged.get("height"):
|
||||
merged["height"] = height
|
||||
if sticker_id and not merged.get("sticker_id"):
|
||||
merged["sticker_id"] = sticker_id
|
||||
url = (media_url or sticker_url or "").strip()
|
||||
if url and not merged.get("url"):
|
||||
merged["url"] = url
|
||||
return serialize_message_content(merged)
|
||||
|
||||
explicit_type = (message_type or "").strip().lower()
|
||||
if explicit_type in ("image", "sticker", "voice", "video", "text"):
|
||||
if explicit_type == "text":
|
||||
return (content or "").strip()
|
||||
payload: dict[str, Any] = {"type": explicit_type}
|
||||
url = (media_url or sticker_url or "").strip()
|
||||
if url:
|
||||
payload["url"] = url
|
||||
if explicit_type == "sticker" and sticker_id:
|
||||
payload["sticker_id"] = sticker_id
|
||||
if width:
|
||||
payload["width"] = width
|
||||
if height:
|
||||
payload["height"] = height
|
||||
text = (content or "").strip()
|
||||
if text:
|
||||
payload["text"] = text
|
||||
elif explicit_type == "image":
|
||||
payload["text"] = "[图片]"
|
||||
elif explicit_type == "sticker":
|
||||
payload["text"] = "[表情包]"
|
||||
return serialize_message_content(payload)
|
||||
|
||||
raw = (content or "").strip()
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and data.get("type"):
|
||||
return serialize_message_content(data)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return raw
|
||||
|
||||
|
||||
def is_media_message(raw: str | None) -> bool:
|
||||
return parse_stored_content(raw).get("type") not in (None, "text")
|
||||
|
||||
|
||||
def format_system_log_message(raw: str | None) -> str:
|
||||
"""系统诊断日志中的消息摘要。"""
|
||||
msg = parse_stored_content(raw)
|
||||
msg_type = msg.get("type") or "text"
|
||||
if msg_type == "text":
|
||||
return str(msg.get("text") or "")
|
||||
parts = [message_preview(raw)]
|
||||
url = str(msg.get("url") or "").strip()
|
||||
if url:
|
||||
parts.append(f"URL: {url}")
|
||||
duration = msg.get("duration")
|
||||
if duration:
|
||||
parts.append(f"时长: {duration}s")
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def extract_urls_from_detail(detail: str | None) -> list[str]:
|
||||
if not detail:
|
||||
return []
|
||||
return re.findall(r"https?://[^\s\]|))\"']+", detail)
|
||||
@@ -0,0 +1,258 @@
|
||||
"""极简 protobuf wire 解码器(无第三方依赖)。
|
||||
|
||||
用于解析抖音 IM 发送私信的响应:官方 Response.proto 只建模了
|
||||
create/get_info/new_message_notify 三种 body,没有“发送消息响应”,
|
||||
导致仅凭 error_desc 为空就误判为发送成功。这里直接按 wire 格式解码,
|
||||
读取真实的 status_code / server_message_id,判定是否真的投递成功。
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
def _format_status_json(status_json: dict) -> str:
|
||||
"""把内嵌 status JSON 格式化为更可读的失败原因。"""
|
||||
code = status_json.get("status_code")
|
||||
raw_check = status_json.get("raw_check_code")
|
||||
decision = status_json.get("decision_type")
|
||||
parts = [f"status_code={code}"]
|
||||
if raw_check is not None:
|
||||
parts.append(f"raw_check_code={raw_check}")
|
||||
if decision:
|
||||
parts.append(f"decision_type={decision}")
|
||||
return ";".join(parts)
|
||||
|
||||
|
||||
def _read_varint(buf: bytes, i: int) -> tuple[int, int]:
|
||||
shift = 0
|
||||
result = 0
|
||||
n = len(buf)
|
||||
while i < n:
|
||||
b = buf[i]
|
||||
i += 1
|
||||
result |= (b & 0x7F) << shift
|
||||
if not (b & 0x80):
|
||||
return result, i
|
||||
shift += 7
|
||||
if shift > 70:
|
||||
break
|
||||
raise ValueError("truncated varint")
|
||||
|
||||
|
||||
def decode_fields(buf: bytes) -> list[tuple[int, int, Any]]:
|
||||
"""返回 [(field_num, wire_type, value), ...]。
|
||||
|
||||
wire_type: 0=varint(int), 1=64bit(int), 2=length-delimited(bytes), 5=32bit(int)
|
||||
"""
|
||||
out: list[tuple[int, int, Any]] = []
|
||||
i = 0
|
||||
n = len(buf)
|
||||
while i < n:
|
||||
key, i = _read_varint(buf, i)
|
||||
field = key >> 3
|
||||
wt = key & 7
|
||||
if wt == 0:
|
||||
val, i = _read_varint(buf, i)
|
||||
out.append((field, wt, val))
|
||||
elif wt == 2:
|
||||
ln, i = _read_varint(buf, i)
|
||||
val = buf[i:i + ln]
|
||||
i += ln
|
||||
out.append((field, wt, val))
|
||||
elif wt == 5:
|
||||
val = int.from_bytes(buf[i:i + 4], "little")
|
||||
i += 4
|
||||
out.append((field, wt, val))
|
||||
elif wt == 1:
|
||||
val = int.from_bytes(buf[i:i + 8], "little")
|
||||
i += 8
|
||||
out.append((field, wt, val))
|
||||
else:
|
||||
raise ValueError(f"unsupported wire type {wt}")
|
||||
return out
|
||||
|
||||
|
||||
def _collect_big_varints(buf: bytes, acc: list[int], depth: int = 0) -> None:
|
||||
"""递归收集疑似 ID 的大整数(server_message_id / short_id 等都是大数)。"""
|
||||
if depth > 6:
|
||||
return
|
||||
try:
|
||||
fields = decode_fields(buf)
|
||||
except Exception:
|
||||
return
|
||||
for _field, wt, val in fields:
|
||||
if wt == 0 and isinstance(val, int) and val > 10 ** 12:
|
||||
acc.append(val)
|
||||
elif wt == 2 and isinstance(val, (bytes, bytearray)) and val:
|
||||
_collect_big_varints(bytes(val), acc, depth + 1)
|
||||
|
||||
|
||||
def _extract_status_json(raw: bytes) -> Optional[dict]:
|
||||
"""抖音发送响应的 body 内嵌一段 JSON:{"status_code":x,"tips":"...","status_msg":{...}}。
|
||||
|
||||
这才是“消息是否真正投递”的权威结论(status_code=0 才是真成功)。
|
||||
顶层 message=OK 只是接口层面的“已受理”,不代表已投递。
|
||||
"""
|
||||
marker = b'"status_code"'
|
||||
idx = raw.find(marker)
|
||||
if idx < 0:
|
||||
return None
|
||||
start = raw.rfind(b"{", 0, idx)
|
||||
if start < 0:
|
||||
return None
|
||||
depth = 0
|
||||
in_str = False
|
||||
esc = False
|
||||
end = -1
|
||||
for i in range(start, len(raw)):
|
||||
c = raw[i]
|
||||
if in_str:
|
||||
if esc:
|
||||
esc = False
|
||||
elif c == 0x5C: # backslash
|
||||
esc = True
|
||||
elif c == 0x22: # quote
|
||||
in_str = False
|
||||
continue
|
||||
if c == 0x22:
|
||||
in_str = True
|
||||
elif c == 0x7B: # {
|
||||
depth += 1
|
||||
elif c == 0x7D: # }
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
end = i + 1
|
||||
break
|
||||
if end < 0:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw[start:end].decode("utf-8", "ignore"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def analyze_send_response(raw: bytes) -> dict:
|
||||
"""分析“发送私信”的 protobuf 响应,判定是否真正投递成功。
|
||||
|
||||
顶层 Response 字段(见 Response.proto):
|
||||
1=cmd, 2=sequence_id, 3=error_desc(string), 4=message(string),
|
||||
5=inbox_type, 6=body(ResponseBody)。
|
||||
|
||||
注意:顶层没有 status_code 字段。真正“消息已写入服务端”的标志是
|
||||
body(field 6) 里带有服务端分配的 server_message_id(大整数)。
|
||||
sequence_id(field 2) 也是大整数,因此只在 body 内部查找 message_id,
|
||||
避免把 sequence_id 误当成投递成功标志。
|
||||
|
||||
返回:
|
||||
ok: 是否真正发送成功(body 内带服务端 message_id,或 message=OK 且有 body)
|
||||
cmd: 顶层 cmd
|
||||
message: 顶层 message 文本(field 4)
|
||||
error_desc: 顶层 error_desc 文本(field 3)
|
||||
server_message_id: body 内服务端消息 ID(投递成功的强信号)
|
||||
has_body: 是否带 body
|
||||
summary: 顶层字段概览 + hex 片段,便于排查
|
||||
"""
|
||||
info = {
|
||||
"ok": False,
|
||||
"cmd": None,
|
||||
"status": None,
|
||||
"status_code": None,
|
||||
"raw_check_code": None,
|
||||
"delivered_with_notice": False,
|
||||
"status_reason": "",
|
||||
"message": "",
|
||||
"error_desc": "",
|
||||
"server_message_id": None,
|
||||
"has_body": False,
|
||||
"summary": "",
|
||||
}
|
||||
if not raw:
|
||||
info["summary"] = "空响应"
|
||||
return info
|
||||
try:
|
||||
fields = decode_fields(raw)
|
||||
except Exception as e:
|
||||
info["summary"] = f"解码失败: {e}; hex={raw[:120].hex()}"
|
||||
return info
|
||||
|
||||
body = None
|
||||
parts = []
|
||||
for field, wt, val in fields:
|
||||
if field == 1 and wt == 0:
|
||||
info["cmd"] = val
|
||||
elif field == 3 and wt == 2:
|
||||
try:
|
||||
info["error_desc"] = bytes(val).decode("utf-8", "ignore")
|
||||
except Exception:
|
||||
pass
|
||||
elif field == 4 and wt == 2:
|
||||
try:
|
||||
info["message"] = bytes(val).decode("utf-8", "ignore")
|
||||
except Exception:
|
||||
pass
|
||||
elif field == 6 and wt == 2:
|
||||
body = bytes(val)
|
||||
info["has_body"] = len(body) > 0
|
||||
|
||||
if wt == 0:
|
||||
parts.append(f"{field}=int:{val}")
|
||||
elif wt == 2:
|
||||
parts.append(f"{field}=bytes[{len(val)}]")
|
||||
else:
|
||||
parts.append(f"{field}={val}")
|
||||
info["summary"] = " ".join(parts) + f" | hex={raw[:120].hex()}"
|
||||
|
||||
# 只在 body 内部查找服务端 message_id(避免误用顶层 sequence_id)
|
||||
if body:
|
||||
ids: list[int] = []
|
||||
_collect_big_varints(body, ids)
|
||||
if ids:
|
||||
info["server_message_id"] = max(ids)
|
||||
|
||||
# 权威结论:body 内嵌 JSON 的 status_code(0 才是真成功)
|
||||
status_json = _extract_status_json(raw)
|
||||
if status_json is not None:
|
||||
info["status_code"] = status_json.get("status_code")
|
||||
info["raw_check_code"] = status_json.get("raw_check_code")
|
||||
tips = (status_json.get("tips") or "").strip()
|
||||
status_msg = status_json.get("status_msg")
|
||||
msg_text = ""
|
||||
if isinstance(status_msg, dict):
|
||||
# 抖音把人类可读提示放在 status_msg.msg_content.tips
|
||||
mc = status_msg.get("msg_content")
|
||||
if isinstance(mc, dict):
|
||||
msg_text = (mc.get("tips") or mc.get("content") or "").strip()
|
||||
if not msg_text:
|
||||
msg_text = (
|
||||
status_msg.get("toast")
|
||||
or status_msg.get("content")
|
||||
or status_msg.get("msg")
|
||||
or ""
|
||||
)
|
||||
elif isinstance(status_msg, str):
|
||||
msg_text = status_msg
|
||||
info["status_reason"] = tips or msg_text or _format_status_json(status_json)
|
||||
|
||||
msg_ok = info["message"].strip().upper() == "OK"
|
||||
|
||||
# 优先用 status_code 判定:明确给了 status_code 就以它为准(0=成功,非0=另判)
|
||||
if info["status_code"] is not None:
|
||||
if info["status_code"] == 0 and not info["error_desc"]:
|
||||
info["ok"] = True
|
||||
elif info["raw_check_code"] == 0 and msg_ok and not info["error_desc"]:
|
||||
# raw_check_code=0 表示已通过抖音风控/安全校验;配合 message=OK,
|
||||
# 说明消息已实际投递。此时非零 status_code 只是“业务侧提示”
|
||||
# (如营销/陌生人限制提醒),对方仍能收到,不应判为发送失败。
|
||||
info["ok"] = True
|
||||
info["delivered_with_notice"] = True
|
||||
else:
|
||||
# raw_check_code=1(被风控拦截)或缺少 OK 标志:判为未送达
|
||||
info["ok"] = False
|
||||
else:
|
||||
# 没有内嵌 status_code 时,退回“message=OK 且 body 内有服务端 message_id”
|
||||
info["ok"] = bool(
|
||||
not info["error_desc"]
|
||||
and msg_ok
|
||||
and info["server_message_id"] is not None
|
||||
)
|
||||
return info
|
||||
@@ -0,0 +1,225 @@
|
||||
"""私信对方用户资料抓取(昵称 / 头像 / UID)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from rpa_engine.device_profiles import resolve_user_agent
|
||||
from .auth import DouyinAuth
|
||||
from .conv_util import resolve_peer_uid
|
||||
from .dy_util import (
|
||||
DEFAULT_USER_AGENT,
|
||||
generate_a_bogus,
|
||||
generate_msToken,
|
||||
generate_webid,
|
||||
splice_url,
|
||||
)
|
||||
from .protocol import _pick_avatar_url
|
||||
from .session import DouyinImSession
|
||||
|
||||
logger = logging.getLogger("douyin_im.peer_profile")
|
||||
|
||||
_profile_cache: dict[str, dict[str, str]] = {}
|
||||
_profile_cache_at: dict[str, float] = {}
|
||||
_PROFILE_SUCCESS_TTL = 6 * 3600
|
||||
_PROFILE_FAILURE_TTL = 5 * 60
|
||||
|
||||
|
||||
def _cache_key(account_id: int, peer_uid: str) -> str:
|
||||
return f"{account_id}:{peer_uid}"
|
||||
|
||||
|
||||
def _pick_str(data: dict, *keys: str) -> str:
|
||||
for key in keys:
|
||||
value = data.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_profile_from_payload(data: Any) -> dict[str, str]:
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
nodes = [data, data.get("user"), data.get("user_info"), data.get("data")]
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
uid = _pick_str(node, "uid", "user_id", "user_uid", "id")
|
||||
nickname = _pick_str(
|
||||
node,
|
||||
"nickname",
|
||||
"nick_name",
|
||||
"unique_id",
|
||||
"display_name",
|
||||
"name",
|
||||
)
|
||||
avatar = _pick_avatar_url(node)
|
||||
if uid or nickname or avatar:
|
||||
return {"uid": uid, "nickname": nickname, "avatar_url": avatar}
|
||||
return {}
|
||||
|
||||
|
||||
def _requests_proxies() -> dict | None:
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
|
||||
return requests_proxies()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _build_auth(session: DouyinImSession) -> tuple[DouyinAuth, str]:
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||||
ua = resolve_user_agent(session.user_agent or DEFAULT_USER_AGENT)
|
||||
return auth, ua
|
||||
|
||||
|
||||
def is_generic_peer_name(name: str, peer_uid: str = "") -> bool:
|
||||
value = (name or "").strip()
|
||||
if not value:
|
||||
return True
|
||||
if peer_uid and value == peer_uid:
|
||||
return True
|
||||
if value.isdigit():
|
||||
return True
|
||||
if value.startswith("用户") and value[2:].isdigit():
|
||||
return True
|
||||
if value.startswith("会话") and len(value) <= 16:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def fetch_peer_profile_sync(
|
||||
session: DouyinImSession,
|
||||
peer_uid: int | str,
|
||||
account_id: int = 0,
|
||||
) -> dict[str, str]:
|
||||
uid = str(peer_uid or "").strip()
|
||||
if not uid.isdigit():
|
||||
return {}
|
||||
|
||||
cache_key = _cache_key(account_id, uid)
|
||||
cached = _profile_cache.get(cache_key)
|
||||
cached_at = _profile_cache_at.get(cache_key, 0.0)
|
||||
if cached:
|
||||
ttl = (
|
||||
_PROFILE_SUCCESS_TTL
|
||||
if cached.get("nickname") or cached.get("avatar_url")
|
||||
else _PROFILE_FAILURE_TTL
|
||||
)
|
||||
if time.time() - cached_at < ttl:
|
||||
return dict(cached)
|
||||
|
||||
result = {"uid": uid, "nickname": "", "avatar_url": ""}
|
||||
try:
|
||||
auth, ua = _build_auth(session)
|
||||
except Exception as exc:
|
||||
logger.warning(f"build auth for peer profile failed: {exc}")
|
||||
_profile_cache[cache_key] = dict(result)
|
||||
_profile_cache_at[cache_key] = time.time()
|
||||
return result
|
||||
|
||||
try:
|
||||
web_id = session.web_id or generate_webid(auth, "https://www.douyin.com/")
|
||||
if web_id and not session.web_id:
|
||||
# Reuse the homepage-derived ID for every peer on this account.
|
||||
session.web_id = str(web_id)
|
||||
except Exception as exc:
|
||||
logger.debug(f"generate webid for peer profile failed: {exc}")
|
||||
web_id = session.web_id or ""
|
||||
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
base_params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"publish_video_strategy_type": "2",
|
||||
"user_id": uid,
|
||||
"sec_user_id": "",
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"webid": web_id,
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
}
|
||||
endpoints = [
|
||||
"https://www.douyin.com/aweme/v1/web/user/profile/other/",
|
||||
"https://www.douyin.com/aweme/v1/web/im/user/info/",
|
||||
]
|
||||
|
||||
proxies = _requests_proxies()
|
||||
for url in endpoints:
|
||||
try:
|
||||
params = dict(base_params)
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
resp = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
verify=False,
|
||||
timeout=12,
|
||||
proxies=proxies,
|
||||
)
|
||||
data = resp.json()
|
||||
extracted = _extract_profile_from_payload(data)
|
||||
if extracted.get("uid") and not result["uid"]:
|
||||
result["uid"] = extracted["uid"]
|
||||
if extracted.get("nickname") and not result["nickname"]:
|
||||
result["nickname"] = extracted["nickname"]
|
||||
if extracted.get("avatar_url") and not result["avatar_url"]:
|
||||
result["avatar_url"] = extracted["avatar_url"]
|
||||
if result["nickname"] and result["avatar_url"]:
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.debug(f"peer profile fetch failed for {url}: {exc}")
|
||||
|
||||
# Cache both success and failure. Without a short negative TTL, missing or
|
||||
# rate-limited profiles were fetched again for every account poll.
|
||||
_profile_cache[cache_key] = dict(result)
|
||||
_profile_cache_at[cache_key] = time.time()
|
||||
return result
|
||||
|
||||
|
||||
async def fetch_peer_profile(
|
||||
session: DouyinImSession,
|
||||
peer_uid: int | str,
|
||||
account_id: int = 0,
|
||||
) -> dict[str, str]:
|
||||
from .traffic_control import get_traffic_controller
|
||||
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(account_id, "peer profile"):
|
||||
return await asyncio.to_thread(
|
||||
fetch_peer_profile_sync,
|
||||
session,
|
||||
peer_uid,
|
||||
account_id,
|
||||
)
|
||||
|
||||
|
||||
def enrich_conversation_item(conv: dict, my_uid: int = 0) -> dict:
|
||||
"""补全会话项中的 peer_uid / sender_id。"""
|
||||
item = dict(conv or {})
|
||||
conv_id = str(item.get("conversation_id") or "").strip()
|
||||
peer_uid = str(item.get("peer_uid") or item.get("sender_id") or "").strip()
|
||||
if (not peer_uid or not peer_uid.isdigit()) and conv_id and my_uid:
|
||||
resolved = resolve_peer_uid(conv_id, int(my_uid))
|
||||
if resolved:
|
||||
peer_uid = str(resolved)
|
||||
if peer_uid:
|
||||
item["peer_uid"] = peer_uid
|
||||
item["sender_id"] = peer_uid
|
||||
elif conv_id:
|
||||
item["sender_id"] = conv_id
|
||||
return item
|
||||
@@ -0,0 +1,120 @@
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
|
||||
from .static import Request_pb2 as RequestProto
|
||||
from .dy_util import (
|
||||
generate_webid,
|
||||
generate_req_sign,
|
||||
generate_millisecond,
|
||||
normalize_client_cert,
|
||||
DEFAULT_USER_AGENT,
|
||||
)
|
||||
|
||||
|
||||
def _ua_headers(auth) -> tuple[str, str]:
|
||||
ua = getattr(auth, "user_agent", None) or DEFAULT_USER_AGENT
|
||||
browser_version = ua.split("Mozilla/", 1)[-1] if "Mozilla/" in ua else ua
|
||||
return ua, browser_version
|
||||
|
||||
|
||||
class ProtoBuilder:
|
||||
@staticmethod
|
||||
def build_normal_request(auth, cmd):
|
||||
ua, browser_version = _ua_headers(auth)
|
||||
request = RequestProto.Request()
|
||||
request.cmd = cmd
|
||||
request.sequence_id = random.randint(10000, 11000)
|
||||
request.sdk_version = "1.1.3"
|
||||
request.token = auth.ticket if auth.ticket else ""
|
||||
request.refer = 3
|
||||
request.inbox_type = 0
|
||||
request.build_number = "5fa6ff1:Detached: 5fa6ff1111fd53aafc4c753505d3c93daad74d27"
|
||||
did = str(getattr(auth, "device_id", "") or "0")
|
||||
request.device_id = did
|
||||
request.device_platform = 'douyin_pc'
|
||||
request.headers['session_aid'] = '6383'
|
||||
request.headers['session_did'] = did
|
||||
request.headers['app_name'] = 'douyin_pc'
|
||||
request.headers['priority_region'] = 'cn'
|
||||
request.headers['user_agent'] = ua
|
||||
request.headers['cookie_enabled'] = 'true'
|
||||
request.headers['browser_language'] = 'zh-CN'
|
||||
request.headers['browser_platform'] = 'Win32'
|
||||
request.headers['browser_name'] = 'Mozilla'
|
||||
request.headers['browser_version'] = browser_version
|
||||
request.headers['browser_online'] = 'true'
|
||||
request.headers['screen_width'] = '1707'
|
||||
request.headers['screen_height'] = '960'
|
||||
request.headers['referer'] = ''
|
||||
request.headers['timezone_name'] = 'Etc/GMT-8'
|
||||
request.headers['deviceId'] = did
|
||||
request.headers['webid'] = generate_webid(auth)
|
||||
request.headers['fp'] = auth.cookie.get('s_v_web_id', '') if auth.cookie else ''
|
||||
request.headers['is-retry'] = '0'
|
||||
request.auth_type = 4
|
||||
request.biz = 'douyin_web'
|
||||
request.access = 'web_sdk'
|
||||
request.ts_sign = auth.ts_sign if auth.ts_sign else ""
|
||||
request.sdk_cert = normalize_client_cert(auth.client_cert or "")
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def build_create_conversation_request(auth, toId, myId):
|
||||
request = ProtoBuilder.build_normal_request(auth, 609)
|
||||
request.body.create_conversation_v2_body.conversation_type = 1
|
||||
request.body.create_conversation_v2_body.participants.extend([int(toId), int(myId)])
|
||||
reuqest_sign = generate_req_sign({
|
||||
"sign_data": f"avatar_url=&idempotent_id=&name=&participants={toId},{myId}",
|
||||
"certType": "cookie",
|
||||
"scene": "web_protect"
|
||||
}, auth.private_key)
|
||||
request.reuqest_sign = reuqest_sign
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def build_get_conversation_list_info_request(auth, toId, myId, conversation_short_id):
|
||||
request = ProtoBuilder.build_normal_request(auth, 610)
|
||||
request.body.get_conversation_info_list_v2_body.data.conversation_id = f"0:1:{myId}:{toId}"
|
||||
request.body.get_conversation_info_list_v2_body.data.conversation_short_id = int(conversation_short_id)
|
||||
request.body.get_conversation_info_list_v2_body.data.conversation_type = 1
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def build_send_message_request(
|
||||
auth,
|
||||
conversation_id,
|
||||
conversation_short_id,
|
||||
ticket,
|
||||
msg_content,
|
||||
message_type: int = 7,
|
||||
):
|
||||
client_message_id = str(uuid.uuid4())
|
||||
request = ProtoBuilder.build_normal_request(auth, 100)
|
||||
request.body.send_message_body.conversation_id = conversation_id
|
||||
request.body.send_message_body.conversation_type = 1
|
||||
request.body.send_message_body.conversation_short_id = int(conversation_short_id)
|
||||
request.body.send_message_body.content = json.dumps(msg_content, ensure_ascii=False,
|
||||
separators=(',', ':'))
|
||||
request.body.send_message_body.ext.append(
|
||||
RequestProto.ExtValue(key='s:client_message_id', value=client_message_id)
|
||||
)
|
||||
request.body.send_message_body.ext.append(
|
||||
RequestProto.ExtValue(key='s:stime', value=str(generate_millisecond()))
|
||||
)
|
||||
request.body.send_message_body.ext.append(
|
||||
RequestProto.ExtValue(key='s:mentioned_users', value='')
|
||||
)
|
||||
request.body.send_message_body.message_type = int(message_type)
|
||||
request.body.send_message_body.ticket = ticket
|
||||
request.body.send_message_body.client_message_id = client_message_id
|
||||
|
||||
# 签名数据计算
|
||||
sign_data_str = f'content={json.dumps(msg_content, separators=(",", ":"), ensure_ascii=False)}' + f'&conversation_id={conversation_id}&conversation_short_id={conversation_short_id}'
|
||||
req_sign = generate_req_sign({
|
||||
"sign_data": sign_data_str,
|
||||
"certType": "cookie",
|
||||
"scene": "web_protect"
|
||||
}, auth.private_key)
|
||||
request.reuqest_sign = req_sign
|
||||
return request
|
||||
@@ -0,0 +1,518 @@
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
|
||||
from .message_content import (
|
||||
MSG_TYPE_IMAGE,
|
||||
MSG_TYPE_LINK_CARD,
|
||||
MSG_TYPE_STICKER,
|
||||
MSG_TYPE_TEXT,
|
||||
MSG_TYPE_VIDEO,
|
||||
MSG_TYPE_VOICE,
|
||||
_coerce_message_type,
|
||||
format_im_message,
|
||||
message_preview,
|
||||
parse_incoming_message,
|
||||
parse_stored_content,
|
||||
serialize_message_content,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("douyin_im.protocol")
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def _is_control_payload(content_json: Any, msg_type: int = 0) -> bool:
|
||||
"""判断是否为「会话控制/状态更新」等非聊天内容帧。
|
||||
|
||||
例如 command_type=6 的连续互动统计(consecutive_chat_data)、ext_data 元数据更新、
|
||||
message_type>=50000 的系统通知等——这些不是用户发的消息,不应记录/展示成聊天气泡,
|
||||
更不应触发自动回复。
|
||||
"""
|
||||
try:
|
||||
if msg_type and int(msg_type) >= 50000:
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if isinstance(content_json, dict) and ("command_type" in content_json or "ext_data" in content_json):
|
||||
return True
|
||||
return False
|
||||
|
||||
_WS_DEBUG_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "ws_media_debug.log")
|
||||
|
||||
|
||||
def _should_emit_ws_message(
|
||||
conversation_id: str,
|
||||
msg_type: int,
|
||||
) -> bool:
|
||||
"""判断 WS 帧是否为用户聊天消息(控制帧已在 _is_control_payload 过滤)。"""
|
||||
if not conversation_id:
|
||||
return False
|
||||
try:
|
||||
if int(msg_type) >= 50000:
|
||||
return False
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _dump_ws_message(msg_type: int, conversation_id: str, content_str: str, msg: Any = None) -> None:
|
||||
"""把每条 WS 消息的全部字段落到调试文件,便于排查媒体字段结构。
|
||||
|
||||
默认关闭,仅当设置环境变量 KEFU_WS_DEBUG=1 时写盘,避免生产环境无界增长 / 泄露聊天内容。
|
||||
content 为空时(如 type=26 瘦推送)会额外打印 protobuf 其余字段,确保「接收到的全部信息」可见。
|
||||
"""
|
||||
if os.getenv("KEFU_WS_DEBUG", "") not in ("1", "true", "True"):
|
||||
return
|
||||
try:
|
||||
extra = ""
|
||||
if msg is not None:
|
||||
fields = {}
|
||||
try:
|
||||
for f, v in msg.ListFields():
|
||||
if f.name == "content":
|
||||
continue
|
||||
fields[f.name] = v
|
||||
except Exception:
|
||||
pass
|
||||
if fields:
|
||||
extra = " | fields=" + json.dumps(fields, ensure_ascii=False, default=str)
|
||||
line = (
|
||||
f"{datetime.now().isoformat()} type={msg_type} "
|
||||
f"conv={conversation_id} content={content_str}{extra}\n"
|
||||
)
|
||||
with open(_WS_DEBUG_PATH, "a", encoding="utf-8") as fh:
|
||||
fh.write(line)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _walk_strings(data: Any, depth: int = 0, max_depth: int = 10):
|
||||
if depth > max_depth:
|
||||
return
|
||||
if isinstance(data, dict):
|
||||
for v in data.values():
|
||||
yield from _walk_strings(v, depth + 1, max_depth)
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
yield from _walk_strings(item, depth + 1, max_depth)
|
||||
elif isinstance(data, str) and data.strip():
|
||||
yield data.strip()
|
||||
|
||||
|
||||
def extract_json_objects(raw: bytes | str) -> list[dict]:
|
||||
"""从二进制帧中尽量提取 JSON 对象"""
|
||||
if isinstance(raw, bytes):
|
||||
for codec in ("utf-8", "latin-1"):
|
||||
try:
|
||||
text = raw.decode(codec, errors="ignore")
|
||||
break
|
||||
except Exception:
|
||||
text = ""
|
||||
else:
|
||||
text = ""
|
||||
else:
|
||||
text = raw
|
||||
|
||||
results = []
|
||||
for match in re.finditer(r"\{[^{}]{0,2000}\}", text):
|
||||
chunk = match.group(0)
|
||||
try:
|
||||
obj = json.loads(chunk)
|
||||
if isinstance(obj, dict):
|
||||
results.append(obj)
|
||||
except Exception:
|
||||
continue
|
||||
return results
|
||||
|
||||
|
||||
def parse_ws_payload(raw: bytes | str) -> list[dict]:
|
||||
"""解析 WebSocket 二进制帧,返回标准化消息 dict 列表"""
|
||||
messages = []
|
||||
|
||||
# 尝试 Protobuf 解包
|
||||
if isinstance(raw, bytes):
|
||||
try:
|
||||
from .static import Live_pb2, Response_pb2
|
||||
frame = Live_pb2.PushFrame()
|
||||
frame.ParseFromString(raw)
|
||||
if frame.payloadType == 'pb':
|
||||
response = Response_pb2.Response()
|
||||
response.ParseFromString(frame.payload)
|
||||
body = response.body
|
||||
if body.HasField("new_message_notify"):
|
||||
notify = body.new_message_notify
|
||||
if notify.HasField("message"):
|
||||
msg = notify.message
|
||||
sender = str(msg.sender)
|
||||
msg_type = msg.message_type
|
||||
conversation_id = msg.conversation_id
|
||||
content_str = msg.content
|
||||
server_message_id = str(getattr(msg, "server_message_id", "") or "")
|
||||
|
||||
_dump_ws_message(msg_type, conversation_id, content_str, msg)
|
||||
|
||||
text_content = ""
|
||||
media_msg: dict = {}
|
||||
content_json: dict = {}
|
||||
try:
|
||||
content_json = json.loads(content_str) if content_str else {}
|
||||
if not isinstance(content_json, dict):
|
||||
content_json = {}
|
||||
media_msg = format_im_message(content_json, msg_type)
|
||||
text_content = media_msg.get("text") or ""
|
||||
except Exception:
|
||||
media_msg = format_im_message(content_str or "", msg_type)
|
||||
text_content = media_msg.get("text") or content_str or ""
|
||||
|
||||
# 跳过会话控制/状态更新等非聊天内容帧(不记录、不展示、不触发自动回复)
|
||||
if _is_control_payload(content_json, msg_type):
|
||||
logger.debug(
|
||||
"Skip control WS frame: type=%s conv=%s", msg_type, conversation_id
|
||||
)
|
||||
return messages
|
||||
|
||||
if _should_emit_ws_message(conversation_id, msg_type):
|
||||
sender_uid = str(msg.sender)
|
||||
if media_msg and (
|
||||
media_msg.get("text")
|
||||
or media_msg.get("type") not in (None, "text", "")
|
||||
):
|
||||
display_content = serialize_message_content(media_msg)
|
||||
else:
|
||||
display_content = text_content or content_str
|
||||
payload = {
|
||||
"sender_name": sender_uid,
|
||||
"sender_uid": sender_uid,
|
||||
"content": display_content,
|
||||
"raw_content": content_str,
|
||||
"conversation_id": conversation_id,
|
||||
"unread_count": 1,
|
||||
"server_message_id": server_message_id,
|
||||
"message_type": msg_type,
|
||||
}
|
||||
if msg_type in (
|
||||
MSG_TYPE_IMAGE,
|
||||
MSG_TYPE_STICKER,
|
||||
MSG_TYPE_VOICE,
|
||||
MSG_TYPE_VIDEO,
|
||||
MSG_TYPE_LINK_CARD,
|
||||
):
|
||||
if not media_msg.get("url") and not media_msg.get("uri"):
|
||||
logger.warning(
|
||||
"Media WS message missing url: type=%s content=%s",
|
||||
msg_type,
|
||||
(content_str or "")[:800],
|
||||
)
|
||||
elif not media_msg.get("url"):
|
||||
logger.info(
|
||||
"Media WS message resolved via uri: type=%s uri=%s",
|
||||
msg_type,
|
||||
media_msg.get("uri"),
|
||||
)
|
||||
messages.append(payload)
|
||||
logger.info(
|
||||
"Protobuf WS message parsed: sender=%s type=%s content=%s conv=%s",
|
||||
sender,
|
||||
msg_type,
|
||||
text_content,
|
||||
conversation_id,
|
||||
)
|
||||
return messages
|
||||
except Exception as e:
|
||||
logger.debug(f"Protobuf WS parse failed: {e}")
|
||||
|
||||
if isinstance(raw, str):
|
||||
payloads = [raw.encode("utf-8", errors="ignore")]
|
||||
else:
|
||||
payloads = [raw]
|
||||
# 尝试 gzip 解压(frontier 常见)
|
||||
try:
|
||||
payloads.append(gzip.decompress(raw))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for payload in payloads:
|
||||
# 1) 直接 JSON
|
||||
if isinstance(payload, bytes):
|
||||
text = payload.decode("utf-8", errors="ignore").strip()
|
||||
else:
|
||||
text = str(payload).strip()
|
||||
if text.startswith("{") or text.startswith("["):
|
||||
try:
|
||||
data = json.loads(text)
|
||||
messages.extend(normalize_im_payload(data))
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2) 嵌入 JSON
|
||||
for obj in extract_json_objects(payload):
|
||||
messages.extend(normalize_im_payload(obj))
|
||||
|
||||
# 3) 纯文本兜底
|
||||
if isinstance(payload, bytes):
|
||||
text = payload.decode("utf-8", errors="ignore")
|
||||
plain = _extract_plain_text(text)
|
||||
if plain:
|
||||
messages.append({"content": plain, "sender_name": "", "raw_content": plain, "raw": True})
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def normalize_im_payload_from_bytes(raw: bytes) -> list[dict]:
|
||||
"""Try to extract conversation/message payloads from binary IM API responses."""
|
||||
results: list[dict] = []
|
||||
for obj in extract_json_objects(raw):
|
||||
results.extend(normalize_im_payload(obj))
|
||||
if results:
|
||||
return results
|
||||
|
||||
try:
|
||||
from .static import Response_pb2
|
||||
response = Response_pb2.Response()
|
||||
response.ParseFromString(raw)
|
||||
body = response.body
|
||||
for field in (
|
||||
"get_conversation_info_list_v2_response_body",
|
||||
"create_conversation_v2_body",
|
||||
):
|
||||
if body.HasField(field):
|
||||
conv_body = getattr(body, field)
|
||||
for conv in conv_body.conversation_info_list:
|
||||
conv_id = conv.conversation_id
|
||||
peer_uid = ""
|
||||
parts = conv_id.split(":")
|
||||
if len(parts) >= 4:
|
||||
peer_uid = parts[-1]
|
||||
label = f"用户{peer_uid[-6:]}" if peer_uid else conv_id
|
||||
results.append({
|
||||
"conversation_id": conv_id,
|
||||
"sender_name": label,
|
||||
"content": "",
|
||||
"unread_count": 0,
|
||||
"peer_uid": peer_uid,
|
||||
"conversation_short_id": str(conv.conversation_short_id),
|
||||
"ticket": conv.ticket,
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return results
|
||||
|
||||
|
||||
def normalize_im_payload(data: Any, depth: int = 0) -> list[dict]:
|
||||
"""递归标准化 IM JSON 为 {sender_name, content, conversation_id, unread_count}"""
|
||||
if depth > 12:
|
||||
return []
|
||||
results = []
|
||||
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
results.extend(normalize_im_payload(item, depth + 1))
|
||||
return results
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return results
|
||||
|
||||
sender = (
|
||||
_pick_str(data, "sender_name", "senderName", "nickname", "nick_name", "userName", "peerName")
|
||||
or _pick_nested(data, ("core_info", "user_info", "peer_info"), "nick_name", "nickname", "name")
|
||||
)
|
||||
sender_avatar = _pick_avatar_url(data)
|
||||
content = _pick_message_text(data)
|
||||
msg_type = _coerce_message_type(
|
||||
data.get("message_type") or data.get("messageType") or data.get("msg_type"),
|
||||
MSG_TYPE_TEXT,
|
||||
)
|
||||
# 会话控制/状态更新帧(command_type / ext_data / 系统通知)直接忽略,不当作聊天消息
|
||||
if _is_control_payload(data, msg_type) or _is_control_payload(data.get("content"), msg_type):
|
||||
return results
|
||||
if msg_type != MSG_TYPE_TEXT or (isinstance(data.get("content"), dict)):
|
||||
parsed = parse_incoming_message(data)
|
||||
if parsed:
|
||||
content = parsed
|
||||
elif content and content.startswith("{"):
|
||||
try:
|
||||
parsed = format_im_message(json.loads(content), msg_type)
|
||||
content = serialize_message_content(parsed)
|
||||
except Exception:
|
||||
pass
|
||||
elif content in _NON_TEXT_MESSAGE_MARKERS:
|
||||
parsed = parse_stored_content(content)
|
||||
content = serialize_message_content(parsed)
|
||||
conv_id = _pick_str(
|
||||
data,
|
||||
"conversation_id",
|
||||
"conversationId",
|
||||
"conv_id",
|
||||
"cid",
|
||||
)
|
||||
unread = data.get("unread_count") or data.get("unreadCount") or data.get("unread_cnt") or 0
|
||||
try:
|
||||
unread = int(unread or 0)
|
||||
except (TypeError, ValueError):
|
||||
unread = 0
|
||||
|
||||
if content and len(content) < 500:
|
||||
from_self = data.get("is_self") or data.get("isSelf") or data.get("fromSelf") or data.get("self")
|
||||
if not from_self:
|
||||
raw_content = _extract_raw_content(data) or content
|
||||
results.append({
|
||||
"sender_name": sender or "未知用户",
|
||||
"sender_avatar": sender_avatar or None,
|
||||
"content": content,
|
||||
"raw_content": raw_content,
|
||||
"conversation_id": conv_id or "",
|
||||
"unread_count": unread,
|
||||
"message_type": msg_type,
|
||||
})
|
||||
|
||||
if sender and unread > 0 and not content:
|
||||
results.append({
|
||||
"sender_name": sender,
|
||||
"sender_avatar": sender_avatar or None,
|
||||
"content": "[未读消息]",
|
||||
"raw_content": "[未读消息]",
|
||||
"conversation_id": conv_id or "",
|
||||
"unread_count": unread,
|
||||
"message_type": msg_type,
|
||||
})
|
||||
|
||||
for key in ("conversations", "conversation_list", "data", "messages", "messagesList", "body"):
|
||||
nested = data.get(key)
|
||||
if nested is not None:
|
||||
results.extend(normalize_im_payload(nested, depth + 1))
|
||||
|
||||
for value in data.values():
|
||||
if isinstance(value, (dict, list)):
|
||||
results.extend(normalize_im_payload(value, depth + 1))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _extract_raw_content(data: dict) -> str:
|
||||
for key in ("content", "message", "msg", "lastMessage", "last_msg", "preview", "brief"):
|
||||
val = data.get(key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
return val.strip()
|
||||
if isinstance(val, dict):
|
||||
return json.dumps(val, ensure_ascii=False, separators=(",", ":"))
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_str(data: dict, *keys: str) -> str:
|
||||
for key in keys:
|
||||
val = data.get(key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
return val.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_nested(data: dict, parent_keys: tuple, *child_keys: str) -> str:
|
||||
for pk in parent_keys:
|
||||
nested = data.get(pk)
|
||||
if isinstance(nested, dict):
|
||||
val = _pick_str(nested, *child_keys)
|
||||
if val:
|
||||
return val
|
||||
return ""
|
||||
|
||||
|
||||
def _avatar_from_value(val: Any) -> str:
|
||||
if isinstance(val, str) and val.strip().startswith("http"):
|
||||
return val.strip()
|
||||
if isinstance(val, dict):
|
||||
direct = val.get("url")
|
||||
if isinstance(direct, str) and direct.startswith("http"):
|
||||
return direct.strip()
|
||||
for list_key in ("url_list", "urls"):
|
||||
urls = val.get(list_key)
|
||||
if isinstance(urls, list):
|
||||
for item in urls:
|
||||
if isinstance(item, str) and item.startswith("http"):
|
||||
return item.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_avatar_url(data: dict) -> str:
|
||||
for key in ("avatar_url", "avatarUrl", "head_url", "headUrl", "avatar"):
|
||||
url = _avatar_from_value(data.get(key))
|
||||
if url:
|
||||
return url
|
||||
for thumb_key in ("avatar_thumb", "avatar_medium", "avatar_larger", "avatarThumb"):
|
||||
url = _avatar_from_value(data.get(thumb_key))
|
||||
if url:
|
||||
return url
|
||||
for parent_key in ("core_info", "user_info", "peer_info", "target_user", "conversation_core_info"):
|
||||
nested = data.get(parent_key)
|
||||
if isinstance(nested, dict):
|
||||
url = _pick_avatar_url(nested)
|
||||
if url:
|
||||
return url
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_message_text(data: dict) -> str:
|
||||
for key in (
|
||||
"text",
|
||||
"content",
|
||||
"message",
|
||||
"msg",
|
||||
"lastMessage",
|
||||
"last_msg",
|
||||
"preview",
|
||||
"brief",
|
||||
):
|
||||
val = data.get(key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
return val.strip()
|
||||
if isinstance(val, dict):
|
||||
inner = _pick_str(val, "text", "content", "message")
|
||||
if inner:
|
||||
return inner
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_plain_text(text: str) -> Optional[str]:
|
||||
text = (text or "").strip()
|
||||
if not text or len(text) > 200:
|
||||
return None
|
||||
if text.startswith("{") or text.startswith("["):
|
||||
return None
|
||||
# 过滤明显二进制垃圾
|
||||
printable = sum(1 for c in text if c.isprintable() or c in "\n\r\t")
|
||||
if printable / max(len(text), 1) < 0.8:
|
||||
return None
|
||||
return text
|
||||
|
||||
|
||||
_NON_TEXT_MESSAGE_MARKERS = {
|
||||
"[表情包]",
|
||||
"[语音]",
|
||||
"[图片]",
|
||||
"[视频]",
|
||||
"[未读消息]",
|
||||
}
|
||||
|
||||
|
||||
def should_skip_auto_reply(content: str) -> tuple[bool, str]:
|
||||
"""判断收到的内容是否不适合触发自动回复(如纯点赞/表情互动)。"""
|
||||
from .message_content import is_media_message, message_preview
|
||||
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
return True, "空消息"
|
||||
if is_media_message(text):
|
||||
return True, f"非文本消息({message_preview(text)})"
|
||||
if text in _NON_TEXT_MESSAGE_MARKERS:
|
||||
return True, f"非文本消息({text})"
|
||||
if re.fullmatch(r"(\[赞\])+", text):
|
||||
return True, "表情互动消息(点赞),抖音通常不允许对此类消息自动回复"
|
||||
if re.fullmatch(r"\[[^\]]+\](\[[^\]]+\])*", text) and "http" not in text:
|
||||
inner = re.sub(r"[\[\]]", "", text)
|
||||
if len(inner) <= 20 and not any(ch.isalnum() for ch in inner):
|
||||
return True, f"非文本互动消息({text})"
|
||||
return False, ""
|
||||
@@ -0,0 +1,330 @@
|
||||
"""自动回复内容解析与 IM 消息体构造(文本 / 网址 / 卡片)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Tuple
|
||||
|
||||
from .message_content import (
|
||||
MSG_TYPE_IMAGE,
|
||||
MSG_TYPE_STICKER,
|
||||
MSG_TYPE_TEXT,
|
||||
message_preview,
|
||||
parse_stored_content,
|
||||
serialize_message_content,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_reply_spec(data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
reply_type = data.get("type")
|
||||
if reply_type in ("text", "link", "card", "image", "sticker"):
|
||||
return data
|
||||
return None
|
||||
|
||||
|
||||
def parse_reply_messages(raw: str) -> list[dict[str, Any]]:
|
||||
"""解析规则中的 reply_content,支持单条或多条回复。"""
|
||||
raw = (raw or "").strip()
|
||||
if not raw:
|
||||
return [{"type": "text", "text": ""}]
|
||||
if raw.startswith("{") or raw.startswith("["):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and isinstance(data.get("messages"), list):
|
||||
specs = [_normalize_reply_spec(item) for item in data["messages"]]
|
||||
specs = [item for item in specs if item]
|
||||
if specs:
|
||||
return specs
|
||||
if isinstance(data, list):
|
||||
specs = [_normalize_reply_spec(item) for item in data]
|
||||
specs = [item for item in specs if item]
|
||||
if specs:
|
||||
return specs
|
||||
if isinstance(data, dict):
|
||||
spec = _normalize_reply_spec(data)
|
||||
if spec:
|
||||
return [spec]
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return [{"type": "text", "text": raw}]
|
||||
|
||||
|
||||
def parse_reply_content(raw: str) -> dict[str, Any]:
|
||||
"""解析发送/规则中的 content。纯字符串视为文本,JSON 为结构化消息。"""
|
||||
raw = (raw or "").strip()
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict):
|
||||
spec = _normalize_reply_spec(data)
|
||||
if spec:
|
||||
return spec
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
parsed = parse_stored_content(raw)
|
||||
if parsed.get("type") != "text":
|
||||
return parsed
|
||||
return parse_reply_messages(raw)[0]
|
||||
|
||||
|
||||
def _format_reply_spec(spec: dict[str, Any]) -> str:
|
||||
reply_type = spec.get("type", "text")
|
||||
if reply_type in ("image", "sticker", "voice", "video"):
|
||||
return message_preview(serialize_message_content(spec))
|
||||
if reply_type == "text":
|
||||
return (spec.get("text") or "").strip()
|
||||
if reply_type == "link":
|
||||
text = (spec.get("text") or spec.get("title") or "").strip()
|
||||
url = (spec.get("url") or "").strip()
|
||||
if text and url:
|
||||
return f"{text} → {url}"
|
||||
return text or url
|
||||
if reply_type == "card":
|
||||
title = (spec.get("title") or "").strip()
|
||||
desc = (spec.get("desc") or spec.get("description") or "").strip()
|
||||
page_url = (spec.get("url") or "").strip()
|
||||
if title and page_url:
|
||||
return f"[卡片] {title} → {page_url}"
|
||||
return title or desc or page_url or "[卡片]"
|
||||
return message_preview(serialize_message_content(spec))
|
||||
|
||||
|
||||
def format_reply_display(raw: str) -> str:
|
||||
"""将 reply_content 格式化为日志/列表中的可读摘要。"""
|
||||
preview = message_preview(raw)
|
||||
if preview:
|
||||
return preview
|
||||
specs = parse_reply_messages(raw)
|
||||
parts = [_format_reply_spec(spec) for spec in specs]
|
||||
parts = [part for part in parts if part]
|
||||
if not parts:
|
||||
return (raw or "").strip()
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def serialize_reply_messages(specs: list[dict[str, Any]]) -> str:
|
||||
cleaned = [spec for spec in specs if _normalize_reply_spec(spec)]
|
||||
if not cleaned:
|
||||
cleaned = [{"type": "text", "text": ""}]
|
||||
if len(cleaned) == 1:
|
||||
return serialize_reply_content(cleaned[0])
|
||||
return json.dumps({"messages": cleaned}, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _extract_link_card_media_path(url: str) -> str:
|
||||
value = (url or "").strip()
|
||||
idx = value.find("/api/media/link-cards/")
|
||||
return value[idx:] if idx >= 0 else ""
|
||||
|
||||
|
||||
def expand_card_spec(spec: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""卡片专属发送规则:展开为「封面图片 + 标题/内容/可点击链接文字」两条消息。
|
||||
|
||||
抖音 Web 协议无法发原生合并卡片(type=70 带图→8004/不带图→空白),因此卡片以
|
||||
「图片消息(横幅) + 文本(标题/内容/链接)」组合呈现:图片提供视觉、文本提供可点击跳转。
|
||||
图片消息仅互关用户可收(陌生人会被 8003 拦截,但文本仍可送达,不影响链接触达)。
|
||||
"""
|
||||
title = (spec.get("title") or "").strip()
|
||||
desc = (spec.get("desc") or spec.get("description") or "").strip()
|
||||
target = (
|
||||
spec.get("target_url")
|
||||
or spec.get("url")
|
||||
or spec.get("link_url")
|
||||
or ""
|
||||
).strip()
|
||||
if "localhost" in target or "127.0.0.1" in target:
|
||||
target = (spec.get("target_url") or spec.get("link_url") or "").strip() or target
|
||||
|
||||
cover = (spec.get("image_path") or "").strip()
|
||||
if not cover:
|
||||
cover = _extract_link_card_media_path(spec.get("cover_url") or "")
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
if cover:
|
||||
out.append({"type": "image", "url": cover, "text": "[图片]"})
|
||||
lines = [x for x in (title, desc, target) if x]
|
||||
text = "\n".join(lines) if lines else target
|
||||
if text:
|
||||
out.append({"type": "text", "text": text})
|
||||
if not out:
|
||||
out.append({"type": "text", "text": target})
|
||||
return out
|
||||
|
||||
|
||||
def split_reply_payloads(raw: str) -> list[str]:
|
||||
"""将规则 reply_content 拆成可逐条发送的 payload 列表。卡片单独展开为图片+文本。"""
|
||||
payloads: list[str] = []
|
||||
for spec in parse_reply_messages(raw):
|
||||
if spec.get("type") == "card":
|
||||
payloads.extend(serialize_reply_content(s) for s in expand_card_spec(spec))
|
||||
else:
|
||||
payloads.append(serialize_reply_content(spec))
|
||||
return payloads
|
||||
|
||||
|
||||
def serialize_reply_log(payloads: list[str]) -> str:
|
||||
"""把实际逐条发送的 payload(JSON 字符串)合并为结构化的日志内容。
|
||||
|
||||
单条直接返回该 payload;多条用 {"messages":[...]} 包裹,便于前端逐条渲染
|
||||
(图片/表情正常显示为媒体,而不是被压扁成 "图片" 这样的占位文本)。
|
||||
"""
|
||||
specs: list[dict[str, Any]] = []
|
||||
for payload in payloads:
|
||||
raw = (payload or "").strip()
|
||||
if not raw:
|
||||
continue
|
||||
spec: dict[str, Any] | None = None
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
spec = _normalize_reply_spec(data)
|
||||
except json.JSONDecodeError:
|
||||
spec = None
|
||||
if spec is None:
|
||||
spec = {"type": "text", "text": raw}
|
||||
specs.append(spec)
|
||||
return serialize_reply_messages(specs)
|
||||
|
||||
|
||||
def serialize_reply_content(spec: dict[str, Any]) -> str:
|
||||
return json.dumps(spec, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def build_msg_payload(spec: dict[str, Any]) -> Tuple[dict[str, Any], int]:
|
||||
"""根据回复规格构造 IM msg_content 与 message_type。"""
|
||||
reply_type = spec.get("type", "text")
|
||||
|
||||
if reply_type == "text":
|
||||
text = (spec.get("text") or "").strip()
|
||||
return (
|
||||
{
|
||||
"mention_users": [],
|
||||
"aweType": 700,
|
||||
"richTextInfos": [],
|
||||
"text": text,
|
||||
},
|
||||
7,
|
||||
)
|
||||
|
||||
if reply_type == "link":
|
||||
display = (spec.get("text") or spec.get("title") or "").strip()
|
||||
url = (spec.get("url") or "").strip()
|
||||
if not display:
|
||||
display = url
|
||||
msg_content: dict[str, Any] = {
|
||||
"mention_users": [],
|
||||
"aweType": 700,
|
||||
"richTextInfos": [],
|
||||
"text": display,
|
||||
}
|
||||
if url and display:
|
||||
msg_content["richTextInfos"] = [
|
||||
{
|
||||
"start": 0,
|
||||
"end": len(display),
|
||||
"type": 2,
|
||||
"link": url,
|
||||
"text": display,
|
||||
}
|
||||
]
|
||||
return msg_content, 7
|
||||
|
||||
if reply_type == "card":
|
||||
# 卡片在 split_reply_payloads 阶段已展开为「图片 + 文本」两条,正常不会走到这里。
|
||||
# 兜底:万一收到未展开的卡片 spec,退化为「标题/内容/链接」文本,确保可送达。
|
||||
title = (spec.get("title") or "").strip()
|
||||
desc = (spec.get("desc") or spec.get("description") or "").strip()
|
||||
target = (
|
||||
spec.get("target_url")
|
||||
or spec.get("url")
|
||||
or spec.get("link_url")
|
||||
or ""
|
||||
).strip()
|
||||
lines = [x for x in (title, desc, target) if x]
|
||||
text = "\n".join(lines) if lines else target
|
||||
return (
|
||||
{"mention_users": [], "aweType": 700, "richTextInfos": [], "text": text},
|
||||
MSG_TYPE_TEXT,
|
||||
)
|
||||
|
||||
if reply_type == "image":
|
||||
uri = (spec.get("uri") or "").strip().lstrip("/")
|
||||
url = (spec.get("url") or "").strip()
|
||||
width = spec.get("width")
|
||||
height = spec.get("height")
|
||||
md5 = (spec.get("md5") or "").strip()
|
||||
url_list = spec.get("url_list")
|
||||
if not isinstance(url_list, list):
|
||||
url_list = [url] if url.startswith("http") else []
|
||||
|
||||
clean_urls = [str(u).strip() for u in url_list if str(u).strip().startswith("http")]
|
||||
|
||||
resource_url: dict[str, Any] = {}
|
||||
if uri:
|
||||
resource_url["uri"] = uri
|
||||
if clean_urls:
|
||||
resource_url["url_list"] = clean_urls
|
||||
if md5:
|
||||
resource_url["md5"] = md5
|
||||
if width:
|
||||
try:
|
||||
resource_url["width"] = int(width)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if height:
|
||||
try:
|
||||
resource_url["height"] = int(height)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
msg_content: dict[str, Any] = {
|
||||
"aweType": 2702,
|
||||
"from_gallery": 1,
|
||||
"create_type": 0,
|
||||
}
|
||||
if resource_url:
|
||||
msg_content["resource_url"] = resource_url
|
||||
if uri:
|
||||
msg_content["local_path"] = uri
|
||||
if md5:
|
||||
msg_content["md5"] = md5
|
||||
if width:
|
||||
try:
|
||||
msg_content["cover_width"] = int(width)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if height:
|
||||
try:
|
||||
msg_content["cover_height"] = int(height)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return msg_content, MSG_TYPE_IMAGE
|
||||
|
||||
if reply_type == "sticker":
|
||||
url = (spec.get("url") or "").strip()
|
||||
sticker_id = spec.get("sticker_id") or spec.get("id")
|
||||
msg_content = {
|
||||
"display_name": (spec.get("name") or spec.get("text") or "[表情包]").strip(),
|
||||
}
|
||||
if sticker_id:
|
||||
msg_content["id"] = sticker_id
|
||||
msg_content["sticker_id"] = sticker_id
|
||||
if url:
|
||||
msg_content["static_url"] = {"url_list": [url]}
|
||||
msg_content["animate_url"] = {"url_list": [url]}
|
||||
return msg_content, MSG_TYPE_STICKER
|
||||
|
||||
text = format_reply_display(serialize_reply_content(spec))
|
||||
return (
|
||||
{
|
||||
"mention_users": [],
|
||||
"aweType": 700,
|
||||
"richTextInfos": [],
|
||||
"text": text,
|
||||
},
|
||||
7,
|
||||
)
|
||||
@@ -0,0 +1,352 @@
|
||||
"""Observable per-account serial queue for delayed automatic replies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Awaitable, Callable, Iterable, Optional
|
||||
|
||||
|
||||
logger = logging.getLogger("douyin_im.reply_queue")
|
||||
|
||||
ReplyCallback = Callable[[], Awaitable[Any]]
|
||||
ErrorCallback = Callable[[str, BaseException], None]
|
||||
DetailsMerger = Callable[[dict[str, Any]], dict[str, Any]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _QueueItem:
|
||||
job_id: str
|
||||
due_at: float
|
||||
slot_seconds: float
|
||||
callback: ReplyCallback
|
||||
description: str
|
||||
queued_at: float
|
||||
merge_keys: frozenset[str] = field(default_factory=frozenset)
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
expedited: bool = False
|
||||
|
||||
|
||||
class AccountReplyQueue:
|
||||
"""Run and expose delayed reply jobs for one hosted account.
|
||||
|
||||
A single consumer is the only code path allowed to invoke callbacks. Jobs
|
||||
selected for immediate delivery are moved to an urgent FIFO, so they can
|
||||
never overlap an already active send. Removing a scheduled job also moves
|
||||
every job behind it forward by the removed job's reserved slot.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
account_id: int,
|
||||
on_error: Optional[ErrorCallback] = None,
|
||||
) -> None:
|
||||
self.account_id = account_id
|
||||
self._on_error = on_error
|
||||
self._waiting: list[_QueueItem] = []
|
||||
self._urgent: deque[_QueueItem] = deque()
|
||||
self._active_item: Optional[_QueueItem] = None
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._running = False
|
||||
self._state_lock = asyncio.Lock()
|
||||
self._wake = asyncio.Event()
|
||||
self._tail_due_at = 0.0
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
"""Approximate active + urgent + waiting count for lightweight badges."""
|
||||
return (
|
||||
len(self._waiting)
|
||||
+ len(self._urgent)
|
||||
+ (1 if self._active_item is not None else 0)
|
||||
)
|
||||
|
||||
async def start(self) -> None:
|
||||
async with self._state_lock:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._running = True
|
||||
self._tail_due_at = 0.0
|
||||
self._wake.clear()
|
||||
self._task = asyncio.create_task(
|
||||
self._run(),
|
||||
name=f"account-reply-queue-{self.account_id}",
|
||||
)
|
||||
|
||||
async def enqueue(
|
||||
self,
|
||||
delay_seconds: float,
|
||||
callback: ReplyCallback,
|
||||
description: str = "",
|
||||
details: Optional[dict[str, Any]] = None,
|
||||
merge_key: str = "",
|
||||
merge_keys: Optional[Iterable[str]] = None,
|
||||
) -> int:
|
||||
"""Append one reply job and return its current 1-based queue position."""
|
||||
interval = max(0.0, float(delay_seconds or 0))
|
||||
loop = asyncio.get_running_loop()
|
||||
async with self._state_lock:
|
||||
if not self._running or not self._task or self._task.done():
|
||||
raise RuntimeError("reply queue is not running")
|
||||
due_at = max(loop.time(), self._tail_due_at) + interval
|
||||
self._tail_due_at = due_at
|
||||
self._waiting.append(
|
||||
_QueueItem(
|
||||
job_id=uuid.uuid4().hex,
|
||||
due_at=due_at,
|
||||
slot_seconds=interval,
|
||||
callback=callback,
|
||||
description=description,
|
||||
queued_at=time.time(),
|
||||
merge_keys=self._normalize_merge_keys(
|
||||
merge_keys if merge_keys is not None else merge_key
|
||||
),
|
||||
details=deepcopy(details or {}),
|
||||
)
|
||||
)
|
||||
position = self.pending_count
|
||||
self._wake.set()
|
||||
return position
|
||||
|
||||
@staticmethod
|
||||
def _normalize_merge_keys(value: str | Iterable[str]) -> frozenset[str]:
|
||||
values = [value] if isinstance(value, str) else list(value or [])
|
||||
return frozenset(str(item or "").strip() for item in values if str(item or "").strip())
|
||||
|
||||
@staticmethod
|
||||
def _merge_keys_match(
|
||||
existing: frozenset[str],
|
||||
incoming: frozenset[str],
|
||||
) -> bool:
|
||||
existing_conversations = {key for key in existing if key.startswith("conv:")}
|
||||
incoming_conversations = {key for key in incoming if key.startswith("conv:")}
|
||||
if existing_conversations & incoming_conversations:
|
||||
return True
|
||||
# Two explicit, different conversation IDs must never merge just because
|
||||
# their partial source data happens to expose the same peer identifier.
|
||||
if existing_conversations and incoming_conversations:
|
||||
return False
|
||||
existing_peers = {key for key in existing if key.startswith("peer:")}
|
||||
incoming_peers = {key for key in incoming if key.startswith("peer:")}
|
||||
return bool(existing_peers & incoming_peers)
|
||||
|
||||
async def merge_pending(
|
||||
self,
|
||||
merge_key: str | Iterable[str],
|
||||
details_merger: DetailsMerger,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge details into one queued conversation without changing its slot.
|
||||
|
||||
Only waiting and urgent jobs are mutable. Once the consumer marks a job
|
||||
active, its callback is sealed and a later message must follow the normal
|
||||
new-message path.
|
||||
"""
|
||||
normalized_keys = self._normalize_merge_keys(merge_key)
|
||||
if not normalized_keys:
|
||||
return {"status": "not_found"}
|
||||
|
||||
async with self._state_lock:
|
||||
if not self._running or not self._task or self._task.done():
|
||||
return {"status": "not_running"}
|
||||
|
||||
active_offset = 1 if self._active_item is not None else 0
|
||||
matches: list[tuple[_QueueItem, str, int]] = []
|
||||
|
||||
for index, candidate in enumerate(self._urgent):
|
||||
if self._merge_keys_match(candidate.merge_keys, normalized_keys):
|
||||
matches.append((candidate, "ready", active_offset + index + 1))
|
||||
|
||||
waiting_offset = active_offset + len(self._urgent)
|
||||
for index, candidate in enumerate(self._waiting):
|
||||
if self._merge_keys_match(candidate.merge_keys, normalized_keys):
|
||||
matches.append((candidate, "waiting", waiting_offset + index + 1))
|
||||
|
||||
if not matches:
|
||||
return {"status": "not_found"}
|
||||
|
||||
incoming_conversations = {
|
||||
key for key in normalized_keys if key.startswith("conv:")
|
||||
}
|
||||
if not incoming_conversations:
|
||||
matched_conversations = {
|
||||
key
|
||||
for candidate, _, _ in matches
|
||||
for key in candidate.merge_keys
|
||||
if key.startswith("conv:")
|
||||
}
|
||||
if len(matched_conversations) > 1:
|
||||
return {"status": "not_found", "reason": "ambiguous_peer"}
|
||||
|
||||
item, item_status, position = matches[0]
|
||||
|
||||
merged_details = details_merger(deepcopy(item.details))
|
||||
if not isinstance(merged_details, dict):
|
||||
raise TypeError("reply queue details merger must return a dict")
|
||||
item.details = deepcopy(merged_details)
|
||||
item.merge_keys = frozenset(item.merge_keys | normalized_keys)
|
||||
return {
|
||||
"status": "merged",
|
||||
"job_id": item.job_id,
|
||||
"position": position,
|
||||
"queue_status": item_status,
|
||||
"message_count": int(item.details.get("message_count") or 1),
|
||||
}
|
||||
|
||||
async def snapshot(self) -> list[dict[str, Any]]:
|
||||
"""Return a callback-free management snapshot ordered by execution."""
|
||||
loop = asyncio.get_running_loop()
|
||||
now_mono = loop.time()
|
||||
now_epoch = time.time()
|
||||
async with self._state_lock:
|
||||
ordered: list[tuple[_QueueItem, str]] = []
|
||||
if self._active_item is not None:
|
||||
ordered.append((self._active_item, "sending"))
|
||||
ordered.extend((item, "ready") for item in self._urgent)
|
||||
ordered.extend((item, "waiting") for item in self._waiting)
|
||||
|
||||
result = []
|
||||
for position, (item, status) in enumerate(ordered, start=1):
|
||||
remaining = 0.0 if status != "waiting" else max(0.0, item.due_at - now_mono)
|
||||
scheduled_epoch = now_epoch + max(0.0, item.due_at - now_mono)
|
||||
payload = {
|
||||
"job_id": item.job_id,
|
||||
"account_id": self.account_id,
|
||||
"position": position,
|
||||
"status": status,
|
||||
"expedited": bool(item.expedited),
|
||||
"description": item.description,
|
||||
"interval_seconds": int(round(item.slot_seconds)),
|
||||
"enqueued_at": datetime.fromtimestamp(
|
||||
item.queued_at, tz=timezone.utc
|
||||
).isoformat(),
|
||||
"scheduled_at": datetime.fromtimestamp(
|
||||
scheduled_epoch, tz=timezone.utc
|
||||
).isoformat(),
|
||||
"remaining_seconds": int(max(0, round(remaining))),
|
||||
}
|
||||
# Details are controlled by DouyinImService and never contain callbacks/session data.
|
||||
payload.update(deepcopy(item.details))
|
||||
result.append(payload)
|
||||
return result
|
||||
|
||||
async def send_now(self, job_id: str) -> dict[str, Any]:
|
||||
"""Move one waiting job to the urgent FIFO and free its future slot."""
|
||||
job_id = str(job_id or "").strip()
|
||||
async with self._state_lock:
|
||||
if not self._running or not self._task or self._task.done():
|
||||
return {"status": "not_running", "job_id": job_id}
|
||||
if self._active_item and self._active_item.job_id == job_id:
|
||||
return {"status": "already_sending", "job_id": job_id}
|
||||
if any(item.job_id == job_id for item in self._urgent):
|
||||
return {"status": "already_requested", "job_id": job_id}
|
||||
|
||||
selected_index = next(
|
||||
(index for index, item in enumerate(self._waiting) if item.job_id == job_id),
|
||||
None,
|
||||
)
|
||||
if selected_index is None:
|
||||
return {"status": "not_found", "job_id": job_id}
|
||||
|
||||
item = self._waiting.pop(selected_index)
|
||||
shift_seconds = max(0.0, item.slot_seconds)
|
||||
shifted_count = 0
|
||||
for later in self._waiting[selected_index:]:
|
||||
later.due_at -= shift_seconds
|
||||
shifted_count += 1
|
||||
|
||||
item.due_at = asyncio.get_running_loop().time()
|
||||
item.expedited = True
|
||||
self._urgent.append(item)
|
||||
self._recalculate_tail_due_at()
|
||||
self._wake.set()
|
||||
return {
|
||||
"status": "accepted",
|
||||
"job_id": job_id,
|
||||
"shifted_count": shifted_count,
|
||||
}
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Cancel the active wait/send and discard all remaining jobs."""
|
||||
async with self._state_lock:
|
||||
self._running = False
|
||||
self._wake.set()
|
||||
task = self._task
|
||||
self._task = None
|
||||
|
||||
if task:
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async with self._state_lock:
|
||||
self._waiting.clear()
|
||||
self._urgent.clear()
|
||||
self._active_item = None
|
||||
self._tail_due_at = 0.0
|
||||
self._wake.clear()
|
||||
|
||||
def _recalculate_tail_due_at(self) -> None:
|
||||
scheduled = [item.due_at for item in self._waiting]
|
||||
self._tail_due_at = max(scheduled, default=0.0)
|
||||
|
||||
async def _run(self) -> None:
|
||||
while True:
|
||||
item: Optional[_QueueItem] = None
|
||||
wait_seconds: Optional[float] = None
|
||||
async with self._state_lock:
|
||||
if not self._running:
|
||||
return
|
||||
if self._urgent:
|
||||
item = self._urgent.popleft()
|
||||
elif self._waiting:
|
||||
candidate = self._waiting[0]
|
||||
remaining = candidate.due_at - asyncio.get_running_loop().time()
|
||||
if remaining <= 0:
|
||||
item = self._waiting.pop(0)
|
||||
else:
|
||||
wait_seconds = remaining
|
||||
|
||||
if item is not None:
|
||||
self._active_item = item
|
||||
self._wake.clear()
|
||||
|
||||
if item is None:
|
||||
try:
|
||||
if wait_seconds is None:
|
||||
await self._wake.wait()
|
||||
else:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=wait_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
continue
|
||||
|
||||
try:
|
||||
await item.callback()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"Account %s queued reply failed (%s)",
|
||||
self.account_id,
|
||||
item.description,
|
||||
)
|
||||
if self._on_error:
|
||||
try:
|
||||
self._on_error(item.description, exc)
|
||||
except Exception:
|
||||
logger.debug("Reply queue error callback failed", exc_info=True)
|
||||
finally:
|
||||
async with self._state_lock:
|
||||
if self._active_item is item:
|
||||
self._active_item = None
|
||||
if not self._waiting:
|
||||
self._tail_due_at = 0.0
|
||||
self._wake.set()
|
||||
@@ -0,0 +1,983 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Awaitable, Callable, Optional
|
||||
|
||||
from utils import system_logger
|
||||
from .frontier import ensure_frontier_ws
|
||||
from .http_client import DouyinImHttpClient, format_session_credential_summary
|
||||
from .session import DouyinImSession
|
||||
from .ws_client import DouyinImWsClient
|
||||
from .reply_queue import AccountReplyQueue
|
||||
from .traffic_control import get_traffic_controller
|
||||
|
||||
from .reply_payload import format_reply_display, serialize_reply_log
|
||||
from .conv_util import resolve_peer_uid
|
||||
from .peer_profile import (
|
||||
enrich_conversation_item,
|
||||
fetch_peer_profile,
|
||||
is_generic_peer_name,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("douyin_im.service")
|
||||
|
||||
MatchReplyFn = Callable[[str], Awaitable[Optional[list[str]]]]
|
||||
LogFn = Callable[..., Awaitable[None]]
|
||||
ReceivedLogFn = Callable[..., Awaitable[None]]
|
||||
|
||||
|
||||
class DouyinImService:
|
||||
"""抖音 IM 直连服务:WebSocket 实时监听 + HTTP 轮询 + 自动回复"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session: DouyinImSession,
|
||||
match_reply: MatchReplyFn,
|
||||
log_fn: LogFn,
|
||||
account_id: int,
|
||||
received_log_fn: Optional[ReceivedLogFn] = None,
|
||||
reply_delay_seconds: int = 0,
|
||||
reply_delay_resolver: Optional[Callable[[], Awaitable[int]]] = None,
|
||||
reply_cooldown_seconds: Optional[int] = None,
|
||||
cooldown_resolver: Optional[Callable[[], Awaitable[int]]] = None,
|
||||
refresh_credentials: Optional[Callable[[], Awaitable[bool]]] = None,
|
||||
follow_tick: Optional[Callable[[], Awaitable[None]]] = None,
|
||||
on_session_invalid: Optional[Callable[[str], Awaitable[None]]] = None,
|
||||
):
|
||||
self.session = session
|
||||
self.match_reply = match_reply
|
||||
self.log_fn = log_fn
|
||||
self.received_log_fn = received_log_fn
|
||||
self.account_id = account_id
|
||||
# 由 worker 注入:周期性检测新粉丝并发送关注欢迎语(约每 60s 触发一次)
|
||||
self.follow_tick = follow_tick
|
||||
# 由 worker 注入:检测到 IM 登录失效(INVALID_REQUEST)时回调,用于自动下线
|
||||
self.on_session_invalid = on_session_invalid
|
||||
self._session_invalid_strikes = 0
|
||||
self._session_invalid_fired = False
|
||||
self.reply_delay_seconds = max(0, int(reply_delay_seconds or 0))
|
||||
# 实时解析账号排队间隔:账号专属优先,否则使用系统默认值。
|
||||
self._reply_delay_resolver = reply_delay_resolver
|
||||
self._reply_queue = AccountReplyQueue(
|
||||
account_id=self.account_id,
|
||||
on_error=self._on_reply_queue_error,
|
||||
)
|
||||
# WS 帧与 HTTP 轮询会并发进入;按到达顺序串行完成预处理/入队,确保 FIFO。
|
||||
self._incoming_lock = asyncio.Lock()
|
||||
# 该账号专属冷却秒数;None 表示继承全局系统设置(仅作为无 resolver 时的兜底)
|
||||
self._cooldown_override = (
|
||||
max(0, int(reply_cooldown_seconds)) if reply_cooldown_seconds is not None else None
|
||||
)
|
||||
# 实时解析冷却秒数的回调(账号专属优先,否则全局);优先于 _cooldown_override
|
||||
self._cooldown_resolver = cooldown_resolver
|
||||
# 由 worker 注入:触发后台重新采集 web_protect/keys(刷新 ts_sign),返回是否刷新成功
|
||||
self.refresh_credentials = refresh_credentials
|
||||
self._running = False
|
||||
self._replied_keys: set[str] = set()
|
||||
self._logged_keys: set[str] = set()
|
||||
self._received_logged_keys: set[str] = set()
|
||||
# 每个对话/用户最近一次自动回复的时间戳(monotonic 秒),用于冷却窗口去重
|
||||
self._last_reply_at: dict[str, float] = {}
|
||||
self._conv_previews: dict[str, str] = {}
|
||||
self._conv_names: dict[str, str] = {} # uid/conv_id -> nickname
|
||||
self._conv_meta: dict[str, dict] = {} # conversation_id -> meta
|
||||
self._ws_client: Optional[DouyinImWsClient] = None
|
||||
self.last_error: str = ""
|
||||
|
||||
def _reply_key(self, conversation_key: str, content: str) -> str:
|
||||
return f"{conversation_key}::{content}"
|
||||
|
||||
@staticmethod
|
||||
def _reply_queue_merge_keys(
|
||||
conversation_id: str,
|
||||
peer_uid: str,
|
||||
) -> tuple[str, ...]:
|
||||
"""Return every stable identifier currently known for one conversation."""
|
||||
conversation_id = str(conversation_id or "").strip()
|
||||
peer_uid = str(peer_uid or "").strip()
|
||||
aliases: list[str] = []
|
||||
if conversation_id:
|
||||
aliases.append(f"conv:{conversation_id}")
|
||||
if peer_uid:
|
||||
aliases.append(f"peer:{peer_uid}")
|
||||
return tuple(aliases)
|
||||
|
||||
@staticmethod
|
||||
def _merge_reply_queue_details(
|
||||
existing: dict,
|
||||
*,
|
||||
incoming_content: str,
|
||||
sender_name: str,
|
||||
sender_id: str,
|
||||
sender_avatar: Optional[str],
|
||||
conversation_id: str,
|
||||
) -> dict:
|
||||
"""Append one received message while preserving the task's one reply."""
|
||||
merged = dict(existing or {})
|
||||
contents = merged.get("incoming_contents")
|
||||
if isinstance(contents, list):
|
||||
contents = list(contents)
|
||||
else:
|
||||
contents = []
|
||||
if not contents and "incoming_content" in merged:
|
||||
contents.append(str(merged.get("incoming_content") or ""))
|
||||
|
||||
latest_content = str(incoming_content or "")
|
||||
contents.append(latest_content)
|
||||
merged["incoming_content"] = latest_content
|
||||
merged["incoming_contents"] = contents
|
||||
merged["message_count"] = len(contents)
|
||||
|
||||
if sender_name:
|
||||
merged["sender_name"] = sender_name
|
||||
if sender_id:
|
||||
merged["sender_id"] = sender_id
|
||||
if sender_avatar:
|
||||
merged["sender_avatar"] = sender_avatar
|
||||
if conversation_id:
|
||||
merged["conversation_id"] = conversation_id
|
||||
return merged
|
||||
|
||||
def _cooldown_seconds_sync(self) -> int:
|
||||
"""无 resolver 时的兜底:账号专属优先,否则取全局设置;0 表示关闭。"""
|
||||
if self._cooldown_override is not None:
|
||||
return self._cooldown_override
|
||||
try:
|
||||
from auth.system_settings import get_cached_settings
|
||||
|
||||
return max(0, int(get_cached_settings().auto_reply_cooldown_seconds or 0))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
async def _resolve_cooldown_seconds(self) -> int:
|
||||
"""实时解析冷却秒数:优先用 worker 注入的 resolver(账号优先、否则全局),否则兜底。"""
|
||||
if self._cooldown_resolver is not None:
|
||||
try:
|
||||
return max(0, int(await self._cooldown_resolver() or 0))
|
||||
except Exception as e:
|
||||
logger.debug(f"cooldown resolver failed: {e}")
|
||||
return self._cooldown_seconds_sync()
|
||||
|
||||
def _reply_delay_seconds_sync(self) -> int:
|
||||
"""无 resolver 时解析排队间隔;0 表示不启用排队规则。"""
|
||||
if self.reply_delay_seconds > 0:
|
||||
return self.reply_delay_seconds
|
||||
try:
|
||||
from auth.system_settings import get_cached_settings
|
||||
|
||||
return max(0, int(get_cached_settings().auto_reply_delay_seconds or 0))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
async def _resolve_reply_delay_seconds(self) -> int:
|
||||
"""实时解析账号生效的回复排队间隔。"""
|
||||
if self._reply_delay_resolver is not None:
|
||||
try:
|
||||
return max(0, int(await self._reply_delay_resolver() or 0))
|
||||
except Exception as exc:
|
||||
logger.debug(f"reply delay resolver failed: {exc}")
|
||||
return self._reply_delay_seconds_sync()
|
||||
|
||||
def _on_reply_queue_error(self, description: str, exc: BaseException) -> None:
|
||||
system_logger.record(
|
||||
"账号回复队列执行失败",
|
||||
detail=f"{description or '自动回复任务'}:{exc}",
|
||||
level="error",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
def _peer_in_cooldown(self, peer_key: str, cooldown: int) -> bool:
|
||||
if cooldown <= 0 or not peer_key:
|
||||
return False
|
||||
last = self._last_reply_at.get(peer_key)
|
||||
if last is None:
|
||||
return False
|
||||
return (time.monotonic() - last) < cooldown
|
||||
|
||||
def _resolve_sender_name(self, msg: dict) -> str:
|
||||
sender_uid = str(msg.get("sender_uid") or msg.get("sender_name") or "").strip()
|
||||
conv_id = str(msg.get("conversation_id") or "")
|
||||
name = (msg.get("sender_name") or "").strip()
|
||||
if name and not name.isdigit():
|
||||
return name
|
||||
if sender_uid and self._conv_names.get(sender_uid):
|
||||
return self._conv_names[sender_uid]
|
||||
if conv_id and self._conv_names.get(conv_id):
|
||||
return self._conv_names[conv_id]
|
||||
if sender_uid:
|
||||
return f"用户{sender_uid[-6:]}" if len(sender_uid) > 6 else f"用户{sender_uid}"
|
||||
return "未知用户"
|
||||
|
||||
def _is_self_message(self, msg: dict) -> bool:
|
||||
sender_uid = str(msg.get("sender_uid") or "").strip()
|
||||
if not sender_uid or not self.session.my_uid:
|
||||
return False
|
||||
try:
|
||||
return int(sender_uid) == int(self.session.my_uid)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
async def _resolve_peer_profile(
|
||||
self,
|
||||
conv_id: str,
|
||||
sender_uid: str,
|
||||
sender: str,
|
||||
sender_avatar: str,
|
||||
) -> tuple[str, str, str]:
|
||||
my_uid = int(self.session.my_uid or 0)
|
||||
peer_uid = str(sender_uid or "").strip()
|
||||
if (not peer_uid or not peer_uid.isdigit()) and conv_id and my_uid:
|
||||
resolved = resolve_peer_uid(conv_id, my_uid)
|
||||
if resolved:
|
||||
peer_uid = str(resolved)
|
||||
|
||||
meta = self._conv_meta.get(conv_id, {}) if conv_id else {}
|
||||
name = (sender or meta.get("sender_name") or "").strip()
|
||||
avatar = (sender_avatar or meta.get("sender_avatar") or "").strip()
|
||||
|
||||
if peer_uid and self._conv_names.get(peer_uid):
|
||||
cached_name = self._conv_names[peer_uid]
|
||||
if is_generic_peer_name(name, peer_uid):
|
||||
name = cached_name
|
||||
if conv_id and self._conv_names.get(conv_id) and is_generic_peer_name(name, peer_uid):
|
||||
name = self._conv_names[conv_id]
|
||||
|
||||
if peer_uid and (is_generic_peer_name(name, peer_uid) or not avatar):
|
||||
profile = await fetch_peer_profile(self.session, peer_uid, self.account_id)
|
||||
if profile.get("nickname"):
|
||||
name = profile["nickname"]
|
||||
self._conv_names[peer_uid] = name
|
||||
if profile.get("avatar_url"):
|
||||
avatar = profile["avatar_url"]
|
||||
if profile.get("uid"):
|
||||
peer_uid = str(profile["uid"])
|
||||
|
||||
if not name:
|
||||
name = self._resolve_sender_name(
|
||||
{"sender_uid": peer_uid, "conversation_id": conv_id, "sender_name": sender}
|
||||
)
|
||||
return name, avatar, peer_uid
|
||||
|
||||
async def _fetch_message_by_id(self, conv_id: str, server_message_id: str) -> dict | None:
|
||||
"""按 server_message_id 调 get_by_conversation 拉取该条消息的完整数据
|
||||
(含真实 content / message_type / URL)。命中返回原始消息 dict,否则 None。"""
|
||||
if not conv_id or not server_message_id:
|
||||
return None
|
||||
try:
|
||||
from .auth import DouyinAuth
|
||||
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "message detail fetch"):
|
||||
meta = self._conv_meta.get(conv_id, {})
|
||||
short_id = str(meta.get("conversation_short_id") or "")
|
||||
auth = DouyinAuth.from_im_session(self.session)
|
||||
my_uid = int(self.session.my_uid or 0)
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
if not short_id:
|
||||
peer_uid = resolve_peer_uid(conv_id, my_uid)
|
||||
if peer_uid:
|
||||
_, short_id, _ = await http.get_conversation_info(
|
||||
auth, int(peer_uid), my_uid, conv_id, 0
|
||||
)
|
||||
if short_id:
|
||||
self._conv_meta[conv_id] = {
|
||||
**self._conv_meta.get(conv_id, {}),
|
||||
"conversation_short_id": short_id,
|
||||
}
|
||||
messages = await http.get_conversation_messages(
|
||||
auth, conv_id, int(short_id or 0), limit=20
|
||||
)
|
||||
for m in messages:
|
||||
if str(m.get("server_message_id") or "") == server_message_id:
|
||||
return m
|
||||
except Exception as e:
|
||||
logger.debug(f"_fetch_message_by_id failed: {e}")
|
||||
return None
|
||||
|
||||
async def _enrich_media_content(self, conv_id: str, server_message_id: str, content: str) -> str:
|
||||
"""媒体消息(相册图片/语音/视频)WS 推送 content 为空时,按 server_message_id
|
||||
调 get_by_conversation 拉取真实内容并补全 URL。命中失败则原样返回。"""
|
||||
if not conv_id or not server_message_id or not content:
|
||||
return content
|
||||
try:
|
||||
from .message_content import parse_stored_content, format_im_message, serialize_message_content
|
||||
|
||||
parsed = parse_stored_content(content)
|
||||
mtype = parsed.get("type")
|
||||
if mtype not in ("image", "voice", "video"):
|
||||
return content
|
||||
if parsed.get("url"):
|
||||
return content # 已有 URL(如商店表情/带 url 的图)
|
||||
|
||||
m = await self._fetch_message_by_id(conv_id, server_message_id)
|
||||
if m:
|
||||
real = format_im_message(m.get("content") or "", int(m.get("message_type") or 0))
|
||||
if real.get("url"):
|
||||
enriched = serialize_message_content(real)
|
||||
logger.info(
|
||||
"Enriched media via get_by_conversation: smid=%s type=%s",
|
||||
server_message_id, real.get("type"),
|
||||
)
|
||||
return enriched
|
||||
except Exception as e:
|
||||
logger.debug(f"_enrich_media_content failed: {e}")
|
||||
return content
|
||||
|
||||
async def _handle_incoming(self, msg: dict):
|
||||
# asyncio.Lock 按等待顺序唤醒。锁只覆盖解析、去重、规则匹配与入队;
|
||||
# 未启用排队时,真正的网络发送仍在锁外执行,保持原有并发行为。
|
||||
async with self._incoming_lock:
|
||||
immediate_reply = await self._prepare_incoming(msg)
|
||||
if immediate_reply is not None and self._running:
|
||||
await immediate_reply()
|
||||
|
||||
async def _prepare_incoming(
|
||||
self,
|
||||
msg: dict,
|
||||
) -> Optional[Callable[[], Awaitable[None]]]:
|
||||
if self._is_self_message(msg):
|
||||
return
|
||||
|
||||
conv_id = msg.get("conversation_id") or ""
|
||||
sender_uid = str(msg.get("sender_uid") or "")
|
||||
sender = self._resolve_sender_name(msg)
|
||||
sender_avatar = str(msg.get("sender_avatar") or "").strip()
|
||||
sender, sender_avatar, peer_uid = await self._resolve_peer_profile(
|
||||
conv_id, sender_uid, sender, sender_avatar
|
||||
)
|
||||
content = (msg.get("content") or "").strip()
|
||||
has_raw_ws = "raw_content" in msg
|
||||
raw_incoming = msg.get("raw_content") if has_raw_ws else None
|
||||
ws_message_type = msg.get("message_type")
|
||||
# 每条 WS 消息带唯一 server_message_id:用它去重,避免“同一用户重复发送
|
||||
# 相同文字(如多次‘你好’)被按内容去重而整条丢弃”,这是“有时收不到”的根因。
|
||||
# HTTP 轮询的会话预览没有该 ID,则退回按 内容 去重(避免对同一未读重复回复)。
|
||||
server_message_id = str(msg.get("server_message_id") or "")
|
||||
# WS 仅推送瘦消息(如 type=26)content 为空:按 server_message_id 回 HTTP 拉取
|
||||
# 完整消息,补全 content / message_type,确保「接收到的全部信息」都被记录。
|
||||
if not content and server_message_id and not (raw_incoming or "").strip():
|
||||
real = await self._fetch_message_by_id(conv_id, server_message_id)
|
||||
if real:
|
||||
real_content = (real.get("content") or "").strip()
|
||||
if real_content:
|
||||
raw_incoming = real.get("content")
|
||||
has_raw_ws = True
|
||||
real_type = real.get("message_type")
|
||||
if real_type is not None:
|
||||
ws_message_type = real_type
|
||||
try:
|
||||
from .message_content import format_im_message, serialize_message_content
|
||||
|
||||
parsed = format_im_message(real.get("content") or "", int(real_type or 0))
|
||||
content = serialize_message_content(parsed) if parsed else real_content
|
||||
except Exception:
|
||||
content = real_content
|
||||
logger.info(
|
||||
"Enriched empty WS push via get_by_conversation: smid=%s type=%s",
|
||||
server_message_id, real_type,
|
||||
)
|
||||
# 相册图片/语音等 WS 推送 content 为空,按 server_message_id 拉取真实内容补 URL
|
||||
content = await self._enrich_media_content(conv_id, server_message_id, content)
|
||||
unread = int(msg.get("unread_count") or 0)
|
||||
|
||||
if conv_id:
|
||||
self._conv_meta[conv_id] = {
|
||||
**self._conv_meta.get(conv_id, {}),
|
||||
"conversation_id": conv_id,
|
||||
"sender_name": sender,
|
||||
"sender_avatar": sender_avatar or self._conv_meta.get(conv_id, {}).get("sender_avatar"),
|
||||
"content": content or self._conv_meta.get(conv_id, {}).get("content", ""),
|
||||
"unread_count": unread,
|
||||
"peer_uid": peer_uid,
|
||||
}
|
||||
if sender and peer_uid:
|
||||
self._conv_names[peer_uid] = sender
|
||||
|
||||
if not content and unread <= 0 and raw_incoming is None and not server_message_id:
|
||||
return
|
||||
|
||||
if content == "[未读消息]" and sender in self._conv_previews:
|
||||
content = self._conv_previews.get(sender, content)
|
||||
|
||||
if server_message_id:
|
||||
log_key = f"mid:{server_message_id}"
|
||||
key = f"mid:{server_message_id}"
|
||||
else:
|
||||
# HTTP 会话预览通常没有 message_id;必须带 conversation_id/peer_uid,
|
||||
# 否则两个同名用户发送相同内容会被误判成同一条消息。
|
||||
conversation_key = str(conv_id or peer_uid or sender or "unknown")
|
||||
log_key = self._reply_key(conversation_key, content or "[未读]")
|
||||
key = self._reply_key(conversation_key, content)
|
||||
|
||||
log_kwargs = {
|
||||
"sender_name": sender,
|
||||
"sender_id": peer_uid or conv_id or None,
|
||||
"sender_avatar": sender_avatar or self._conv_meta.get(conv_id, {}).get("sender_avatar"),
|
||||
"message": content or (raw_incoming if raw_incoming is not None else ""),
|
||||
}
|
||||
|
||||
# 接收消息原始日志:WS content 原样落库(瘦推送已回 HTTP 补全为真实 content)
|
||||
if self.received_log_fn and has_raw_ws:
|
||||
recv_key = f"recv:mid:{server_message_id}" if server_message_id else f"recv:{log_key}"
|
||||
if recv_key not in self._received_logged_keys:
|
||||
self._received_logged_keys.add(recv_key)
|
||||
message_type = ws_message_type
|
||||
try:
|
||||
message_type = int(message_type) if message_type is not None else None
|
||||
except (TypeError, ValueError):
|
||||
message_type = None
|
||||
await self.received_log_fn(
|
||||
sender_name=sender,
|
||||
sender_id=peer_uid or conv_id or None,
|
||||
sender_avatar=log_kwargs.get("sender_avatar"),
|
||||
raw_content="" if raw_incoming is None else raw_incoming,
|
||||
conversation_id=conv_id or None,
|
||||
message_type=message_type,
|
||||
server_message_id=server_message_id or None,
|
||||
)
|
||||
|
||||
if log_key not in self._logged_keys and content:
|
||||
self._logged_keys.add(log_key)
|
||||
await self.log_fn(
|
||||
**log_kwargs,
|
||||
reply=None,
|
||||
status="received",
|
||||
)
|
||||
try:
|
||||
from .message_content import format_system_log_message, parse_stored_content
|
||||
|
||||
parsed = parse_stored_content(content)
|
||||
msg_type = parsed.get("type") or "text"
|
||||
detail = format_system_log_message(content)
|
||||
if server_message_id:
|
||||
detail = f"{detail} | mid={server_message_id}"
|
||||
system_logger.record(
|
||||
f"收到{'' if msg_type == 'text' else '['+msg_type+']'}消息:{sender}",
|
||||
detail=detail,
|
||||
level="info",
|
||||
category="recv",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(f"record recv system log failed: {exc}")
|
||||
|
||||
if key in self._replied_keys:
|
||||
return
|
||||
# WS 与 HTTP 轮询可能同时发现同一条消息。检查后立即占位(中间不 await),
|
||||
# 防止延迟排队期间被重复加入发送队列。
|
||||
self._replied_keys.add(key)
|
||||
|
||||
# 同账号、同会话只保留一个尚未发送的回复任务。后续来信只追加到
|
||||
# 原任务详情,不改变它的发送时间、位置或已经匹配好的回复。
|
||||
queue_merge_keys = self._reply_queue_merge_keys(conv_id, peer_uid)
|
||||
if queue_merge_keys and self._running:
|
||||
merge_result = await self._reply_queue.merge_pending(
|
||||
queue_merge_keys,
|
||||
lambda existing: self._merge_reply_queue_details(
|
||||
existing,
|
||||
incoming_content=content or "",
|
||||
sender_name=sender,
|
||||
sender_id=peer_uid or conv_id or "",
|
||||
sender_avatar=log_kwargs.get("sender_avatar"),
|
||||
conversation_id=conv_id,
|
||||
),
|
||||
)
|
||||
if merge_result.get("status") == "merged":
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
message_count = int(merge_result.get("message_count") or 1)
|
||||
logger.info(
|
||||
"Merged message into queued reply for %s on account %s: "
|
||||
"job=%s messages=%s position=%s",
|
||||
sender,
|
||||
self.account_id,
|
||||
merge_result.get("job_id"),
|
||||
message_count,
|
||||
merge_result.get("position"),
|
||||
)
|
||||
system_logger.record(
|
||||
"同一会话消息已合并到回复队列",
|
||||
detail=(
|
||||
f"{sender} 的新消息已并入原任务;当前共 {message_count} 条消息,"
|
||||
"发送时间和队列位置保持不变。"
|
||||
),
|
||||
level="info",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
|
||||
# 收到新消息即尝试自动回复,不按消息类型/托管关系/内容形态过滤
|
||||
replies = await self.match_reply(content if content != "[未读消息]" else "")
|
||||
if not replies:
|
||||
replies = await self.match_reply("")
|
||||
if not replies:
|
||||
await self.log_fn(
|
||||
**log_kwargs,
|
||||
reply=None,
|
||||
status="ignored",
|
||||
error="未配置任何自动回复规则,请在「自动回复规则」中添加至少一条启用规则",
|
||||
)
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
return
|
||||
|
||||
# 冷却窗口:同一用户在设定时间内,无论发多少条消息,只自动回复一次(账号设置优先,否则全局)
|
||||
peer_key = (peer_uid or conv_id or sender or "").strip()
|
||||
cooldown = await self._resolve_cooldown_seconds()
|
||||
if self._peer_in_cooldown(peer_key, cooldown):
|
||||
logger.info(
|
||||
f"Auto-reply to {sender} skipped: within {cooldown}s cooldown window"
|
||||
)
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
system_logger.record(
|
||||
"自动回复已跳过(冷却中)",
|
||||
detail=f"{sender} 在 {cooldown} 秒冷却窗口内重复发送,未重复回复",
|
||||
level="info",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
# 提前标记回复时间,确保冷却窗口内(含延迟期间)的后续消息都被抑制
|
||||
if peer_key and cooldown > 0:
|
||||
self._last_reply_at[peer_key] = time.monotonic()
|
||||
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
|
||||
delay_seconds = await self._resolve_reply_delay_seconds()
|
||||
if not self._running:
|
||||
return
|
||||
|
||||
async def send_reply() -> None:
|
||||
await self._send_auto_reply(
|
||||
sender=sender,
|
||||
content=content,
|
||||
conv_id=conv_id,
|
||||
replies=replies,
|
||||
peer_key=peer_key,
|
||||
cooldown=cooldown,
|
||||
log_kwargs=log_kwargs,
|
||||
)
|
||||
|
||||
if delay_seconds > 0:
|
||||
position = await self._reply_queue.enqueue(
|
||||
delay_seconds,
|
||||
send_reply,
|
||||
description=f"回复 {sender}",
|
||||
details={
|
||||
"sender_name": sender,
|
||||
"sender_id": peer_uid or conv_id or None,
|
||||
"sender_avatar": log_kwargs.get("sender_avatar"),
|
||||
"conversation_id": conv_id or None,
|
||||
"incoming_content": content or "",
|
||||
"incoming_contents": [content or ""],
|
||||
"message_count": 1,
|
||||
"replies": list(replies),
|
||||
},
|
||||
merge_keys=queue_merge_keys,
|
||||
)
|
||||
logger.info(
|
||||
"Queued reply to %s for account %s: position=%s interval=%ss",
|
||||
sender,
|
||||
self.account_id,
|
||||
position,
|
||||
delay_seconds,
|
||||
)
|
||||
system_logger.record(
|
||||
"自动回复已进入账号队列",
|
||||
detail=(
|
||||
f"{sender} 当前排在第 {position} 位;账号生效间隔为 {delay_seconds} 秒,"
|
||||
"账号内计时与排位独立;到点后再进入全局带宽队列逐条投递。"
|
||||
),
|
||||
level="info",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
|
||||
# 账号与系统均未配置排队间隔:跳过排队规则,保持原来的立即回复。
|
||||
return send_reply
|
||||
|
||||
async def _send_auto_reply(
|
||||
self,
|
||||
*,
|
||||
sender: str,
|
||||
content: str,
|
||||
conv_id: str,
|
||||
replies: list[str],
|
||||
peer_key: str,
|
||||
cooldown: int,
|
||||
log_kwargs: dict,
|
||||
) -> None:
|
||||
"""发送一项已匹配的自动回复任务,并记录原有消息/系统日志。"""
|
||||
if not self._running:
|
||||
return
|
||||
reply_displays: list[str] = []
|
||||
sent_any = False
|
||||
send_error = ""
|
||||
meta = self._conv_meta.get(conv_id, {})
|
||||
for index, reply in enumerate(replies):
|
||||
if not self._running:
|
||||
send_error = self.last_error or "托管已停止,后续回复已取消"
|
||||
break
|
||||
if index > 0:
|
||||
await asyncio.sleep(0.6)
|
||||
if not self._running:
|
||||
send_error = self.last_error or "托管已停止,后续回复已取消"
|
||||
break
|
||||
reply_display = format_reply_display(reply)
|
||||
reply_displays.append(reply_display)
|
||||
sent = False
|
||||
if conv_id:
|
||||
sent, resolved = await self._send_text(
|
||||
conv_id,
|
||||
reply,
|
||||
conversation_short_id=str(meta.get("conversation_short_id") or ""),
|
||||
)
|
||||
if sent:
|
||||
if resolved:
|
||||
meta = {**meta, **resolved, "conversation_id": conv_id}
|
||||
self._conv_meta[conv_id] = meta
|
||||
else:
|
||||
send_error = self.last_error or "IM API 发送失败"
|
||||
else:
|
||||
send_error = "缺少会话 ID,无法发送自动回复"
|
||||
if sent:
|
||||
sent_any = True
|
||||
|
||||
combined_display = " | ".join(reply_displays)
|
||||
# 日志里存结构化内容(单条直接存 payload,多条用 {"messages":[...]} 包裹),
|
||||
# 这样图片/表情等媒体回复会被前端渲染为真实媒体,而不是被压成 "图片" 占位文字。
|
||||
reply_log_content = serialize_reply_log(replies)
|
||||
if not sent_any:
|
||||
logger.warning(
|
||||
f"IM API send failed for [{sender}]: {send_error}; reply saved to log only"
|
||||
)
|
||||
# 发送彻底失败:清除冷却时间戳,避免把没收到回复的用户锁在冷却窗口内
|
||||
if peer_key and cooldown > 0:
|
||||
self._last_reply_at.pop(peer_key, None)
|
||||
|
||||
await self.log_fn(
|
||||
**log_kwargs,
|
||||
reply=reply_log_content,
|
||||
status="replied" if sent_any else "failed",
|
||||
error=None if sent_any else (send_error or "IM API 发送失败"),
|
||||
)
|
||||
if sent_any:
|
||||
system_logger.record(
|
||||
"自动回复成功",
|
||||
detail=f"已回复 {sender}:{combined_display}",
|
||||
level="success",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
else:
|
||||
system_logger.record(
|
||||
"自动回复失败",
|
||||
detail=f"回复 {sender} 失败:{send_error}(收到:{content})",
|
||||
level="error",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
logger.info(
|
||||
f"Auto-reply to {sender}: {content!r} -> {combined_display!r} "
|
||||
f"(sent={sent_any}, count={len(replies)})"
|
||||
)
|
||||
|
||||
async def _index_conversations(self, conversations: list[dict]):
|
||||
my_uid = int(self.session.my_uid or 0)
|
||||
for raw in conversations:
|
||||
conv = enrich_conversation_item(raw, my_uid)
|
||||
conv_id = str(conv.get("conversation_id") or "")
|
||||
name = (conv.get("sender_name") or "").strip()
|
||||
avatar = str(conv.get("sender_avatar") or "").strip()
|
||||
peer_uid = str(conv.get("peer_uid") or "")
|
||||
|
||||
if peer_uid and (is_generic_peer_name(name, peer_uid) or not avatar):
|
||||
profile = await fetch_peer_profile(self.session, peer_uid, self.account_id)
|
||||
if profile.get("nickname"):
|
||||
name = profile["nickname"]
|
||||
conv["sender_name"] = name
|
||||
if profile.get("avatar_url"):
|
||||
avatar = profile["avatar_url"]
|
||||
conv["sender_avatar"] = avatar
|
||||
|
||||
if conv_id:
|
||||
self._conv_meta[conv_id] = {
|
||||
**conv,
|
||||
"sender_name": name,
|
||||
"sender_avatar": avatar or None,
|
||||
"peer_uid": peer_uid,
|
||||
}
|
||||
if name:
|
||||
self._conv_names[conv_id] = name
|
||||
if peer_uid and name:
|
||||
self._conv_names[peer_uid] = name
|
||||
|
||||
async def _poll_conversations(self):
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "conversation poll"):
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
unread_total = await http.get_unread_count()
|
||||
if unread_total:
|
||||
logger.info(f"IM unread total: {unread_total}")
|
||||
conversations = await http.get_conversations()
|
||||
await self._index_conversations(conversations)
|
||||
# Message handling may wait in the global send lane. Do not keep one
|
||||
# of the scarce background HTTP slots occupied while that happens.
|
||||
for conv in conversations:
|
||||
unread = int(conv.get("unread_count") or 0)
|
||||
if unread > 0 or conv.get("content"):
|
||||
await self._handle_incoming(conv)
|
||||
|
||||
async def _verify_account_uid(self):
|
||||
"""启动时用 query/user 接口核验账号真实 UID,修正采集端可能取错的 my_uid/device_id。
|
||||
|
||||
采集端从 tea_cache 推断的 my_uid 可能是访客/对方 id,会导致会话列表为 0、
|
||||
创建会话 INVALID_REQUEST。这里在建连前先校正,保证后续所有请求身份正确。
|
||||
"""
|
||||
if getattr(self.session, "uid_verified", False) and self.session.my_uid:
|
||||
return
|
||||
try:
|
||||
from .auth import DouyinAuth
|
||||
auth = DouyinAuth.from_im_session(self.session)
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "account UID verify"):
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
old = int(self.session.my_uid or 0)
|
||||
resolved = await asyncio.to_thread(http._resolve_authoritative_uid, auth)
|
||||
if resolved and old and int(resolved) != old:
|
||||
system_logger.record(
|
||||
"已自动校正账号 UID",
|
||||
detail=f"采集端识别 UID={old},接口核验真实 UID={resolved},已修正后再建立私信连接。",
|
||||
level="info",
|
||||
category="system",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"启动核验账号 UID 失败(沿用采集值):{e}")
|
||||
|
||||
async def run(self):
|
||||
"""主循环:WebSocket + HTTP 轮询"""
|
||||
self._running = True
|
||||
await self._reply_queue.start()
|
||||
await self._verify_account_uid()
|
||||
# ensure_frontier_ws 可能触发签名/HTTP(阻塞),放线程池避免多账号启动时卡死事件循环
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "frontier discovery"):
|
||||
await asyncio.to_thread(ensure_frontier_ws, self.session)
|
||||
has_ws = bool(self.session.frontier_ws_url())
|
||||
cred_summary = format_session_credential_summary(self.session)
|
||||
logger.info(cred_summary)
|
||||
logger.info(
|
||||
f"Starting IM direct service for account {self.account_id} "
|
||||
f"(ws={'yes' if has_ws else 'no'})"
|
||||
)
|
||||
system_logger.record(
|
||||
"私信托管已启动",
|
||||
detail=f"实时接收通道:{'已就绪' if has_ws else '不可用(仅 HTTP 轮询)'}\n{cred_summary}",
|
||||
level="success" if has_ws else "warning",
|
||||
category="system",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
try:
|
||||
from .emoji_pack import ensure_emoji_map, is_fresh
|
||||
|
||||
if not is_fresh():
|
||||
async with controller.background_slot(self.account_id, "emoji preload"):
|
||||
await asyncio.to_thread(ensure_emoji_map, self.session)
|
||||
except Exception as e:
|
||||
logger.debug(f"emoji map preload failed: {e}")
|
||||
|
||||
self._ws_client = DouyinImWsClient(
|
||||
self.session, self._handle_incoming, account_id=self.account_id
|
||||
)
|
||||
await self._ws_client.start()
|
||||
|
||||
try:
|
||||
await self._poll_conversations()
|
||||
except Exception as e:
|
||||
logger.warning(f"Initial conversation poll failed: {e}")
|
||||
system_logger.record(
|
||||
"首次会话轮询失败",
|
||||
detail=f"{e}",
|
||||
level="warning",
|
||||
category="poll",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
loop_count = 0
|
||||
while self._running:
|
||||
# The initial poll above is authoritative. Sleep before the next
|
||||
# recurring tick so startup cannot issue two back-to-back polls.
|
||||
await asyncio.sleep(5)
|
||||
if not self._running:
|
||||
break
|
||||
loop_count += 1
|
||||
try:
|
||||
if loop_count % 3 == 0:
|
||||
await self._poll_conversations()
|
||||
if loop_count % 6 == 0:
|
||||
logger.info(f"IM direct tick #{loop_count} account={self.account_id}")
|
||||
# 关注欢迎语:约每 60s 检测一次新粉丝(独立于私信轮询,失败不影响主循环)
|
||||
if self.follow_tick and loop_count % 12 == 0:
|
||||
try:
|
||||
await self.follow_tick()
|
||||
except Exception as e:
|
||||
logger.error(f"follow welcome tick error: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"IM poll error: {e}")
|
||||
system_logger.record(
|
||||
"会话轮询出错",
|
||||
detail=f"拉取会话/未读时出错:{e}",
|
||||
level="error",
|
||||
category="poll",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
async def stop(self):
|
||||
self._running = False
|
||||
await get_traffic_controller().send_queue.cancel_account(self.account_id)
|
||||
await self._reply_queue.stop()
|
||||
if self._ws_client:
|
||||
await self._ws_client.stop()
|
||||
|
||||
def get_cached_conversations(self) -> list[dict]:
|
||||
"""返回运行中缓存的会话(来自 WS / 轮询)。"""
|
||||
results = []
|
||||
seen = set()
|
||||
for conv_id, meta in self._conv_meta.items():
|
||||
name = (meta.get("sender_name") or "").strip()
|
||||
key = conv_id or name
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
results.append({
|
||||
"conversation_id": conv_id,
|
||||
"sender_name": name or f"会话{conv_id[-8:]}" if conv_id else "未知用户",
|
||||
"sender_avatar": meta.get("sender_avatar") or None,
|
||||
"sender_id": str(meta.get("peer_uid") or meta.get("sender_id") or conv_id or ""),
|
||||
"peer_uid": str(meta.get("peer_uid") or ""),
|
||||
"content": str(meta.get("content") or ""),
|
||||
"unread_count": int(meta.get("unread_count") or 0),
|
||||
})
|
||||
return results
|
||||
|
||||
async def get_reply_queue_snapshot(self) -> list[dict]:
|
||||
"""返回当前账号自动回复队列的可管理快照。"""
|
||||
return await self._reply_queue.snapshot()
|
||||
|
||||
async def send_queued_reply_now(self, job_id: str) -> dict:
|
||||
"""把指定自动回复任务移入账号紧急队列;实际发送仍由单消费者串行执行。"""
|
||||
return await self._reply_queue.send_now(job_id)
|
||||
|
||||
async def _send_text(
|
||||
self,
|
||||
conversation_id: str,
|
||||
content: str,
|
||||
conversation_short_id: str = "",
|
||||
) -> tuple[bool, Optional[dict]]:
|
||||
"""发送一条私信;若因签名凭证失效(7911)失败,刷新 web_protect 后自动重试一次。
|
||||
|
||||
返回 (是否成功, 解析到的会话 meta)。失败原因写入 self.last_error。
|
||||
"""
|
||||
for attempt in range(2):
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
sent = await http.send_text_message(
|
||||
conversation_id,
|
||||
content,
|
||||
conversation_short_id=conversation_short_id,
|
||||
)
|
||||
self.last_error = http.last_error
|
||||
needs_refresh = http.last_send_needs_refresh
|
||||
if sent:
|
||||
resolved = http.last_send_meta.get(conversation_id)
|
||||
self.session.conv_meta.update(http.session.conv_meta)
|
||||
self._session_invalid_strikes = 0 # 发送成功 → 登录有效
|
||||
return True, resolved
|
||||
|
||||
# 仅在“签名凭证失效”时刷新并重试一次
|
||||
if attempt == 0 and needs_refresh and self.refresh_credentials:
|
||||
logger.warning(
|
||||
f"Send hit credential-expiry(7911) for {conversation_id}; "
|
||||
"refreshing web_protect and retrying once..."
|
||||
)
|
||||
try:
|
||||
refreshed = await self.refresh_credentials()
|
||||
except Exception as e:
|
||||
logger.warning(f"refresh_credentials raised: {e}")
|
||||
refreshed = False
|
||||
if refreshed:
|
||||
continue
|
||||
break
|
||||
await self._note_session_invalid(self.last_error)
|
||||
return False, None
|
||||
|
||||
async def _note_session_invalid(self, error: str) -> None:
|
||||
"""根据发送失败原因判断 IM 是否已退出登录;连续 INVALID_REQUEST 即触发自动下线。
|
||||
|
||||
INVALID_REQUEST 来自 create_conversation/发送:会话/签名被抖音判为无效,强相关于「登录失效」。
|
||||
而 8xxx/7xxx 等业务错误(关系/频控/内容)说明请求已到达抖音、登录仍有效,重置计数。
|
||||
"""
|
||||
err = error or ""
|
||||
if "INVALID_REQUEST" not in err:
|
||||
self._session_invalid_strikes = 0
|
||||
return
|
||||
self._session_invalid_strikes += 1
|
||||
if self._session_invalid_strikes < 2 or self._session_invalid_fired:
|
||||
return
|
||||
self._session_invalid_fired = True
|
||||
reason = "IM 会话失效(INVALID_REQUEST),登录可能已退出"
|
||||
logger.warning(
|
||||
f"Account {self.account_id} {reason};连续 {self._session_invalid_strikes} 次 -> 自动下线"
|
||||
)
|
||||
system_logger.record(
|
||||
"IM 登录失效,自动下线",
|
||||
detail=f"{reason}(连续 {self._session_invalid_strikes} 次发送返回 INVALID_REQUEST)。"
|
||||
"请停止托管后用浏览器模式重新登录并打开私信页,再重新启动托管。",
|
||||
level="error",
|
||||
category="auth",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
self._running = False # 让主循环尽快退出
|
||||
if self.on_session_invalid:
|
||||
try:
|
||||
await self.on_session_invalid(reason)
|
||||
except Exception as e:
|
||||
logger.error(f"on_session_invalid handler error: {e}")
|
||||
|
||||
async def send_message(self, conversation_id: str, content: str) -> bool:
|
||||
"""手动发送私信"""
|
||||
from .conv_util import normalize_conversation_id
|
||||
from .auth import DouyinAuth
|
||||
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(
|
||||
self.session.cookie_header(),
|
||||
self.session.web_protect_str,
|
||||
self.session.keys_str,
|
||||
)
|
||||
if getattr(self.session, "uid_verified", False) and self.session.my_uid:
|
||||
my_uid = self.session.my_uid
|
||||
else:
|
||||
my_uid = await asyncio.to_thread(lambda: auth.get_uid()) or self.session.my_uid
|
||||
if my_uid:
|
||||
conversation_id = normalize_conversation_id(conversation_id, my_uid)
|
||||
|
||||
meta = self._conv_meta.get(conversation_id, {})
|
||||
sent, resolved = await self._send_text(
|
||||
conversation_id,
|
||||
content,
|
||||
conversation_short_id=str(meta.get("conversation_short_id") or ""),
|
||||
)
|
||||
if sent and resolved:
|
||||
self._conv_meta[conversation_id] = {
|
||||
**meta,
|
||||
**resolved,
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
return sent
|
||||
@@ -0,0 +1,260 @@
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
IM_TOKEN_COOKIES = ("sessionid", "sessionid_ss")
|
||||
|
||||
|
||||
def is_frontier_ws_url(url: str) -> bool:
|
||||
"""判断是否为抖音 IM frontier 长连接地址。
|
||||
|
||||
真实抓包里 host 可能是 frontier-im.douyin.com,也可能是
|
||||
frontierNN-normal.zijieapi.com 这类内部别名,二者都要认。
|
||||
"""
|
||||
if not url or "token=" not in url:
|
||||
return False
|
||||
return "frontier-im.douyin.com" in url or ("frontier" in url and "zijieapi.com" in url)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DouyinImSession:
|
||||
"""抖音 IM 直连所需会话信息(从 Cookie + 浏览器抓包获得)"""
|
||||
|
||||
cookies: dict = field(default_factory=dict)
|
||||
ws_urls: list = field(default_factory=list)
|
||||
device_id: str = ""
|
||||
web_id: str = ""
|
||||
user_agent: str = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
keys_str: str = ""
|
||||
web_protect_str: str = ""
|
||||
my_uid: int = 0
|
||||
# my_uid 是否已用 query/user 接口核验过(采集端推断的 my_uid 可能取错 tea_cache id)
|
||||
uid_verified: bool = False
|
||||
conv_meta: dict = field(default_factory=dict)
|
||||
# 方案 A:直接复用浏览器抓到的真实 frontier 连接凭证(绕开我们自己推导 token/access_key 不准的问题)
|
||||
sdk_cert: str = "" # bd-ticket-guard 客户端证书(frontier sdk_cert / HTTP client-cert)
|
||||
frontier_ts_sign: str = "" # 抓包得到的新鲜 ts_sign(覆盖 web_protect 里可能已过期的)
|
||||
|
||||
@classmethod
|
||||
def from_storage_state(cls, data: dict, extra: Optional[dict] = None) -> "DouyinImSession":
|
||||
extra = extra or {}
|
||||
cookies = {}
|
||||
cookie_items = data.get("cookies", [])
|
||||
priority_names = set(IM_TOKEN_COOKIES)
|
||||
|
||||
def pick_best_cookie(name: str) -> str:
|
||||
matches = [
|
||||
c for c in cookie_items
|
||||
if c.get("name") == name and c.get("value")
|
||||
]
|
||||
if not matches:
|
||||
return ""
|
||||
matches.sort(
|
||||
key=lambda c: (
|
||||
0 if ".douyin.com" in (c.get("domain") or "") else 1,
|
||||
-len(c.get("value") or ""),
|
||||
)
|
||||
)
|
||||
return matches[0].get("value") or ""
|
||||
|
||||
for item in cookie_items:
|
||||
name = item.get("name")
|
||||
if not name or any(c in name for c in "()[]{}'\"\n \t\\"):
|
||||
continue
|
||||
val = item.get("value") or ""
|
||||
if name in priority_names:
|
||||
best = pick_best_cookie(name)
|
||||
if best:
|
||||
cookies[name] = best
|
||||
elif name not in cookies or len(val) > len(cookies.get(name, "")):
|
||||
cookies[name] = val
|
||||
|
||||
device_id = extra.get("device_id") or cookies.get("device_id") or ""
|
||||
web_id = extra.get("web_id") or ""
|
||||
|
||||
keys_str = extra.get("keys_str") or ""
|
||||
web_protect_str = extra.get("web_protect_str") or ""
|
||||
|
||||
# my_uid 优先级:浏览器实时采集(extra) > storage_state 顶层(凭证采集工具手填/抓取)。
|
||||
# 顶层 my_uid 让导入的明文数字 UID 直接生效,避免后端用加密 uid_tt 解析失败而回退联网查询。
|
||||
def _as_uid(v) -> int:
|
||||
try:
|
||||
return int(str(v).strip())
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
my_uid = _as_uid(extra.get("my_uid")) or _as_uid(data.get("my_uid"))
|
||||
user_agent = str(extra.get("user_agent") or data.get("user_agent") or "").strip()
|
||||
|
||||
if not device_id or not web_id or not keys_str or not web_protect_str or not my_uid:
|
||||
for origin in data.get("origins", []):
|
||||
for entry in origin.get("localStorage", []):
|
||||
name = entry.get("name", "")
|
||||
value = entry.get("value", "")
|
||||
if not value:
|
||||
continue
|
||||
if name == "security-sdk/s_sdk_crypt_sdk" and not keys_str:
|
||||
keys_str = value
|
||||
if name == "security-sdk/s_sdk_sign_data_key/web_protect" and not web_protect_str:
|
||||
web_protect_str = value
|
||||
if "tea_cache_tokens" in name and not web_id:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
web_id = str(
|
||||
parsed.get("web_id")
|
||||
or parsed.get("user_unique_id")
|
||||
or ""
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if name == "web_runtime_security_uid" and not device_id:
|
||||
if str(value or "").isdigit():
|
||||
device_id = value
|
||||
if "tea_cache_tokens" in name and not my_uid:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
uid = parsed.get("user_unique_id")
|
||||
if uid and str(uid).isdigit():
|
||||
my_uid = int(uid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not my_uid:
|
||||
for item in data.get("cookies", []):
|
||||
cname = item.get("name") or ""
|
||||
if cname in ("uid_tt", "uid_tt_ss") and item.get("value"):
|
||||
try:
|
||||
my_uid = int(item.get("value"))
|
||||
break
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if not device_id and my_uid:
|
||||
device_id = str(my_uid)
|
||||
elif not device_id and web_id:
|
||||
device_id = web_id
|
||||
|
||||
ws_urls = list(extra.get("ws_urls") or [])
|
||||
|
||||
# 方案 A:凭证采集工具可携带浏览器抓到的真实 frontier 连接(含 token/sdk_cert/ts_sign)。
|
||||
frontier_ws_url = str(
|
||||
extra.get("frontier_ws_url") or data.get("frontier_ws_url") or ""
|
||||
).strip()
|
||||
sdk_cert = str(extra.get("sdk_cert") or data.get("sdk_cert") or "").strip()
|
||||
frontier_ts_sign = str(
|
||||
extra.get("frontier_ts_sign")
|
||||
or data.get("frontier_ts_sign")
|
||||
or data.get("ts_sign")
|
||||
or ""
|
||||
).strip()
|
||||
if is_frontier_ws_url(frontier_ws_url):
|
||||
# 真实抓包 URL 优先,放在最前面
|
||||
ws_urls = [frontier_ws_url] + [u for u in ws_urls if u != frontier_ws_url]
|
||||
# 从真实 URL 里补抽 sdk_cert / ts_sign(用户只贴了 URL 时)。
|
||||
# 注意:不能用 parse_qs(它会把 + 解成空格,毁掉 base64 证书),用 unquote。
|
||||
def _q(url: str, key: str) -> str:
|
||||
m = re.search(rf"[?&]{re.escape(key)}=([^&\s]+)", url)
|
||||
return unquote(m.group(1)) if m else ""
|
||||
if not sdk_cert:
|
||||
sdk_cert = _q(frontier_ws_url, "sdk_cert")
|
||||
if not frontier_ts_sign:
|
||||
frontier_ts_sign = _q(frontier_ws_url, "ts_sign")
|
||||
|
||||
return cls(
|
||||
cookies=cookies,
|
||||
ws_urls=ws_urls,
|
||||
device_id=str(device_id or ""),
|
||||
web_id=str(web_id or ""),
|
||||
keys_str=keys_str,
|
||||
web_protect_str=web_protect_str,
|
||||
my_uid=my_uid,
|
||||
user_agent=user_agent if user_agent else cls.user_agent,
|
||||
sdk_cert=sdk_cert,
|
||||
frontier_ts_sign=frontier_ts_sign,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"cookies": self.cookies,
|
||||
"ws_urls": self.ws_urls,
|
||||
"device_id": self.device_id,
|
||||
"web_id": self.web_id,
|
||||
"user_agent": self.user_agent,
|
||||
"keys_str": self.keys_str,
|
||||
"web_protect_str": self.web_protect_str,
|
||||
"my_uid": self.my_uid,
|
||||
"conv_meta": self.conv_meta,
|
||||
"sdk_cert": self.sdk_cert,
|
||||
"frontier_ts_sign": self.frontier_ts_sign,
|
||||
"saved_at": time.time(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "DouyinImSession":
|
||||
if not data:
|
||||
return cls()
|
||||
return cls(
|
||||
cookies=data.get("cookies") or {},
|
||||
ws_urls=data.get("ws_urls") or [],
|
||||
device_id=str(data.get("device_id") or ""),
|
||||
web_id=str(data.get("web_id") or ""),
|
||||
user_agent=data.get("user_agent") or cls.user_agent,
|
||||
keys_str=str(data.get("keys_str") or ""),
|
||||
web_protect_str=str(data.get("web_protect_str") or ""),
|
||||
my_uid=int(data.get("my_uid") or 0),
|
||||
conv_meta=dict(data.get("conv_meta") or {}),
|
||||
sdk_cert=str(data.get("sdk_cert") or ""),
|
||||
frontier_ts_sign=str(data.get("frontier_ts_sign") or ""),
|
||||
)
|
||||
|
||||
def cookie_header(self) -> str:
|
||||
parts = []
|
||||
for name, value in self.cookies.items():
|
||||
if name and value is not None:
|
||||
parts.append(f"{name}={value}")
|
||||
return "; ".join(parts)
|
||||
|
||||
def has_login(self) -> bool:
|
||||
login_keys = {
|
||||
"sessionid",
|
||||
"sessionid_ss",
|
||||
"sid_tt",
|
||||
"sid_guard",
|
||||
"passport_auth_status",
|
||||
"odin_tt",
|
||||
}
|
||||
return any(self.cookies.get(k) for k in login_keys)
|
||||
|
||||
def can_direct_im(self) -> bool:
|
||||
"""是否具备 Cookie 直连 IM 的最低条件(无需浏览器)"""
|
||||
return self.has_login() and bool(
|
||||
self.cookies.get("sessionid") or self.cookies.get("sessionid_ss")
|
||||
)
|
||||
|
||||
def frontier_ws_url(self) -> Optional[str]:
|
||||
"""仅返回 IM frontier 地址,忽略浏览器抓到的 bytelink 等无关 WS。"""
|
||||
for url in self.ws_urls:
|
||||
if is_frontier_ws_url(url):
|
||||
return url
|
||||
return None
|
||||
|
||||
def sanitize_ws_urls(self) -> None:
|
||||
self.ws_urls = [url for url in self.ws_urls if is_frontier_ws_url(url)]
|
||||
|
||||
def common_params(self) -> dict[str, str]:
|
||||
return {
|
||||
"aid": "6383",
|
||||
"app_name": "douyin_web",
|
||||
"device_platform": "webapp",
|
||||
"channel": "channel_pc_web",
|
||||
"pc_client_type": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"device_id": self.device_id or self.web_id or "",
|
||||
"webid": self.web_id or self.device_id or "",
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,60 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# NO CHECKED-IN PROTOBUF GENCODE
|
||||
# source: Request.proto
|
||||
# Protobuf Python Version: 5.27.1
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import descriptor_pool as _descriptor_pool
|
||||
from google.protobuf import runtime_version as _runtime_version
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf.internal import builder as _builder
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(
|
||||
_runtime_version.Domain.PUBLIC,
|
||||
5,
|
||||
27,
|
||||
1,
|
||||
'',
|
||||
'Request.proto'
|
||||
)
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rRequest.proto\"&\n\x08\x45xtValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x94\x04\n\x07Request\x12\x0b\n\x03\x63md\x18\x01 \x01(\x05\x12\x13\n\x0bsequence_id\x18\x02 \x01(\x03\x12\x13\n\x0bsdk_version\x18\x03 \x01(\t\x12\r\n\x05token\x18\x04 \x01(\t\x12\r\n\x05refer\x18\x05 \x01(\x05\x12\x12\n\ninbox_type\x18\x06 \x01(\x03\x12\x14\n\x0c\x62uild_number\x18\x07 \x01(\t\x12\x1a\n\x04\x62ody\x18\x08 \x01(\x0b\x32\x0c.RequestBody\x12\x11\n\tdevice_id\x18\t \x01(\t\x12\x0f\n\x07\x63hannel\x18\n \x01(\t\x12\x17\n\x0f\x64\x65vice_platform\x18\x0b \x01(\t\x12\x13\n\x0b\x64\x65vice_type\x18\x0c \x01(\t\x12\x12\n\nos_version\x18\r \x01(\t\x12\x14\n\x0cversion_code\x18\x0e \x01(\t\x12&\n\x07headers\x18\x0f \x03(\x0b\x32\x15.Request.HeadersEntry\x12\x11\n\tconfig_id\x18\x10 \x01(\x05\x12\x1e\n\ntoken_info\x18\x11 \x01(\x0b\x32\n.TokenInfo\x12\x11\n\tauth_type\x18\x12 \x01(\x05\x12\x0b\n\x03\x62iz\x18\x15 \x01(\t\x12\x0e\n\x06\x61\x63\x63\x65ss\x18\x16 \x01(\t\x12\x0f\n\x07ts_sign\x18\x17 \x01(\t\x12\x10\n\x08sdk_cert\x18\x18 \x01(\t\x12\x14\n\x0creuqest_sign\x18\x19 \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xeb\x01\n\x0bRequestBody\x12\x34\n\x11send_message_body\x18\x64 \x01(\x0b\x32\x17.SendMessageRequestBodyH\x00\x12H\n\x1b\x63reate_conversation_v2_body\x18\xe1\x04 \x01(\x0b\x32 .CreateConversationV2RequestBodyH\x00\x12T\n\"get_conversation_info_list_v2_body\x18\xe2\x04 \x01(\x0b\x32%.GetConversationInfoListV2RequestBodyH\x00\x42\x06\n\x04\x62ody\"\xb8\x02\n\x16SendMessageRequestBody\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x02 \x01(\x05\x12\x1d\n\x15\x63onversation_short_id\x18\x03 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x04 \x01(\t\x12\x16\n\x03\x65xt\x18\x05 \x03(\x0b\x32\t.ExtValue\x12\x14\n\x0cmessage_type\x18\x06 \x01(\x05\x12\x0e\n\x06ticket\x18\x07 \x01(\t\x12\x19\n\x11\x63lient_message_id\x18\x08 \x01(\t\x12\x17\n\x0fmentioned_users\x18\t \x03(\x03\x12\x1a\n\x12ignore_badge_count\x18\n \x01(\x08\x12,\n\x0cref_msg_info\x18\x0b \x01(\x0b\x32\x16.ReferencedMessageInfo\"y\n\x15ReferencedMessageInfo\x12\x1b\n\x13original_message_id\x18\x01 \x01(\x03\x12\x1f\n\x17original_message_sender\x18\x02 \x01(\t\x12\"\n\x1aoriginal_message_timestamp\x18\x03 \x01(\x03\"^\n\tTokenInfo\x12\x0f\n\x07mark_id\x18\x01 \x01(\x05\x12\x0c\n\x04type\x18\x02 \x01(\x05\x12\x0e\n\x06\x61pp_id\x18\x03 \x01(\x05\x12\x0f\n\x07user_id\x18\x04 \x01(\x03\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\"\xa2\x02\n\x1f\x43reateConversationV2RequestBody\x12\x19\n\x11\x63onversation_type\x18\x01 \x01(\x05\x12\x14\n\x0cparticipants\x18\x02 \x03(\x03\x12\x12\n\npersistent\x18\x03 \x01(\x08\x12\x15\n\ridempotent_id\x18\x04 \x01(\t\x12\x0c\n\x04name\x18\x05 \x01(\t\x12\x12\n\navatar_url\x18\x06 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x07 \x01(\t\x12=\n\x07\x62iz_ext\x18\x08 \x03(\x0b\x32,.CreateConversationV2RequestBody.BizExtEntry\x1a-\n\x0b\x42izExtEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n$GetConversationInfoListV2RequestBody\x12\x38\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32*.GetConversationInfoListV2ResponseBodyData\"~\n)GetConversationInfoListV2ResponseBodyData\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x1d\n\x15\x63onversation_short_id\x18\x02 \x01(\x03\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x62\x06proto3')
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'Request_pb2', _globals)
|
||||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
DESCRIPTOR._loaded_options = None
|
||||
_globals['_REQUEST_HEADERSENTRY']._loaded_options = None
|
||||
_globals['_REQUEST_HEADERSENTRY']._serialized_options = b'8\001'
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._loaded_options = None
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_options = b'8\001'
|
||||
_globals['_EXTVALUE']._serialized_start=17
|
||||
_globals['_EXTVALUE']._serialized_end=55
|
||||
_globals['_REQUEST']._serialized_start=58
|
||||
_globals['_REQUEST']._serialized_end=590
|
||||
_globals['_REQUEST_HEADERSENTRY']._serialized_start=544
|
||||
_globals['_REQUEST_HEADERSENTRY']._serialized_end=590
|
||||
_globals['_REQUESTBODY']._serialized_start=593
|
||||
_globals['_REQUESTBODY']._serialized_end=828
|
||||
_globals['_SENDMESSAGEREQUESTBODY']._serialized_start=831
|
||||
_globals['_SENDMESSAGEREQUESTBODY']._serialized_end=1143
|
||||
_globals['_REFERENCEDMESSAGEINFO']._serialized_start=1145
|
||||
_globals['_REFERENCEDMESSAGEINFO']._serialized_end=1266
|
||||
_globals['_TOKENINFO']._serialized_start=1268
|
||||
_globals['_TOKENINFO']._serialized_end=1362
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY']._serialized_start=1365
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY']._serialized_end=1655
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_start=1610
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_end=1655
|
||||
_globals['_GETCONVERSATIONINFOLISTV2REQUESTBODY']._serialized_start=1657
|
||||
_globals['_GETCONVERSATIONINFOLISTV2REQUESTBODY']._serialized_end=1753
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODYDATA']._serialized_start=1755
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODYDATA']._serialized_end=1881
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -0,0 +1,46 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# NO CHECKED-IN PROTOBUF GENCODE
|
||||
# source: Response.proto
|
||||
# Protobuf Python Version: 5.27.1
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import descriptor_pool as _descriptor_pool
|
||||
from google.protobuf import runtime_version as _runtime_version
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf.internal import builder as _builder
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(
|
||||
_runtime_version.Domain.PUBLIC,
|
||||
5,
|
||||
27,
|
||||
1,
|
||||
'',
|
||||
'Response.proto'
|
||||
)
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eResponse.proto\"\x82\x01\n\x08Response\x12\x0b\n\x03\x63md\x18\x01 \x01(\x05\x12\x13\n\x0bsequence_id\x18\x02 \x01(\x03\x12\x12\n\nerror_desc\x18\x03 \x01(\t\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\x12\n\ninbox_type\x18\x05 \x01(\x03\x12\x1b\n\x04\x62ody\x18\x06 \x01(\x0b\x32\r.ResponseBody\"\xf8\x01\n\x0cResponseBody\x12\x30\n\x12new_message_notify\x18\xf4\x03 \x01(\x0b\x32\x11.NewMessageNotifyH\x00\x12N\n\x1b\x63reate_conversation_v2_body\x18\xe1\x04 \x01(\x0b\x32&.GetConversationInfoListV2ResponseBodyH\x00\x12^\n+get_conversation_info_list_v2_response_body\x18\xe2\x04 \x01(\x0b\x32&.GetConversationInfoListV2ResponseBodyH\x00\x42\x06\n\x04\x62ody\"z\n\x10NewMessageNotify\x12\x17\n\x0f\x63onversation_id\x18\x02 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x12\x13\n\x0bnotify_type\x18\x04 \x01(\x05\x12\x1d\n\x07message\x18\x05 \x01(\x0b\x32\x0c.MessageBody\"\xd1\x01\n\x0bMessageBody\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x02 \x01(\x05\x12\x19\n\x11server_message_id\x18\x03 \x01(\x03\x12\x1d\n\x15index_in_conversation\x18\x04 \x01(\x03\x12\x1d\n\x15\x63onversation_short_id\x18\x05 \x01(\x03\x12\x14\n\x0cmessage_type\x18\x06 \x01(\x05\x12\x0e\n\x06sender\x18\x07 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x08 \x01(\t\"g\n%GetConversationInfoListV2ResponseBody\x12>\n\x16\x63onversation_info_list\x18\x01 \x03(\x0b\x32\x1e.GetConversationInfoV2Response\"\x82\x01\n\x1dGetConversationInfoV2Response\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x1d\n\x15\x63onversation_short_id\x18\x02 \x01(\x03\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x12\x0e\n\x06ticket\x18\x04 \x01(\tb\x06proto3')
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'Response_pb2', _globals)
|
||||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
DESCRIPTOR._loaded_options = None
|
||||
_globals['_RESPONSE']._serialized_start=19
|
||||
_globals['_RESPONSE']._serialized_end=149
|
||||
_globals['_RESPONSEBODY']._serialized_start=152
|
||||
_globals['_RESPONSEBODY']._serialized_end=400
|
||||
_globals['_NEWMESSAGENOTIFY']._serialized_start=402
|
||||
_globals['_NEWMESSAGENOTIFY']._serialized_end=524
|
||||
_globals['_MESSAGEBODY']._serialized_start=527
|
||||
_globals['_MESSAGEBODY']._serialized_end=736
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODY']._serialized_start=738
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODY']._serialized_end=841
|
||||
_globals['_GETCONVERSATIONINFOV2RESPONSE']._serialized_start=844
|
||||
_globals['_GETCONVERSATIONINFOV2RESPONSE']._serialized_end=974
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,167 @@
|
||||
ea = function() {
|
||||
return (ea = Object.assign || function(e) {
|
||||
for (var t, n = 1, r = arguments.length; n < r; n++)
|
||||
for (var o in t = arguments[n])
|
||||
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
|
||||
return e
|
||||
}
|
||||
).apply(this, arguments)
|
||||
}
|
||||
var H = function(e) {
|
||||
for (var t, n = e.toString(), r = [], o = 0; o < n.length; o++)
|
||||
0 <= (t = n.charCodeAt(o)) && t <= 127 ? r.push(t) : 128 <= t && t <= 2047 ? (r.push(192 | 31 & t >> 6),
|
||||
r.push(128 | 63 & t)) : (2048 <= t && t <= 55295 || 57344 <= t && t <= 65535) && (r.push(224 | 15 & t >> 12),
|
||||
r.push(128 | 63 & t >> 6),
|
||||
r.push(128 | 63 & t));
|
||||
for (var i = 0; i < r.length; i++)
|
||||
r[i] &= 255;
|
||||
return r
|
||||
}
|
||||
V = function(e) {
|
||||
var t = []
|
||||
, n = [];
|
||||
if (void 0 === e)
|
||||
return "";
|
||||
n = H(e);
|
||||
for (var r = 0, o = n.length; r < o; ++r)
|
||||
t.push((5 ^ n[r]).toString(16));
|
||||
return t.join("")
|
||||
}
|
||||
function generate_account_sdk_source_info(){
|
||||
let browserInfo = {
|
||||
"hardwareConcurrency": 20,
|
||||
"webdriver": false,
|
||||
"chromedriver": false,
|
||||
"shelldriver": false,
|
||||
"plugins": 5,
|
||||
"permissions": [
|
||||
{
|
||||
"name": "notifications",
|
||||
"state": "prompt"
|
||||
}
|
||||
],
|
||||
"innerHeight": 1442,
|
||||
"innerWidth": 1166,
|
||||
"outerHeight": 1552,
|
||||
"outerWidth": 2560,
|
||||
"stoargeStatus": {
|
||||
"indexedDB": {
|
||||
"idb": "object",
|
||||
"open": "function",
|
||||
"indexedDB": "object",
|
||||
"IDBKeyRange": "function",
|
||||
"openDatabase": "undefined",
|
||||
"isSafari": false,
|
||||
"hasFetch": false
|
||||
},
|
||||
"localStorage": {
|
||||
"isSupportLStorage": true,
|
||||
"size": 46382,
|
||||
"write": true
|
||||
},
|
||||
"storageQuotaStatus": {
|
||||
"usage": 149822,
|
||||
"quota": 128849645568,
|
||||
"isPrivate": false
|
||||
}
|
||||
},
|
||||
"notificationPermission": "default",
|
||||
"performance": {
|
||||
// "timeOrigin": 1723036093298,
|
||||
"timeOrigin": new Date().getTime(),
|
||||
"usedJSHeapSize": 137509473,
|
||||
"navigationTiming": {
|
||||
"decodedBodySize": 633258,
|
||||
"entryType": "navigation",
|
||||
"initiatorType": "navigation",
|
||||
"name": "https://www.douyin.com/?recommend=1",
|
||||
"renderBlockingStatus": "non-blocking",
|
||||
"serverTiming": "inner,tt_agw,cdn-cache,edge,origin",
|
||||
"guleStart": 1496.2999999523163,
|
||||
"guleDuration": 14.799999952316284
|
||||
}
|
||||
}
|
||||
}
|
||||
let data = {
|
||||
request_host: "www.douyin.com",
|
||||
request_pathname: '/'
|
||||
}
|
||||
return V(JSON.stringify(ea(ea({}, browserInfo || {}), data)))
|
||||
}
|
||||
// console.log(generate_account_sdk_source_info())
|
||||
|
||||
|
||||
eD = function(e, t) {
|
||||
var n, r = 0, o = 0;
|
||||
if ("object" != typeof e || !t || t.length <= 0)
|
||||
return e;
|
||||
for (var i = ej({
|
||||
mix_mode: r
|
||||
}, e), a = 0, c = t.length; a < c; ++a)
|
||||
void 0 !== (n = i[t[a]]) && (r |= 1,
|
||||
o |= 1,
|
||||
i[t[a]] = eN(n));
|
||||
return i.mix_mode = r,
|
||||
i.fixed_mix_mode = o,
|
||||
i
|
||||
}
|
||||
var ek = function(e) {
|
||||
for (var t, n = e.toString(), r = [], o = 0; o < n.length; o++)
|
||||
0 <= (t = n.charCodeAt(o)) && t <= 127 ? r.push(t) : 128 <= t && t <= 2047 ? (r.push(192 | 31 & t >> 6),
|
||||
r.push(128 | 63 & t)) : (2048 <= t && t <= 55295 || 57344 <= t && t <= 65535) && (r.push(224 | 15 & t >> 12),
|
||||
r.push(128 | 63 & t >> 6),
|
||||
r.push(128 | 63 & t));
|
||||
for (var i = 0; i < r.length; i++)
|
||||
r[i] &= 255;
|
||||
return r
|
||||
}
|
||||
eN = function(e) {
|
||||
var t = []
|
||||
, n = [];
|
||||
if (void 0 === e)
|
||||
return "";
|
||||
n = ek(e);
|
||||
for (var r = 0, o = n.length; r < o; ++r)
|
||||
t.push((5 ^ n[r]).toString(16));
|
||||
return t.join("")
|
||||
}
|
||||
ej = function() {
|
||||
return (ej = Object.assign || function(e) {
|
||||
for (var t, n = 1, r = arguments.length; n < r; n++)
|
||||
for (var o in t = arguments[n])
|
||||
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
|
||||
return e
|
||||
}
|
||||
).apply(this, arguments)
|
||||
};
|
||||
function generateSecretPhoneNum(phoneNum) {
|
||||
return eD({
|
||||
"mobile": "+86 " + phoneNum,
|
||||
"type": 24,
|
||||
"is6Digits": 1
|
||||
}, ["mobile", "type"])
|
||||
}
|
||||
// let res = eD({
|
||||
// "mobile": "+86 15751076989",
|
||||
// "type": 24,
|
||||
// "is6Digits": 1
|
||||
// }, ["mobile", "type"])
|
||||
//
|
||||
// console.log(res)
|
||||
nx = function() {
|
||||
return (nx = Object.assign || function(e) {
|
||||
for (var t, n = 1, r = arguments.length; n < r; n++)
|
||||
for (var o in t = arguments[n])
|
||||
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
|
||||
return e
|
||||
}
|
||||
).apply(this, arguments)
|
||||
}
|
||||
function generateSecretCode(phoneNum, code) {
|
||||
return eD({
|
||||
"mobile": "+86 " + phoneNum,
|
||||
"code": code,
|
||||
"service": "https://www.douyin.com"
|
||||
}, ["mobile", "code", "password"])
|
||||
}
|
||||
// console.log(generateSecretCode("15751076989", "090625"))
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "static",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"dependencies": {
|
||||
"jsrsasign": "^11.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/jsrsasign": {
|
||||
"version": "11.1.3",
|
||||
"resolved": "https://registry.npmjs.org/jsrsasign/-/jsrsasign-11.1.3.tgz",
|
||||
"integrity": "sha512-nPnK5D/4lv0Dwr7TlzrKtAd8JlLZwFTqTUUB3NQCbtdobcRcohGFxjbPySDVh74iWUudcCsapYT6OxoyhJLhhA==",
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"jsrsasign": "^11.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,537 @@
|
||||
"""Process-wide traffic control for hosted Douyin accounts.
|
||||
|
||||
The backend normally runs one asyncio event loop. A controller is kept per
|
||||
event loop so unit tests that create a fresh loop for every test do not reuse
|
||||
asyncio primitives bound to an older loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import weakref
|
||||
from collections import deque
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Awaitable, Callable, Optional, TypeVar
|
||||
|
||||
|
||||
logger = logging.getLogger("douyin_im.traffic")
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
|
||||
try:
|
||||
return max(minimum, float(os.getenv(name, str(default))))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, minimum: int = 1) -> int:
|
||||
try:
|
||||
return max(minimum, int(os.getenv(name, str(default))))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class _SendJob:
|
||||
account_id: int
|
||||
operation: Callable[[], Awaitable[Any]]
|
||||
future: asyncio.Future
|
||||
description: str
|
||||
queued_at: float
|
||||
cancelled: bool = False
|
||||
operation_task: Optional[asyncio.Task] = None
|
||||
|
||||
|
||||
class GlobalSendQueue:
|
||||
"""One paced, account-fair dispatcher for every outbound IM write.
|
||||
|
||||
Each account owns a FIFO. After one job is selected, that account goes to
|
||||
the back of the rotation. A busy account therefore cannot starve smaller
|
||||
accounts, while messages within one account retain their original order.
|
||||
"""
|
||||
|
||||
def __init__(self, interval_seconds: float = 1.0) -> None:
|
||||
self.interval_seconds = max(0.0, float(interval_seconds or 0.0))
|
||||
self._queues: dict[int, deque[_SendJob]] = {}
|
||||
self._rotation: deque[int] = deque()
|
||||
self._in_rotation: set[int] = set()
|
||||
self._active_job: Optional[_SendJob] = None
|
||||
self._active_account: Optional[int] = None
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._running = False
|
||||
self._state_lock = asyncio.Lock()
|
||||
self._wake = asyncio.Event()
|
||||
self._next_allowed_at = 0.0
|
||||
|
||||
@property
|
||||
def queued_count(self) -> int:
|
||||
return sum(len(queue) for queue in self._queues.values())
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
return self.queued_count + (1 if self._active_job is not None else 0)
|
||||
|
||||
async def start(self) -> None:
|
||||
async with self._state_lock:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._running = True
|
||||
self._wake.clear()
|
||||
self._task = asyncio.create_task(
|
||||
self._run(),
|
||||
name="douyin-global-send-queue",
|
||||
)
|
||||
|
||||
async def submit(
|
||||
self,
|
||||
account_id: int,
|
||||
operation: Callable[[], Awaitable[T]],
|
||||
description: str = "",
|
||||
) -> T:
|
||||
await self.start()
|
||||
loop = asyncio.get_running_loop()
|
||||
account_key = int(account_id or 0)
|
||||
future: asyncio.Future = loop.create_future()
|
||||
job = _SendJob(
|
||||
account_id=account_key,
|
||||
operation=operation,
|
||||
future=future,
|
||||
description=str(description or "send"),
|
||||
queued_at=time.time(),
|
||||
)
|
||||
|
||||
async with self._state_lock:
|
||||
queue = self._queues.setdefault(account_key, deque())
|
||||
queue.append(job)
|
||||
if account_key != self._active_account and account_key not in self._in_rotation:
|
||||
self._rotation.append(account_key)
|
||||
self._in_rotation.add(account_key)
|
||||
position = self.pending_count
|
||||
self._wake.set()
|
||||
|
||||
if position > 1:
|
||||
logger.info(
|
||||
"Outbound queued account=%s position=%s pending=%s (%s)",
|
||||
account_key,
|
||||
position,
|
||||
position,
|
||||
job.description,
|
||||
)
|
||||
|
||||
try:
|
||||
# Shield lets us remove/cancel the exact queued job when its caller
|
||||
# is cancelled instead of asyncio cancelling only the result future.
|
||||
return await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
active = await self._cancel_job(job)
|
||||
if active:
|
||||
# Once the network operation has begun, return its real result
|
||||
# instead of reporting a timeout and then delivering later.
|
||||
# This also keeps the single lane occupied until a to_thread
|
||||
# upload has truly finished.
|
||||
return await asyncio.shield(future)
|
||||
raise
|
||||
|
||||
async def _cancel_job(self, job: _SendJob) -> bool:
|
||||
async with self._state_lock:
|
||||
if job.cancelled:
|
||||
return bool(job.operation_task and not job.operation_task.done())
|
||||
if (
|
||||
self._active_job is job
|
||||
and job.operation_task is not None
|
||||
):
|
||||
# An active image upload may be running in asyncio.to_thread().
|
||||
# Cancelling its awaiter does not stop the thread and would let
|
||||
# the next send overlap it. Let the active job drain and make
|
||||
# the cancelled caller wait for its authoritative result.
|
||||
return True
|
||||
|
||||
job.cancelled = True
|
||||
queue = self._queues.get(job.account_id)
|
||||
if queue:
|
||||
try:
|
||||
queue.remove(job)
|
||||
except ValueError:
|
||||
pass
|
||||
if not queue:
|
||||
self._queues.pop(job.account_id, None)
|
||||
self._remove_from_rotation(job.account_id)
|
||||
if not job.future.done():
|
||||
job.future.cancel()
|
||||
self._wake.set()
|
||||
return False
|
||||
|
||||
def _remove_from_rotation(self, account_id: int) -> None:
|
||||
if account_id not in self._in_rotation:
|
||||
return
|
||||
self._in_rotation.discard(account_id)
|
||||
try:
|
||||
self._rotation.remove(account_id)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def _pop_next_locked(self) -> Optional[_SendJob]:
|
||||
while self._rotation:
|
||||
account_id = self._rotation.popleft()
|
||||
self._in_rotation.discard(account_id)
|
||||
queue = self._queues.get(account_id)
|
||||
if not queue:
|
||||
self._queues.pop(account_id, None)
|
||||
continue
|
||||
|
||||
while queue and (queue[0].cancelled or queue[0].future.cancelled()):
|
||||
queue.popleft()
|
||||
if not queue:
|
||||
self._queues.pop(account_id, None)
|
||||
continue
|
||||
|
||||
job = queue.popleft()
|
||||
if not queue:
|
||||
self._queues.pop(account_id, None)
|
||||
self._active_job = job
|
||||
self._active_account = account_id
|
||||
return job
|
||||
return None
|
||||
|
||||
async def _finish_job(self, job: _SendJob) -> None:
|
||||
async with self._state_lock:
|
||||
if self._active_job is job:
|
||||
self._active_job = None
|
||||
self._active_account = None
|
||||
|
||||
queue = self._queues.get(job.account_id)
|
||||
while queue and (queue[0].cancelled or queue[0].future.cancelled()):
|
||||
queue.popleft()
|
||||
if queue:
|
||||
if job.account_id not in self._in_rotation:
|
||||
# Reinsert only after the active operation finishes. New
|
||||
# accounts that arrived while it ran get their turn first.
|
||||
self._rotation.append(job.account_id)
|
||||
self._in_rotation.add(job.account_id)
|
||||
else:
|
||||
self._queues.pop(job.account_id, None)
|
||||
self._remove_from_rotation(job.account_id)
|
||||
self._wake.set()
|
||||
|
||||
async def _run(self) -> None:
|
||||
try:
|
||||
while True:
|
||||
async with self._state_lock:
|
||||
if not self._running:
|
||||
return
|
||||
job = self._pop_next_locked()
|
||||
if job is None:
|
||||
self._wake.clear()
|
||||
|
||||
if job is None:
|
||||
await self._wake.wait()
|
||||
continue
|
||||
|
||||
try:
|
||||
remaining = self._next_allowed_at - asyncio.get_running_loop().time()
|
||||
if remaining > 0:
|
||||
await asyncio.sleep(remaining)
|
||||
|
||||
if job.cancelled or job.future.cancelled():
|
||||
continue
|
||||
|
||||
operation_task = asyncio.create_task(job.operation())
|
||||
async with self._state_lock:
|
||||
job.operation_task = operation_task
|
||||
cancelled_before_start = job.cancelled or job.future.cancelled()
|
||||
if cancelled_before_start:
|
||||
operation_task.cancel()
|
||||
|
||||
try:
|
||||
result = await operation_task
|
||||
except asyncio.CancelledError:
|
||||
if not job.future.done():
|
||||
job.future.cancel()
|
||||
if not self._running:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if not job.future.done():
|
||||
job.future.set_exception(exc)
|
||||
else:
|
||||
if not job.future.done():
|
||||
job.future.set_result(result)
|
||||
finally:
|
||||
self._next_allowed_at = (
|
||||
asyncio.get_running_loop().time() + self.interval_seconds
|
||||
)
|
||||
await self._finish_job(job)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
async def snapshot(self) -> dict[str, Any]:
|
||||
async with self._state_lock:
|
||||
per_account = {
|
||||
account_id: len(queue)
|
||||
for account_id, queue in self._queues.items()
|
||||
if queue
|
||||
}
|
||||
return {
|
||||
"running": bool(self._task and not self._task.done()),
|
||||
"pending_count": sum(per_account.values())
|
||||
+ (1 if self._active_job is not None else 0),
|
||||
"queued_count": sum(per_account.values()),
|
||||
"active_account_id": self._active_account,
|
||||
"active_description": (
|
||||
self._active_job.description if self._active_job else ""
|
||||
),
|
||||
"interval_seconds": self.interval_seconds,
|
||||
"per_account": per_account,
|
||||
}
|
||||
|
||||
async def cancel_account(self, account_id: int) -> int:
|
||||
"""Cancel callers for one stopped account without overlapping traffic.
|
||||
|
||||
Waiting jobs are removed. An already-running operation is allowed to
|
||||
drain inside the dispatcher (notably, Python cannot stop a running
|
||||
to_thread image upload), but its caller is released immediately and no
|
||||
later job can start until that drain finishes.
|
||||
"""
|
||||
account_key = int(account_id or 0)
|
||||
cancelled = 0
|
||||
async with self._state_lock:
|
||||
queue = self._queues.pop(account_key, deque())
|
||||
self._remove_from_rotation(account_key)
|
||||
for job in queue:
|
||||
job.cancelled = True
|
||||
if not job.future.done():
|
||||
job.future.cancel()
|
||||
cancelled += 1
|
||||
|
||||
if self._active_job and self._active_job.account_id == account_key:
|
||||
self._active_job.cancelled = True
|
||||
if not self._active_job.future.done():
|
||||
self._active_job.future.cancel()
|
||||
cancelled += 1
|
||||
self._wake.set()
|
||||
return cancelled
|
||||
|
||||
async def stop(self) -> None:
|
||||
drain_timeout = _env_float("KEFU_SEND_SHUTDOWN_DRAIN_SECONDS", 30.0)
|
||||
async with self._state_lock:
|
||||
self._running = False
|
||||
task = self._task
|
||||
self._task = None
|
||||
active_job = self._active_job
|
||||
active_task = active_job.operation_task if active_job else None
|
||||
futures = [
|
||||
job.future
|
||||
for queue in self._queues.values()
|
||||
for job in queue
|
||||
]
|
||||
for queue in self._queues.values():
|
||||
for job in queue:
|
||||
job.cancelled = True
|
||||
if active_job:
|
||||
active_job.cancelled = True
|
||||
futures.append(active_job.future)
|
||||
self._queues.clear()
|
||||
self._rotation.clear()
|
||||
self._in_rotation.clear()
|
||||
self._wake.set()
|
||||
|
||||
for future in futures:
|
||||
if not future.done():
|
||||
future.cancel()
|
||||
|
||||
if active_task and not active_task.done() and drain_timeout > 0:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(active_task),
|
||||
timeout=drain_timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"Active outbound task did not drain within %.1fs during shutdown (%s)",
|
||||
drain_timeout,
|
||||
active_job.description if active_job else "send",
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
# The dispatcher owns normal operation error reporting/results.
|
||||
pass
|
||||
|
||||
if active_task and not active_task.done():
|
||||
active_task.cancel()
|
||||
if task and task is not asyncio.current_task():
|
||||
if not task.done():
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.shield(task), timeout=1.0)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
task.cancel()
|
||||
if not task.done():
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
async with self._state_lock:
|
||||
self._active_job = None
|
||||
self._active_account = None
|
||||
self._next_allowed_at = 0.0
|
||||
self._wake.clear()
|
||||
|
||||
|
||||
class TrafficController:
|
||||
"""Independent lanes for message writes and lower-priority network work."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.send_queue = GlobalSendQueue(
|
||||
interval_seconds=_env_float(
|
||||
"KEFU_GLOBAL_SEND_INTERVAL_SECONDS",
|
||||
1.0,
|
||||
)
|
||||
)
|
||||
self._background = asyncio.Semaphore(
|
||||
_env_int("KEFU_BACKGROUND_NETWORK_CONCURRENCY", 2)
|
||||
)
|
||||
self._browser = asyncio.Semaphore(
|
||||
_env_int("KEFU_BROWSER_START_CONCURRENCY", 1)
|
||||
)
|
||||
self._media_proxy = asyncio.Semaphore(
|
||||
_env_int("KEFU_MEDIA_PROXY_CONCURRENCY", 4)
|
||||
)
|
||||
self._background_owner: contextvars.ContextVar[tuple[Optional[asyncio.Task], int]] = (
|
||||
contextvars.ContextVar(
|
||||
f"douyin_background_owner_{id(self)}",
|
||||
default=(None, 0),
|
||||
)
|
||||
)
|
||||
self.background_waiting = 0
|
||||
self.background_active = 0
|
||||
self.browser_waiting = 0
|
||||
self.browser_active = 0
|
||||
self.media_proxy_active = 0
|
||||
|
||||
@asynccontextmanager
|
||||
async def background_slot(self, account_id: int = 0, description: str = "request"):
|
||||
current_task = asyncio.current_task()
|
||||
owner_task, depth = self._background_owner.get()
|
||||
if owner_task is current_task and depth > 0:
|
||||
token = self._background_owner.set((current_task, depth + 1))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._background_owner.reset(token)
|
||||
return
|
||||
|
||||
started = asyncio.get_running_loop().time()
|
||||
self.background_waiting += 1
|
||||
try:
|
||||
await self._background.acquire()
|
||||
except BaseException:
|
||||
self.background_waiting -= 1
|
||||
raise
|
||||
self.background_waiting -= 1
|
||||
self.background_active += 1
|
||||
token = self._background_owner.set((current_task, 1))
|
||||
waited = asyncio.get_running_loop().time() - started
|
||||
if waited >= 1.0:
|
||||
logger.info(
|
||||
"Background request dequeued account=%s waited=%.2fs (%s)",
|
||||
account_id,
|
||||
waited,
|
||||
description,
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._background_owner.reset(token)
|
||||
self.background_active -= 1
|
||||
self._background.release()
|
||||
|
||||
@asynccontextmanager
|
||||
async def browser_slot(self, account_id: int = 0, description: str = "browser-login"):
|
||||
started = asyncio.get_running_loop().time()
|
||||
self.browser_waiting += 1
|
||||
try:
|
||||
await self._browser.acquire()
|
||||
except BaseException:
|
||||
self.browser_waiting -= 1
|
||||
raise
|
||||
self.browser_waiting -= 1
|
||||
self.browser_active += 1
|
||||
waited = asyncio.get_running_loop().time() - started
|
||||
if waited >= 1.0:
|
||||
logger.info(
|
||||
"Browser task dequeued account=%s waited=%.2fs (%s)",
|
||||
account_id,
|
||||
waited,
|
||||
description,
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.browser_active -= 1
|
||||
self._browser.release()
|
||||
|
||||
@asynccontextmanager
|
||||
async def media_proxy_slot(self):
|
||||
await self._media_proxy.acquire()
|
||||
self.media_proxy_active += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.media_proxy_active -= 1
|
||||
self._media_proxy.release()
|
||||
|
||||
async def snapshot(self) -> dict[str, Any]:
|
||||
return {
|
||||
"send": await self.send_queue.snapshot(),
|
||||
"background": {
|
||||
"active": self.background_active,
|
||||
"waiting": self.background_waiting,
|
||||
},
|
||||
"browser": {
|
||||
"active": self.browser_active,
|
||||
"waiting": self.browser_waiting,
|
||||
},
|
||||
"media_proxy": {"active": self.media_proxy_active},
|
||||
}
|
||||
|
||||
async def stop(self) -> None:
|
||||
await self.send_queue.stop()
|
||||
|
||||
|
||||
_CONTROLLERS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, TrafficController]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
def get_traffic_controller() -> TrafficController:
|
||||
loop = asyncio.get_running_loop()
|
||||
controller = _CONTROLLERS.get(loop)
|
||||
if controller is None:
|
||||
controller = TrafficController()
|
||||
_CONTROLLERS[loop] = controller
|
||||
return controller
|
||||
|
||||
|
||||
async def submit_outbound(
|
||||
account_id: int,
|
||||
operation: Callable[[], Awaitable[T]],
|
||||
description: str = "",
|
||||
) -> T:
|
||||
return await get_traffic_controller().send_queue.submit(
|
||||
account_id,
|
||||
operation,
|
||||
description=description,
|
||||
)
|
||||
|
||||
|
||||
async def shutdown_traffic_controller() -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
controller = _CONTROLLERS.pop(loop, None)
|
||||
if controller is not None:
|
||||
await controller.stop()
|
||||
@@ -0,0 +1,230 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from typing import Awaitable, Callable, Optional
|
||||
|
||||
from websocket import WebSocketApp
|
||||
|
||||
from utils import system_logger
|
||||
from .protocol import parse_ws_payload
|
||||
from .session import DouyinImSession
|
||||
|
||||
logger = logging.getLogger("douyin_im.ws")
|
||||
|
||||
MessageHandler = Callable[[dict], Awaitable[None]]
|
||||
|
||||
|
||||
class DouyinImWsClient:
|
||||
"""直连 frontier-im WebSocket(websocket-client,与 DouYin_Spider 一致)"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session: DouyinImSession,
|
||||
on_message: MessageHandler,
|
||||
account_id: int | None = None,
|
||||
):
|
||||
self.session = session
|
||||
self.on_message = on_message
|
||||
self.account_id = account_id
|
||||
self._running = False
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._ws_app: Optional[WebSocketApp] = None
|
||||
self._ws_lock = threading.Lock()
|
||||
|
||||
async def start(self):
|
||||
url = self.session.frontier_ws_url()
|
||||
if not url:
|
||||
logger.warning("No frontier WebSocket URL captured; WS listener disabled")
|
||||
system_logger.record(
|
||||
"实时接收未启用:未获取到 frontier WebSocket 地址",
|
||||
detail="缺少有效的 device_id 或 sessionid,无法建立实时私信通道,将仅依赖 HTTP 轮询。",
|
||||
level="warning",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
self._running = True
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._task = asyncio.create_task(self._run_loop(url))
|
||||
|
||||
async def stop(self):
|
||||
self._running = False
|
||||
with self._ws_lock:
|
||||
if self._ws_app:
|
||||
try:
|
||||
self._ws_app.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._ws_app = None
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
|
||||
async def _run_ws_thread(self, url: str):
|
||||
"""在独立守护线程中跑 run_forever,直到连接断开/关闭。
|
||||
|
||||
不能用共享默认线程池(run_in_executor(None)/asyncio.to_thread):
|
||||
WS 长连接会永久占用一个池线程,账号数超过池大小(默认 64)后,
|
||||
所有账号的签名/轮询任务被饿死,表现为“启动几十个账号后全部卡死超时”。
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
done = asyncio.Event()
|
||||
error: list[BaseException] = []
|
||||
|
||||
def _runner():
|
||||
try:
|
||||
self._connect_sync(url)
|
||||
except BaseException as e:
|
||||
error.append(e)
|
||||
finally:
|
||||
try:
|
||||
loop.call_soon_threadsafe(done.set)
|
||||
except RuntimeError:
|
||||
pass # 事件循环已关闭
|
||||
|
||||
thread = threading.Thread(
|
||||
target=_runner,
|
||||
name=f"im-ws-{self.account_id or 'na'}",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
try:
|
||||
await done.wait()
|
||||
except asyncio.CancelledError:
|
||||
# stop() 会 close ws_app 使 run_forever 退出,线程随之结束
|
||||
raise
|
||||
if error:
|
||||
raise error[0]
|
||||
|
||||
async def _run_loop(self, url: str):
|
||||
retry = 0
|
||||
while self._running:
|
||||
from .frontier import ensure_frontier_ws
|
||||
from .traffic_control import get_traffic_controller
|
||||
|
||||
# ensure_frontier_ws 可能触发签名/HTTP(阻塞),放线程池避免卡事件循环
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id or 0, "websocket prepare"):
|
||||
await asyncio.to_thread(ensure_frontier_ws, self.session)
|
||||
connect_url = self.session.frontier_ws_url() or url
|
||||
try:
|
||||
logger.info(f"Connecting IM WebSocket: {connect_url[:100]}...")
|
||||
await self._run_ws_thread(connect_url)
|
||||
retry = 0
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"IM WebSocket error: {e}")
|
||||
system_logger.record(
|
||||
"实时接收连接异常",
|
||||
detail=f"建立 frontier WebSocket 失败:{e}",
|
||||
level="error",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
if not self._running:
|
||||
break
|
||||
retry += 1
|
||||
# Stable per-account jitter prevents every hosted account from
|
||||
# reconnecting in the same second after a shared network outage.
|
||||
jitter = ((int(self.account_id or 0) * 2654435761) % 5000) / 1000.0
|
||||
wait = min(30.0, 2.0 * retry) + jitter
|
||||
logger.info(f"IM WebSocket reconnect in {wait:.1f}s...")
|
||||
system_logger.record(
|
||||
f"实时接收断开,{wait:.1f}s 后重连",
|
||||
detail="frontier WebSocket 连接已断开,正在自动重连。",
|
||||
level="warning",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
def _connect_sync(self, url: str):
|
||||
if not self._loop:
|
||||
return
|
||||
|
||||
def on_open(_ws):
|
||||
logger.info("IM WebSocket connected")
|
||||
system_logger.record(
|
||||
"实时接收通道已连接",
|
||||
detail="frontier WebSocket 已建立,可实时接收私信。",
|
||||
level="success",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
def on_message(_ws, message):
|
||||
asyncio.run_coroutine_threadsafe(self._dispatch(message), self._loop)
|
||||
|
||||
def on_error(_ws, error):
|
||||
if self._running:
|
||||
logger.warning(f"IM WebSocket error: {error}")
|
||||
system_logger.record(
|
||||
"实时接收通道报错",
|
||||
detail=f"{error}",
|
||||
level="error",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
def on_close(_ws, code, msg):
|
||||
logger.info(f"IM WebSocket closed: code={code}, msg={msg}")
|
||||
if self._running:
|
||||
system_logger.record(
|
||||
"实时接收通道关闭",
|
||||
detail=f"code={code}, msg={msg}",
|
||||
level="warning",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
headers = {
|
||||
"User-Agent": self.session.user_agent,
|
||||
"Pragma": "no-cache",
|
||||
"Cache-Control": "no-cache",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Sec-WebSocket-Protocol": "binary, base64, pbbp2",
|
||||
"Sec-WebSocket-Extensions": "permessage-deflate; client_max_window_bits",
|
||||
}
|
||||
ws_app = WebSocketApp(
|
||||
url,
|
||||
header=headers,
|
||||
cookie=self.session.cookie_header(),
|
||||
on_open=on_open,
|
||||
on_message=on_message,
|
||||
on_error=on_error,
|
||||
on_close=on_close,
|
||||
)
|
||||
with self._ws_lock:
|
||||
self._ws_app = ws_app
|
||||
try:
|
||||
ws_app.run_forever(origin="https://www.douyin.com", ping_interval=20, ping_timeout=10)
|
||||
finally:
|
||||
with self._ws_lock:
|
||||
if self._ws_app is ws_app:
|
||||
self._ws_app = None
|
||||
|
||||
async def _dispatch(self, raw):
|
||||
if isinstance(raw, str):
|
||||
payload = raw.encode("utf-8", errors="ignore")
|
||||
else:
|
||||
payload = raw
|
||||
items = parse_ws_payload(payload)
|
||||
for item in items:
|
||||
try:
|
||||
await self.on_message(item)
|
||||
except Exception as e:
|
||||
logger.debug(f"WS message handler error: {e}")
|
||||
system_logger.record(
|
||||
"实时消息处理失败",
|
||||
detail=f"处理收到的私信时出错:{e}",
|
||||
level="error",
|
||||
category="recv",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,129 @@
|
||||
"""运行时环境配置:住宅代理 + 浏览器显示。
|
||||
|
||||
用于解决「部署到云服务器后」两类常见问题:
|
||||
1. 机房 IP 触发抖音风控(7911)—— 通过 KEFU_DOUYIN_PROXY 让抖音请求走住宅代理。
|
||||
2. 无图形界面的 Linux 起不来有头浏览器 —— 自动拉起 Xvfb 虚拟显示。
|
||||
|
||||
全部通过环境变量控制,无需改代码:
|
||||
|
||||
KEFU_DOUYIN_PROXY 抖音 IM HTTP 请求与浏览器登录走的代理,绕开机房 IP 风控。
|
||||
形如 http://user:pass@host:port 或 socks5://host:port
|
||||
KEFU_BROWSER_HEADLESS 是否使用无头浏览器(1/true 开启)。默认 false:抖音安全 SDK
|
||||
对 headless 判定严格,无头易生成无效 ts_sign,反而刷新无效。
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
logger = logging.getLogger("rpa_engine.runtime")
|
||||
|
||||
_TRUE = {"1", "true", "yes", "on"}
|
||||
_FALSE = {"0", "false", "no", "off"}
|
||||
|
||||
_NO_DISPLAY_HINT = (
|
||||
"当前是无图形界面的 Linux 服务器,且无法启动虚拟显示来运行有头浏览器。"
|
||||
"抖音安全 SDK 对 headless 判定严格,扫码登录 / 刷新凭证需要有头 Chromium。请任选其一:\n"
|
||||
" 1) 安装 Xvfb + pyvirtualdisplay,让程序自动拉起虚拟显示:\n"
|
||||
" Debian/Ubuntu: apt install -y xvfb && pip install pyvirtualdisplay\n"
|
||||
" CentOS/Rocky : yum install -y xorg-x11-server-Xvfb && pip install pyvirtualdisplay\n"
|
||||
" 2) 或用 xvfb-run 启动后端:xvfb-run -a ./start_web.sh\n"
|
||||
" 3) 或设置 KEFU_BROWSER_HEADLESS=1 强制无头(更易触发抖音风控,不推荐)。"
|
||||
)
|
||||
|
||||
|
||||
def get_douyin_proxy() -> Optional[str]:
|
||||
"""读取抖音请求代理 URL(未配置返回 None)。"""
|
||||
val = (os.getenv("KEFU_DOUYIN_PROXY") or "").strip()
|
||||
return val or None
|
||||
|
||||
|
||||
def httpx_proxy() -> Optional[str]:
|
||||
"""供 httpx.AsyncClient(proxy=...) 使用的代理 URL。"""
|
||||
return get_douyin_proxy()
|
||||
|
||||
|
||||
def requests_proxies() -> Optional[dict]:
|
||||
"""供 requests.get(proxies=...) 使用的代理字典。"""
|
||||
url = get_douyin_proxy()
|
||||
if not url:
|
||||
return None
|
||||
return {"http": url, "https": url}
|
||||
|
||||
|
||||
def playwright_proxy() -> Optional[dict]:
|
||||
"""转成 Playwright launch(proxy=...) 所需结构(未配置或无法解析返回 None)。"""
|
||||
url = get_douyin_proxy()
|
||||
if not url:
|
||||
return None
|
||||
parsed = urlparse(url)
|
||||
if not parsed.hostname:
|
||||
logger.warning("KEFU_DOUYIN_PROXY 格式无法解析,已忽略:%s", url)
|
||||
return None
|
||||
server = f"{parsed.scheme or 'http'}://{parsed.hostname}"
|
||||
if parsed.port:
|
||||
server += f":{parsed.port}"
|
||||
proxy: dict[str, str] = {"server": server}
|
||||
if parsed.username:
|
||||
proxy["username"] = parsed.username
|
||||
if parsed.password:
|
||||
proxy["password"] = parsed.password
|
||||
return proxy
|
||||
|
||||
|
||||
def resolve_headless(default: bool = False) -> bool:
|
||||
"""根据 KEFU_BROWSER_HEADLESS 决定是否无头;未设置时用 default。"""
|
||||
val = (os.getenv("KEFU_BROWSER_HEADLESS") or "").strip().lower()
|
||||
if val in _TRUE:
|
||||
return True
|
||||
if val in _FALSE:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
# 进程内仅启动一次的虚拟显示(Xvfb)句柄
|
||||
_virtual_display = None
|
||||
_virtual_display_failed = False
|
||||
|
||||
|
||||
def _start_virtual_display_sync() -> Optional[str]:
|
||||
"""在无 DISPLAY 的 Linux 上启动一次 Xvfb 虚拟显示(阻塞,需放线程执行)。"""
|
||||
global _virtual_display, _virtual_display_failed
|
||||
|
||||
# 仅 Linux 且无 DISPLAY 时才需要虚拟显示;Windows/macOS 有桌面,直接返回。
|
||||
if os.name != "posix":
|
||||
return os.environ.get("DISPLAY")
|
||||
if os.environ.get("DISPLAY"):
|
||||
return os.environ["DISPLAY"]
|
||||
if _virtual_display is not None:
|
||||
return os.environ.get("DISPLAY")
|
||||
if _virtual_display_failed:
|
||||
raise RuntimeError(_NO_DISPLAY_HINT)
|
||||
|
||||
try:
|
||||
from pyvirtualdisplay import Display
|
||||
except ImportError as e:
|
||||
_virtual_display_failed = True
|
||||
raise RuntimeError(_NO_DISPLAY_HINT) from e
|
||||
|
||||
try:
|
||||
disp = Display(visible=False, size=(1280, 800))
|
||||
disp.start() # 设置 os.environ['DISPLAY']
|
||||
except Exception as e:
|
||||
_virtual_display_failed = True
|
||||
raise RuntimeError(_NO_DISPLAY_HINT) from e
|
||||
|
||||
_virtual_display = disp
|
||||
logger.info("已启动 Xvfb 虚拟显示 DISPLAY=%s 供有头浏览器使用", os.environ.get("DISPLAY"))
|
||||
return os.environ.get("DISPLAY")
|
||||
|
||||
|
||||
async def ensure_browser_display(headless: bool) -> None:
|
||||
"""有头模式在无 DISPLAY 的 Linux 上自动拉起 Xvfb 虚拟显示。
|
||||
|
||||
headless=True 时无需显示,直接返回;启动失败抛出带操作指引的 RuntimeError。
|
||||
"""
|
||||
if headless:
|
||||
return
|
||||
await asyncio.to_thread(_start_virtual_display_sync)
|
||||
@@ -0,0 +1,21 @@
|
||||
"""应用配置:支持本地开发与 Linux/宝塔 Web 部署。"""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parent
|
||||
PROJECT_ROOT = BACKEND_DIR.parent
|
||||
|
||||
# 前端构建产物目录(npm run build 输出)
|
||||
STATIC_DIR = Path(os.getenv("KEFU_STATIC_DIR", PROJECT_ROOT / "frontend" / "dist"))
|
||||
|
||||
HOST = os.getenv("KEFU_HOST", "0.0.0.0")
|
||||
PORT = int(os.getenv("KEFU_PORT", "8000"))
|
||||
|
||||
# 是否由 FastAPI 托管前端静态文件(单端口 Web 访问)
|
||||
SERVE_WEB = os.getenv("KEFU_SERVE_WEB", "true").lower() in ("1", "true", "yes")
|
||||
|
||||
# CORS:同域部署用默认即可;前后端分离时设置 KEFU_CORS_ORIGINS=https://your.domain
|
||||
_cors_raw = os.getenv("KEFU_CORS_ORIGINS", "*").strip()
|
||||
CORS_ORIGINS: list[str] = (
|
||||
["*"] if _cors_raw == "*" else [o.strip() for o in _cors_raw.split(",") if o.strip()]
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 514 B |
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,239 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
import main
|
||||
|
||||
|
||||
class _RowsResult:
|
||||
def __init__(self, rows):
|
||||
self._rows = list(rows)
|
||||
|
||||
def all(self):
|
||||
return list(self._rows)
|
||||
|
||||
|
||||
def _fake_db(rows):
|
||||
return SimpleNamespace(execute=AsyncMock(return_value=_RowsResult(rows)))
|
||||
|
||||
|
||||
class BatchStartApiTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_start_all_uses_lightweight_select_and_submits_once(self):
|
||||
db = _fake_db([(1, False), (2, True), (3, False), (4, False)])
|
||||
user = SimpleNamespace(id=9, role="admin")
|
||||
submitted_response = {
|
||||
"batch_id": "all-batch",
|
||||
"accepted_count": 2,
|
||||
"complete": False,
|
||||
}
|
||||
submit = AsyncMock(return_value=submitted_response)
|
||||
|
||||
with (
|
||||
patch.object(main.batch_start_queue, "submit", submit),
|
||||
patch.object(
|
||||
main.manager,
|
||||
"is_running",
|
||||
side_effect=lambda account_id: account_id == 3,
|
||||
) as is_running,
|
||||
patch.object(main, "_build_account_response") as build_response,
|
||||
):
|
||||
response = await main.submit_account_start_batch(
|
||||
body=main.BatchStartRequest(all_accounts=True),
|
||||
db=db,
|
||||
user=user,
|
||||
)
|
||||
|
||||
self.assertEqual(response, submitted_response)
|
||||
db.execute.assert_awaited_once()
|
||||
submit.assert_awaited_once_with(
|
||||
[1, 4],
|
||||
owner_id=9,
|
||||
metadata={
|
||||
"requested_count": 4,
|
||||
"accessible_count": 4,
|
||||
"skipped_running_count": 1,
|
||||
"skipped_disabled_count": 1,
|
||||
},
|
||||
)
|
||||
self.assertEqual(is_running.call_count, 3)
|
||||
build_response.assert_not_called()
|
||||
|
||||
statement = db.execute.await_args.args[0]
|
||||
selected_names = [entry.get("name") for entry in statement.column_descriptions]
|
||||
self.assertEqual(selected_names, ["id", "quota_disabled"])
|
||||
self.assertNotIn("cookie_data", str(statement).lower())
|
||||
self.assertNotIn("im_session_data", str(statement).lower())
|
||||
|
||||
async def test_selected_ids_are_deduplicated_scoped_and_filtered(self):
|
||||
db = _fake_db([(5, False), (3, False)])
|
||||
user = SimpleNamespace(id=42, role="operator")
|
||||
submit = AsyncMock(
|
||||
return_value={
|
||||
"batch_id": "selected-batch",
|
||||
"accepted_count": 2,
|
||||
"complete": False,
|
||||
}
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(main.batch_start_queue, "submit", submit),
|
||||
patch.object(main.manager, "is_running", return_value=False),
|
||||
):
|
||||
await main.submit_account_start_batch(
|
||||
body=main.BatchStartRequest(
|
||||
account_ids=[5, 5, 3, -1, 999],
|
||||
all_accounts=False,
|
||||
),
|
||||
db=db,
|
||||
user=user,
|
||||
)
|
||||
|
||||
submit.assert_awaited_once_with(
|
||||
[5, 3],
|
||||
owner_id=42,
|
||||
metadata={
|
||||
"requested_count": 3,
|
||||
"accessible_count": 2,
|
||||
"skipped_running_count": 0,
|
||||
"skipped_disabled_count": 0,
|
||||
},
|
||||
)
|
||||
|
||||
statement = db.execute.await_args.args[0]
|
||||
compiled_params = statement.compile().params
|
||||
parameter_values = list(compiled_params.values())
|
||||
self.assertIn(42, parameter_values)
|
||||
self.assertIn([5, 3, 999], parameter_values)
|
||||
sql = str(statement).lower()
|
||||
self.assertIn("owner_id", sql)
|
||||
self.assertIn("accounts.id in", sql)
|
||||
|
||||
async def test_metadata_counts_inaccessible_running_and_disabled_accounts(self):
|
||||
db = _fake_db([(101, False), (102, True), (103, False)])
|
||||
user = SimpleNamespace(id=7, role="operator")
|
||||
submit = AsyncMock(return_value={"batch_id": "metadata-batch"})
|
||||
|
||||
with (
|
||||
patch.object(main.batch_start_queue, "submit", submit),
|
||||
patch.object(
|
||||
main.manager,
|
||||
"is_running",
|
||||
side_effect=lambda account_id: account_id == 103,
|
||||
),
|
||||
):
|
||||
await main.submit_account_start_batch(
|
||||
body=main.BatchStartRequest(account_ids=[101, 102, 103, 104]),
|
||||
db=db,
|
||||
user=user,
|
||||
)
|
||||
|
||||
submit.assert_awaited_once_with(
|
||||
[101],
|
||||
owner_id=7,
|
||||
metadata={
|
||||
"requested_count": 4,
|
||||
"accessible_count": 3,
|
||||
"skipped_running_count": 1,
|
||||
"skipped_disabled_count": 1,
|
||||
},
|
||||
)
|
||||
|
||||
async def test_batch_lookup_is_isolated_by_owner(self):
|
||||
async def get_batch(batch_id, owner_id, include_items=False):
|
||||
if batch_id == "owned-batch" and owner_id == 11:
|
||||
return {
|
||||
"batch_id": batch_id,
|
||||
"accepted_count": 2,
|
||||
"complete": True,
|
||||
}
|
||||
return None
|
||||
|
||||
lookup = AsyncMock(side_effect=get_batch)
|
||||
with patch.object(main.batch_start_queue, "get_batch", lookup):
|
||||
owned = await main.get_account_start_batch(
|
||||
batch_id="owned-batch",
|
||||
user=SimpleNamespace(id=11, role="operator"),
|
||||
)
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await main.get_account_start_batch(
|
||||
batch_id="owned-batch",
|
||||
user=SimpleNamespace(id=12, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(owned["batch_id"], "owned-batch")
|
||||
self.assertEqual(caught.exception.status_code, 404)
|
||||
self.assertEqual(
|
||||
lookup.await_args_list[0].kwargs,
|
||||
{"owner_id": 11, "include_items": False},
|
||||
)
|
||||
self.assertEqual(
|
||||
lookup.await_args_list[1].kwargs,
|
||||
{"owner_id": 12, "include_items": False},
|
||||
)
|
||||
|
||||
async def test_single_account_start_cancels_queued_batch_job_first(self):
|
||||
account = SimpleNamespace(id=77)
|
||||
db = SimpleNamespace()
|
||||
user = SimpleNamespace(id=5, role="operator")
|
||||
events: list[str] = []
|
||||
|
||||
async def cancel_account(account_id: int):
|
||||
events.append("cancel")
|
||||
return 1
|
||||
|
||||
@asynccontextmanager
|
||||
async def preparation_lock(account_id: int):
|
||||
self.assertEqual(account_id, 77)
|
||||
events.append("lock-enter")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.append("lock-exit")
|
||||
|
||||
async def start_impl(selected_account, selected_db, requested_login_mode):
|
||||
events.append("start")
|
||||
self.assertIs(selected_account, account)
|
||||
self.assertIs(selected_db, db)
|
||||
self.assertEqual(requested_login_mode, "im_direct")
|
||||
return {"status": "starting", "login_mode": "im_direct"}
|
||||
|
||||
cancel = AsyncMock(side_effect=cancel_account)
|
||||
get_owned = AsyncMock(return_value=account)
|
||||
start = AsyncMock(side_effect=start_impl)
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", get_owned),
|
||||
patch.object(main.batch_start_queue, "cancel_account", cancel),
|
||||
patch.object(main.manager, "preparation_lock", side_effect=preparation_lock),
|
||||
patch.object(main, "_start_account_rpa_impl", start),
|
||||
):
|
||||
response = await main.start_account_rpa(
|
||||
account_id=77,
|
||||
body=main.StartAccountRequest(login_mode="im_direct"),
|
||||
db=db,
|
||||
user=user,
|
||||
)
|
||||
|
||||
self.assertEqual(response["status"], "starting")
|
||||
get_owned.assert_awaited_once_with(db, user, 77, write=True)
|
||||
cancel.assert_awaited_once_with(77)
|
||||
start.assert_awaited_once_with(account, db, "im_direct")
|
||||
self.assertEqual(events, ["cancel", "lock-enter", "start", "lock-exit"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,288 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from rpa_engine.batch_start import BatchStartQueue
|
||||
from rpa_engine import batch_start as batch_start_module
|
||||
|
||||
|
||||
class BatchStartQueueTests(unittest.IsolatedAsyncioTestCase):
|
||||
def _make_queue(self, handler, *, concurrency: int = 2) -> BatchStartQueue:
|
||||
queue = BatchStartQueue(handler, concurrency=concurrency)
|
||||
self.addAsyncCleanup(queue.stop)
|
||||
return queue
|
||||
|
||||
async def _wait_for_complete(
|
||||
self,
|
||||
queue: BatchStartQueue,
|
||||
batch_id: str,
|
||||
*,
|
||||
timeout: float = 0.5,
|
||||
) -> dict:
|
||||
async def wait() -> dict:
|
||||
while True:
|
||||
snapshot = await queue.get_batch(batch_id, include_items=True)
|
||||
self.assertIsNotNone(snapshot)
|
||||
if snapshot["complete"]:
|
||||
return snapshot
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
return await asyncio.wait_for(wait(), timeout=timeout)
|
||||
|
||||
async def test_concurrency_limit_is_two(self):
|
||||
active = 0
|
||||
maximum_active = 0
|
||||
started: list[int] = []
|
||||
two_started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
async def handler(account_id: int) -> dict:
|
||||
nonlocal active, maximum_active
|
||||
active += 1
|
||||
maximum_active = max(maximum_active, active)
|
||||
started.append(account_id)
|
||||
if len(started) == 2:
|
||||
two_started.set()
|
||||
try:
|
||||
await release.wait()
|
||||
return {"message": f"started-{account_id}", "login_mode": "im_direct"}
|
||||
finally:
|
||||
active -= 1
|
||||
|
||||
queue = self._make_queue(handler, concurrency=2)
|
||||
submitted = await queue.submit(list(range(1, 9)))
|
||||
|
||||
await asyncio.wait_for(two_started.wait(), timeout=0.2)
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
self.assertEqual(len(started), 2)
|
||||
self.assertEqual(maximum_active, 2)
|
||||
self.assertEqual(active, 2)
|
||||
|
||||
release.set()
|
||||
completed = await self._wait_for_complete(queue, submitted["batch_id"])
|
||||
|
||||
self.assertEqual(completed["submitted_count"], 8)
|
||||
self.assertEqual(completed["failed_count"], 0)
|
||||
self.assertEqual(maximum_active, 2)
|
||||
self.assertEqual(active, 0)
|
||||
|
||||
async def test_submit_returns_while_handler_is_blocked(self):
|
||||
handler_started = asyncio.Event()
|
||||
release_handler = asyncio.Event()
|
||||
handler_finished = asyncio.Event()
|
||||
|
||||
async def handler(account_id: int) -> dict:
|
||||
handler_started.set()
|
||||
await release_handler.wait()
|
||||
handler_finished.set()
|
||||
return {"message": f"started-{account_id}"}
|
||||
|
||||
queue = self._make_queue(handler, concurrency=1)
|
||||
|
||||
submitted = await asyncio.wait_for(queue.submit([11]), timeout=0.2)
|
||||
|
||||
self.assertEqual(submitted["accepted_count"], 1)
|
||||
self.assertEqual(submitted["queued_count"], 1)
|
||||
self.assertFalse(submitted["complete"])
|
||||
self.assertFalse(handler_finished.is_set())
|
||||
|
||||
await asyncio.wait_for(handler_started.wait(), timeout=0.2)
|
||||
processing = await queue.get_batch(submitted["batch_id"])
|
||||
self.assertEqual(processing["processing_count"], 1)
|
||||
self.assertFalse(handler_finished.is_set())
|
||||
|
||||
release_handler.set()
|
||||
completed = await self._wait_for_complete(queue, submitted["batch_id"])
|
||||
self.assertTrue(handler_finished.is_set())
|
||||
self.assertEqual(completed["submitted_count"], 1)
|
||||
|
||||
async def test_duplicate_ids_and_overlapping_batches_are_deduplicated(self):
|
||||
first_started = asyncio.Event()
|
||||
release_first = asyncio.Event()
|
||||
calls: list[int] = []
|
||||
|
||||
async def handler(account_id: int) -> dict:
|
||||
calls.append(account_id)
|
||||
if account_id == 7:
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
return {"message": f"started-{account_id}"}
|
||||
|
||||
queue = self._make_queue(handler, concurrency=1)
|
||||
first = await queue.submit([7, 7])
|
||||
self.assertEqual(first["total_count"], 1)
|
||||
self.assertEqual(first["accepted_count"], 1)
|
||||
await asyncio.wait_for(first_started.wait(), timeout=0.2)
|
||||
|
||||
overlapping = await queue.submit([7, 8, 8])
|
||||
|
||||
self.assertEqual(overlapping["total_count"], 2)
|
||||
self.assertEqual(overlapping["accepted_count"], 1)
|
||||
self.assertEqual(overlapping["skipped_count"], 1)
|
||||
by_account = {item["account_id"]: item for item in overlapping["items"]}
|
||||
self.assertEqual(by_account[7]["status"], "already_queued")
|
||||
self.assertEqual(by_account[8]["status"], "queued")
|
||||
|
||||
release_first.set()
|
||||
await self._wait_for_complete(queue, first["batch_id"])
|
||||
completed_overlap = await self._wait_for_complete(
|
||||
queue,
|
||||
overlapping["batch_id"],
|
||||
)
|
||||
|
||||
self.assertEqual(calls, [7, 8])
|
||||
self.assertEqual(completed_overlap["submitted_count"], 1)
|
||||
self.assertEqual(completed_overlap["skipped_count"], 1)
|
||||
|
||||
async def test_failure_does_not_block_following_accounts(self):
|
||||
calls: list[int] = []
|
||||
|
||||
async def handler(account_id: int) -> dict:
|
||||
calls.append(account_id)
|
||||
if account_id == 31:
|
||||
raise RuntimeError("expected startup failure")
|
||||
return {"message": f"started-{account_id}"}
|
||||
|
||||
queue = self._make_queue(handler, concurrency=1)
|
||||
with patch.object(batch_start_module.logger, "exception"):
|
||||
submitted = await queue.submit([31, 32, 33])
|
||||
completed = await self._wait_for_complete(queue, submitted["batch_id"])
|
||||
|
||||
self.assertEqual(calls, [31, 32, 33])
|
||||
self.assertEqual(completed["failed_count"], 1)
|
||||
self.assertEqual(completed["submitted_count"], 2)
|
||||
by_account = {item["account_id"]: item for item in completed["items"]}
|
||||
self.assertEqual(by_account[31]["status"], "failed")
|
||||
self.assertEqual(by_account[31]["message"], "expected startup failure")
|
||||
self.assertEqual(by_account[32]["status"], "submitted")
|
||||
self.assertEqual(by_account[33]["status"], "submitted")
|
||||
|
||||
async def test_failed_account_can_be_submitted_again(self):
|
||||
attempts = 0
|
||||
|
||||
async def handler(account_id: int) -> dict:
|
||||
nonlocal attempts
|
||||
attempts += 1
|
||||
if attempts == 1:
|
||||
raise RuntimeError("fail once")
|
||||
return {"message": f"started-{account_id}"}
|
||||
|
||||
queue = self._make_queue(handler, concurrency=1)
|
||||
with patch.object(batch_start_module.logger, "exception"):
|
||||
first = await queue.submit([40])
|
||||
first_completed = await self._wait_for_complete(queue, first["batch_id"])
|
||||
|
||||
second = await queue.submit([40])
|
||||
second_completed = await self._wait_for_complete(queue, second["batch_id"])
|
||||
|
||||
self.assertEqual(first_completed["failed_count"], 1)
|
||||
self.assertEqual(second["accepted_count"], 1)
|
||||
self.assertEqual(second["skipped_count"], 0)
|
||||
self.assertEqual(second_completed["submitted_count"], 1)
|
||||
self.assertEqual(attempts, 2)
|
||||
|
||||
async def test_cancel_then_immediate_resubmit_keeps_new_job_deduplicated(self):
|
||||
blocker_started = asyncio.Event()
|
||||
release_blocker = asyncio.Event()
|
||||
replacement_started = asyncio.Event()
|
||||
release_replacement = asyncio.Event()
|
||||
calls: list[int] = []
|
||||
|
||||
async def handler(account_id: int) -> dict:
|
||||
calls.append(account_id)
|
||||
if account_id == 60:
|
||||
blocker_started.set()
|
||||
await release_blocker.wait()
|
||||
elif account_id == 61:
|
||||
replacement_started.set()
|
||||
await release_replacement.wait()
|
||||
return {"message": f"started-{account_id}"}
|
||||
|
||||
queue = self._make_queue(handler, concurrency=1)
|
||||
blocker = await queue.submit([60])
|
||||
await asyncio.wait_for(blocker_started.wait(), timeout=0.2)
|
||||
|
||||
cancelled_batch = await queue.submit([61])
|
||||
self.assertEqual(await queue.cancel_account(61), 1)
|
||||
|
||||
replacement = await queue.submit([61])
|
||||
self.assertEqual(replacement["accepted_count"], 1)
|
||||
|
||||
# Releasing the blocker makes the worker consume the stale cancelled
|
||||
# queue entry before it starts the replacement. Cleanup for that old
|
||||
# token must not release the replacement's pending ownership.
|
||||
release_blocker.set()
|
||||
await asyncio.wait_for(replacement_started.wait(), timeout=0.2)
|
||||
|
||||
duplicate = await queue.submit([61])
|
||||
self.assertEqual(duplicate["accepted_count"], 0)
|
||||
self.assertEqual(duplicate["skipped_count"], 1)
|
||||
|
||||
release_replacement.set()
|
||||
await self._wait_for_complete(queue, blocker["batch_id"])
|
||||
cancelled = await self._wait_for_complete(
|
||||
queue,
|
||||
cancelled_batch["batch_id"],
|
||||
)
|
||||
completed = await self._wait_for_complete(queue, replacement["batch_id"])
|
||||
|
||||
self.assertEqual(cancelled["cancelled_count"], 1)
|
||||
self.assertEqual(completed["submitted_count"], 1)
|
||||
self.assertEqual(calls, [60, 61])
|
||||
|
||||
async def test_stop_cancels_active_and_queued_jobs_and_cleans_workers(self):
|
||||
active = 0
|
||||
two_started = asyncio.Event()
|
||||
started: list[int] = []
|
||||
never_release = asyncio.Event()
|
||||
|
||||
async def handler(account_id: int) -> dict:
|
||||
nonlocal active
|
||||
active += 1
|
||||
started.append(account_id)
|
||||
if len(started) == 2:
|
||||
two_started.set()
|
||||
try:
|
||||
await never_release.wait()
|
||||
return {"message": f"started-{account_id}"}
|
||||
finally:
|
||||
active -= 1
|
||||
|
||||
queue = self._make_queue(handler, concurrency=2)
|
||||
submitted = await queue.submit([51, 52, 53, 54])
|
||||
await asyncio.wait_for(two_started.wait(), timeout=0.2)
|
||||
|
||||
await asyncio.wait_for(queue.stop(), timeout=0.2)
|
||||
completed = await queue.get_batch(submitted["batch_id"])
|
||||
|
||||
self.assertTrue(completed["complete"])
|
||||
self.assertEqual(completed["cancelled_count"], 4)
|
||||
self.assertEqual(active, 0)
|
||||
self.assertEqual(queue._pending_jobs, {})
|
||||
self.assertEqual(queue._workers, [])
|
||||
self.assertTrue(queue._queue.empty())
|
||||
await asyncio.wait_for(queue._queue.join(), timeout=0.1)
|
||||
|
||||
leaked = [
|
||||
task
|
||||
for task in asyncio.all_tasks()
|
||||
if task is not asyncio.current_task()
|
||||
and task.get_name().startswith("account-batch-start-")
|
||||
and not task.done()
|
||||
]
|
||||
self.assertEqual(leaked, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,80 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
|
||||
from rpa_engine.douyin_im.session import DouyinImSession
|
||||
|
||||
|
||||
class ConversationPollBandwidthTests(unittest.IsolatedAsyncioTestCase):
|
||||
def _make_client(self) -> DouyinImHttpClient:
|
||||
return DouyinImHttpClient(
|
||||
DouyinImSession(cookies={"sessionid": "test"}, my_uid=10001),
|
||||
account_id=9,
|
||||
)
|
||||
|
||||
async def test_terminal_token_error_does_not_probe_other_payloads(self):
|
||||
client = self._make_client()
|
||||
client._request = AsyncMock(
|
||||
return_value={
|
||||
"status_code": 500,
|
||||
"error_desc": "empty token",
|
||||
"body": {},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(await client.get_conversations(), [])
|
||||
client._request.assert_awaited_once()
|
||||
self.assertEqual(client._request.await_args.args[0], "POST")
|
||||
|
||||
async def test_successful_empty_response_stops_after_first_payload(self):
|
||||
client = self._make_client()
|
||||
client._request = AsyncMock(
|
||||
return_value={"status_code": 0, "body": {"conversation_list": []}}
|
||||
)
|
||||
|
||||
self.assertEqual(await client.get_conversations(), [])
|
||||
client._request.assert_awaited_once()
|
||||
|
||||
async def test_parameter_error_can_fall_through_to_compatible_payload(self):
|
||||
client = self._make_client()
|
||||
client._request = AsyncMock(
|
||||
side_effect=[
|
||||
{"status_code": 400, "error_desc": "invalid parameter"},
|
||||
{"status_code": 0, "body": {"conversation_list": []}},
|
||||
]
|
||||
)
|
||||
|
||||
self.assertEqual(await client.get_conversations(), [])
|
||||
self.assertEqual(client._request.await_count, 2)
|
||||
self.assertTrue(
|
||||
all(call.args[0] == "POST" for call in client._request.await_args_list)
|
||||
)
|
||||
|
||||
async def test_get_fallback_only_runs_after_transport_failure(self):
|
||||
client = self._make_client()
|
||||
client._request = AsyncMock(
|
||||
side_effect=[None, {"status_code": 0, "body": {}}]
|
||||
)
|
||||
|
||||
self.assertEqual(await client.get_conversations(), [])
|
||||
self.assertEqual(
|
||||
[call.args[0] for call in client._request.await_args_list],
|
||||
["POST", "GET"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,263 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
import main
|
||||
from rpa_engine import account_profile as account_profile_module
|
||||
|
||||
|
||||
class CookieCredentialLockTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_invalid_cookie_does_not_stop_or_lock_hosting(self):
|
||||
db = SimpleNamespace()
|
||||
user = SimpleNamespace(id=9, role="operator")
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
main,
|
||||
"validate_cookie_json",
|
||||
side_effect=ValueError("invalid json"),
|
||||
),
|
||||
patch.object(main, "get_owned_account", new=AsyncMock()) as get_owned,
|
||||
patch.object(
|
||||
main.batch_start_queue,
|
||||
"cancel_account",
|
||||
new=AsyncMock(),
|
||||
) as cancel,
|
||||
patch.object(main.manager, "stop_worker", new=AsyncMock()) as stop,
|
||||
patch.object(main.manager, "preparation_lock") as preparation_lock,
|
||||
):
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await main.update_account_cookie(
|
||||
account_id=81,
|
||||
body=main.AccountCookieUpdate(cookie_data="{bad-json"),
|
||||
db=db,
|
||||
user=user,
|
||||
)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 400)
|
||||
get_owned.assert_not_awaited()
|
||||
cancel.assert_not_awaited()
|
||||
stop.assert_not_awaited()
|
||||
preparation_lock.assert_not_called()
|
||||
|
||||
async def test_cookie_update_stays_locked_through_stop_and_commit(self):
|
||||
account = SimpleNamespace(
|
||||
id=82,
|
||||
cookie_data="old-cookie",
|
||||
cookie_path="old-path",
|
||||
cookie_updated_at=None,
|
||||
updated_at=None,
|
||||
)
|
||||
user = SimpleNamespace(id=9, role="operator")
|
||||
events: list[str] = []
|
||||
lock_active = False
|
||||
|
||||
def require_lock(event: str) -> None:
|
||||
self.assertTrue(lock_active, f"{event} ran outside preparation lock")
|
||||
events.append(event)
|
||||
|
||||
@asynccontextmanager
|
||||
async def preparation_lock(account_id: int):
|
||||
nonlocal lock_active
|
||||
self.assertEqual(account_id, 82)
|
||||
lock_active = True
|
||||
events.append("lock-enter")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.append("lock-exit")
|
||||
lock_active = False
|
||||
|
||||
async def get_owned(db, selected_user, account_id, *, write=False):
|
||||
require_lock("authorize")
|
||||
self.assertIs(db, fake_db)
|
||||
self.assertIs(selected_user, user)
|
||||
self.assertEqual(account_id, 82)
|
||||
self.assertTrue(write)
|
||||
return account
|
||||
|
||||
async def cancel_account(account_id: int):
|
||||
require_lock("cancel")
|
||||
self.assertEqual(account_id, 82)
|
||||
return 0
|
||||
|
||||
async def stop_worker(account_id: int):
|
||||
require_lock("stop")
|
||||
self.assertEqual(account_id, 82)
|
||||
return True
|
||||
|
||||
async def execute(_statement):
|
||||
require_lock("clear-profile")
|
||||
return None
|
||||
|
||||
async def commit():
|
||||
require_lock("commit")
|
||||
|
||||
async def refresh(selected_account):
|
||||
require_lock("refresh")
|
||||
self.assertIs(selected_account, account)
|
||||
|
||||
async def apply_profile(db, selected_account, cookie_data):
|
||||
require_lock("sync-profile")
|
||||
self.assertIs(db, fake_db)
|
||||
self.assertIs(selected_account, account)
|
||||
self.assertIn("sessionid", cookie_data)
|
||||
|
||||
fake_db = SimpleNamespace(
|
||||
execute=AsyncMock(side_effect=execute),
|
||||
commit=AsyncMock(side_effect=commit),
|
||||
refresh=AsyncMock(side_effect=refresh),
|
||||
)
|
||||
|
||||
def write_cookie(account_id: int, cookie_data: str) -> str:
|
||||
require_lock("write-cookie")
|
||||
self.assertEqual(account_id, 82)
|
||||
self.assertIn("sessionid", cookie_data)
|
||||
return "new-cookie-path"
|
||||
|
||||
with (
|
||||
patch.object(main, "validate_cookie_json", return_value={"cookies": [{"name": "sessionid", "value": "new"}]}),
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(side_effect=get_owned)) as get_owned_mock,
|
||||
patch.object(main.manager, "preparation_lock", side_effect=preparation_lock),
|
||||
patch.object(main.batch_start_queue, "cancel_account", new=AsyncMock(side_effect=cancel_account)) as cancel,
|
||||
patch.object(main.manager, "stop_worker", new=AsyncMock(side_effect=stop_worker)) as stop,
|
||||
patch.object(main, "write_cookie_file", side_effect=write_cookie),
|
||||
patch.object(account_profile_module, "apply_douyin_profile", new=AsyncMock(side_effect=apply_profile)),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value={"account_id": 82})) as build_response,
|
||||
):
|
||||
response = await main.update_account_cookie(
|
||||
account_id=82,
|
||||
body=main.AccountCookieUpdate(cookie_data="valid-cookie"),
|
||||
db=fake_db,
|
||||
user=user,
|
||||
)
|
||||
|
||||
self.assertEqual(response, {"account_id": 82})
|
||||
self.assertEqual(
|
||||
events,
|
||||
[
|
||||
"lock-enter",
|
||||
"authorize",
|
||||
"cancel",
|
||||
"stop",
|
||||
"write-cookie",
|
||||
"clear-profile",
|
||||
"sync-profile",
|
||||
"commit",
|
||||
"refresh",
|
||||
"lock-exit",
|
||||
],
|
||||
)
|
||||
get_owned_mock.assert_awaited_once()
|
||||
cancel.assert_awaited_once_with(82)
|
||||
stop.assert_awaited_once_with(82)
|
||||
self.assertEqual(account.cookie_path, "new-cookie-path")
|
||||
build_response.assert_awaited_once()
|
||||
|
||||
async def test_cookie_delete_stays_locked_through_stop_and_commit(self):
|
||||
account = SimpleNamespace(
|
||||
id=83,
|
||||
cookie_data="old-cookie",
|
||||
cookie_path="old-path",
|
||||
cookie_updated_at=object(),
|
||||
im_session_data="old-im-session",
|
||||
updated_at=None,
|
||||
)
|
||||
user = SimpleNamespace(id=9, role="operator")
|
||||
events: list[str] = []
|
||||
lock_active = False
|
||||
|
||||
def require_lock(event: str) -> None:
|
||||
self.assertTrue(lock_active, f"{event} ran outside preparation lock")
|
||||
events.append(event)
|
||||
|
||||
@asynccontextmanager
|
||||
async def preparation_lock(account_id: int):
|
||||
nonlocal lock_active
|
||||
self.assertEqual(account_id, 83)
|
||||
lock_active = True
|
||||
events.append("lock-enter")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.append("lock-exit")
|
||||
lock_active = False
|
||||
|
||||
async def get_owned(_db, _user, account_id, *, write=False):
|
||||
require_lock("authorize")
|
||||
self.assertEqual(account_id, 83)
|
||||
self.assertTrue(write)
|
||||
return account
|
||||
|
||||
async def cancel_account(_account_id: int):
|
||||
require_lock("cancel")
|
||||
return 0
|
||||
|
||||
async def stop_worker(_account_id: int):
|
||||
require_lock("stop")
|
||||
return True
|
||||
|
||||
async def execute(_statement):
|
||||
require_lock("clear-profile")
|
||||
|
||||
async def commit():
|
||||
require_lock("commit")
|
||||
|
||||
fake_db = SimpleNamespace(
|
||||
execute=AsyncMock(side_effect=execute),
|
||||
commit=AsyncMock(side_effect=commit),
|
||||
)
|
||||
|
||||
def clear_cookie(account_id: int) -> None:
|
||||
require_lock("clear-cookie-file")
|
||||
self.assertEqual(account_id, 83)
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(side_effect=get_owned)),
|
||||
patch.object(main.manager, "preparation_lock", side_effect=preparation_lock),
|
||||
patch.object(main.batch_start_queue, "cancel_account", new=AsyncMock(side_effect=cancel_account)),
|
||||
patch.object(main.manager, "stop_worker", new=AsyncMock(side_effect=stop_worker)),
|
||||
patch.object(main, "clear_cookie_file", side_effect=clear_cookie),
|
||||
):
|
||||
response = await main.delete_account_cookie(
|
||||
account_id=83,
|
||||
db=fake_db,
|
||||
user=user,
|
||||
)
|
||||
|
||||
self.assertEqual(response["message"], "Cookie cleared successfully.")
|
||||
self.assertEqual(
|
||||
events,
|
||||
[
|
||||
"lock-enter",
|
||||
"authorize",
|
||||
"cancel",
|
||||
"stop",
|
||||
"clear-cookie-file",
|
||||
"clear-profile",
|
||||
"commit",
|
||||
"lock-exit",
|
||||
],
|
||||
)
|
||||
self.assertIsNone(account.cookie_data)
|
||||
self.assertIsNone(account.cookie_path)
|
||||
self.assertIsNone(account.cookie_updated_at)
|
||||
self.assertIsNone(account.im_session_data)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,202 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
import main
|
||||
|
||||
|
||||
class _ProfileResult:
|
||||
def __init__(self, profile):
|
||||
self.profile = profile
|
||||
|
||||
def scalar_one_or_none(self):
|
||||
return self.profile
|
||||
|
||||
|
||||
class DesktopLoginSecUserIdTests(unittest.IsolatedAsyncioTestCase):
|
||||
@staticmethod
|
||||
def _account(*, cookie_data: str | None = "cookie-json"):
|
||||
return SimpleNamespace(
|
||||
id=501,
|
||||
cookie_data=cookie_data,
|
||||
cookie_path=None,
|
||||
cookie_updated_at=datetime(2026, 7, 22, 8, 0, 0),
|
||||
im_session_data=None,
|
||||
)
|
||||
|
||||
async def test_verified_sec_user_id_returns_credential(self):
|
||||
account = self._account()
|
||||
profile = SimpleNamespace(
|
||||
sec_user_id=" MS4wLjAB-valid ",
|
||||
synced_at=account.cookie_updated_at + timedelta(seconds=1),
|
||||
)
|
||||
db = SimpleNamespace(execute=AsyncMock(return_value=_ProfileResult(profile)))
|
||||
response = main.AccountCookieResponse(account_id=501, cookie_data="cookie-json")
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value=response)) as build,
|
||||
):
|
||||
result = await main.get_desktop_login_credential(
|
||||
account_id=501,
|
||||
db=db,
|
||||
user=SimpleNamespace(id=9, role="operator"),
|
||||
)
|
||||
|
||||
self.assertIs(result, response)
|
||||
db.execute.assert_awaited_once()
|
||||
build.assert_awaited_once_with(account, "cookie-json")
|
||||
|
||||
async def test_missing_or_blank_sec_user_id_is_rejected_without_cookie(self):
|
||||
for missing_value in (None, "", " "):
|
||||
with self.subTest(sec_user_id=missing_value):
|
||||
account = self._account()
|
||||
profile = SimpleNamespace(
|
||||
sec_user_id=missing_value,
|
||||
synced_at=account.cookie_updated_at,
|
||||
)
|
||||
db = SimpleNamespace(
|
||||
execute=AsyncMock(return_value=_ProfileResult(profile))
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
|
||||
):
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await main.get_desktop_login_credential(
|
||||
account_id=501,
|
||||
db=db,
|
||||
user=SimpleNamespace(id=9, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 409)
|
||||
self.assertIn("缺少 sec_user_id", str(caught.exception.detail))
|
||||
build.assert_not_awaited()
|
||||
|
||||
async def test_missing_profile_or_stale_identity_is_rejected(self):
|
||||
account = self._account()
|
||||
stale_profile = SimpleNamespace(
|
||||
sec_user_id="MS4wLjAB-old",
|
||||
synced_at=account.cookie_updated_at - timedelta(seconds=1),
|
||||
)
|
||||
for profile in (None, stale_profile):
|
||||
with self.subTest(profile=profile):
|
||||
db = SimpleNamespace(
|
||||
execute=AsyncMock(return_value=_ProfileResult(profile))
|
||||
)
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
|
||||
):
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await main.get_desktop_login_credential(
|
||||
account_id=501,
|
||||
db=db,
|
||||
user=SimpleNamespace(id=9, role="operator"),
|
||||
)
|
||||
self.assertEqual(caught.exception.status_code, 409)
|
||||
build.assert_not_awaited()
|
||||
|
||||
async def test_profile_check_failure_is_unknown_and_fails_closed(self):
|
||||
account = self._account()
|
||||
db = SimpleNamespace(execute=AsyncMock(side_effect=RuntimeError("db locked")))
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
|
||||
patch.object(main.logger, "exception") as log_exception,
|
||||
):
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await main.get_desktop_login_credential(
|
||||
account_id=501,
|
||||
db=db,
|
||||
user=SimpleNamespace(id=9, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 503)
|
||||
self.assertNotIn("缺少 sec_user_id", str(caught.exception.detail))
|
||||
build.assert_not_awaited()
|
||||
log_exception.assert_called_once()
|
||||
|
||||
async def test_no_cookie_keeps_existing_no_login_state_flow(self):
|
||||
account = self._account(cookie_data=None)
|
||||
db = SimpleNamespace(execute=AsyncMock())
|
||||
response = main.AccountCookieResponse(account_id=501, cookie_data=None)
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
|
||||
patch.object(main, "_get_account_cookie_data", return_value=None),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value=response)) as build,
|
||||
):
|
||||
result = await main.get_desktop_login_credential(
|
||||
account_id=501,
|
||||
db=db,
|
||||
user=SimpleNamespace(id=9, role="operator"),
|
||||
)
|
||||
|
||||
self.assertIs(result, response)
|
||||
db.execute.assert_not_awaited()
|
||||
build.assert_awaited_once_with(account, None)
|
||||
|
||||
async def test_normal_cookie_management_endpoint_is_not_guarded(self):
|
||||
account = self._account()
|
||||
db = SimpleNamespace(execute=AsyncMock())
|
||||
response = main.AccountCookieResponse(account_id=501, cookie_data="cookie-json")
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value=response)) as build,
|
||||
):
|
||||
result = await main.get_account_cookie(
|
||||
account_id=501,
|
||||
purpose="management",
|
||||
db=db,
|
||||
user=SimpleNamespace(id=9, role="operator"),
|
||||
)
|
||||
|
||||
self.assertIs(result, response)
|
||||
db.execute.assert_not_awaited()
|
||||
build.assert_awaited_once_with(account, "cookie-json")
|
||||
|
||||
async def test_legacy_cookie_request_is_guarded_for_existing_desktop_clients(self):
|
||||
account = self._account()
|
||||
profile = SimpleNamespace(
|
||||
sec_user_id=" ",
|
||||
synced_at=account.cookie_updated_at,
|
||||
)
|
||||
db = SimpleNamespace(execute=AsyncMock(return_value=_ProfileResult(profile)))
|
||||
|
||||
with (
|
||||
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
|
||||
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
|
||||
):
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await main.get_account_cookie(
|
||||
account_id=501,
|
||||
purpose=None,
|
||||
db=db,
|
||||
user=SimpleNamespace(id=9, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 409)
|
||||
self.assertIn("缺少 sec_user_id", str(caught.exception.detail))
|
||||
build.assert_not_awaited()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,154 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
import main
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
final_host: str,
|
||||
headers: dict[str, str] | None = None,
|
||||
chunks: tuple[bytes, ...] = (),
|
||||
) -> None:
|
||||
self.url = SimpleNamespace(host=final_host)
|
||||
self.headers = headers or {}
|
||||
self._chunks = chunks
|
||||
self.raise_for_status = MagicMock()
|
||||
self.iteration_started = False
|
||||
|
||||
async def aiter_bytes(self):
|
||||
self.iteration_started = True
|
||||
for chunk in self._chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
class _AsyncContext:
|
||||
def __init__(self, value) -> None:
|
||||
self.value = value
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.value
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback):
|
||||
return False
|
||||
|
||||
|
||||
class _FakeAsyncClient:
|
||||
def __init__(self, response: _FakeResponse) -> None:
|
||||
self.response = response
|
||||
self.stream_calls: list[tuple[tuple, dict]] = []
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback):
|
||||
return False
|
||||
|
||||
def stream(self, *args, **kwargs):
|
||||
self.stream_calls.append((args, kwargs))
|
||||
return _AsyncContext(self.response)
|
||||
|
||||
|
||||
class MediaProxyHostGuardTests(unittest.TestCase):
|
||||
def test_allows_each_root_domain_and_its_subdomains(self):
|
||||
for allowed in main._MEDIA_PROXY_HOSTS:
|
||||
with self.subTest(host=allowed):
|
||||
self.assertTrue(main._is_allowed_media_host(allowed))
|
||||
self.assertTrue(main._is_allowed_media_host(f"cdn.images.{allowed}"))
|
||||
|
||||
self.assertTrue(main._is_allowed_media_host("CDN.DOUYINPIC.COM."))
|
||||
|
||||
def test_rejects_empty_suffix_tricks_and_similar_domains(self):
|
||||
rejected = (
|
||||
"",
|
||||
".",
|
||||
"douyin.com.evil",
|
||||
"evildouyin.com",
|
||||
"byteimg.com.evil.example",
|
||||
"evilbyteimg.com",
|
||||
"ibyteimg.comevil",
|
||||
"douyin.co",
|
||||
"douyinpic.co",
|
||||
"douyin-static.com",
|
||||
"snssdk.example",
|
||||
)
|
||||
|
||||
for hostname in rejected:
|
||||
with self.subTest(host=hostname):
|
||||
self.assertFalse(main._is_allowed_media_host(hostname))
|
||||
|
||||
|
||||
class MediaProxyResponseGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def _call_proxy(self, response: _FakeResponse):
|
||||
client = _FakeAsyncClient(response)
|
||||
constructor = MagicMock(return_value=client)
|
||||
with patch.object(main.httpx, "AsyncClient", constructor):
|
||||
result = await main.proxy_media(
|
||||
url="https://cdn.douyinpic.com/media/test.jpg"
|
||||
)
|
||||
return result, client, constructor
|
||||
|
||||
async def test_rejects_redirect_to_non_allowlisted_final_domain(self):
|
||||
response = _FakeResponse(
|
||||
final_host="attacker.example",
|
||||
headers={"content-type": "image/jpeg"},
|
||||
chunks=(b"not-read",),
|
||||
)
|
||||
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await self._call_proxy(response)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 400)
|
||||
self.assertFalse(response.iteration_started)
|
||||
|
||||
async def test_rejects_oversized_content_length_before_streaming(self):
|
||||
response = _FakeResponse(
|
||||
final_host="cdn.douyinpic.com",
|
||||
headers={"content-length": "9", "content-type": "image/jpeg"},
|
||||
chunks=(b"not-read",),
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(main, "_MEDIA_PROXY_MAX_BYTES", 8),
|
||||
self.assertRaises(main.HTTPException) as caught,
|
||||
):
|
||||
await self._call_proxy(response)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 413)
|
||||
self.assertFalse(response.iteration_started)
|
||||
|
||||
async def test_rejects_stream_when_actual_bytes_exceed_limit(self):
|
||||
response = _FakeResponse(
|
||||
final_host="cdn.douyinpic.com",
|
||||
headers={"content-length": "0", "content-type": "image/jpeg"},
|
||||
chunks=(b"12345", b"6789"),
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(main, "_MEDIA_PROXY_MAX_BYTES", 8),
|
||||
self.assertRaises(main.HTTPException) as caught,
|
||||
):
|
||||
await self._call_proxy(response)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 413)
|
||||
self.assertTrue(response.iteration_started)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,458 @@
|
||||
import asyncio
|
||||
import sys
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from rpa_engine.douyin_im.reply_queue import AccountReplyQueue
|
||||
|
||||
|
||||
class AccountReplyQueueTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def _start_queue(self, account_id: int, **kwargs) -> AccountReplyQueue:
|
||||
queue = AccountReplyQueue(account_id=account_id, **kwargs)
|
||||
await queue.start()
|
||||
self.addAsyncCleanup(queue.stop)
|
||||
return queue
|
||||
|
||||
async def _wait_until_idle(
|
||||
self,
|
||||
queue: AccountReplyQueue,
|
||||
timeout: float = 0.5,
|
||||
) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
while queue.pending_count and loop.time() < deadline:
|
||||
await asyncio.sleep(0.002)
|
||||
self.assertEqual(queue.pending_count, 0)
|
||||
|
||||
async def test_one_account_runs_three_jobs_at_successive_fifo_slots(self):
|
||||
queue = await self._start_queue(account_id=101)
|
||||
interval = 0.05
|
||||
loop = asyncio.get_running_loop()
|
||||
started_at = loop.time()
|
||||
calls: list[tuple[int, float]] = []
|
||||
finished = asyncio.Event()
|
||||
|
||||
def callback_for(index: int):
|
||||
async def callback() -> None:
|
||||
calls.append((index, loop.time() - started_at))
|
||||
if len(calls) == 3:
|
||||
finished.set()
|
||||
|
||||
return callback
|
||||
|
||||
for index in range(3):
|
||||
await queue.enqueue(interval, callback_for(index), description=str(index))
|
||||
|
||||
await asyncio.wait_for(finished.wait(), timeout=0.75)
|
||||
await self._wait_until_idle(queue)
|
||||
|
||||
self.assertEqual([index for index, _ in calls], [0, 1, 2])
|
||||
elapsed = [timestamp for _, timestamp in calls]
|
||||
for timestamp, expected in zip(elapsed, (interval, interval * 2, interval * 3)):
|
||||
self.assertGreaterEqual(timestamp, expected - 0.015)
|
||||
self.assertLess(timestamp, expected + 0.15)
|
||||
self.assertGreaterEqual(elapsed[1] - elapsed[0], interval - 0.02)
|
||||
self.assertGreaterEqual(elapsed[2] - elapsed[1], interval - 0.02)
|
||||
|
||||
async def test_separate_account_queues_reach_first_slot_without_blocking(self):
|
||||
first_queue = await self._start_queue(account_id=201)
|
||||
second_queue = await self._start_queue(account_id=202)
|
||||
interval = 0.04
|
||||
loop = asyncio.get_running_loop()
|
||||
started_at = loop.time()
|
||||
first_started = asyncio.Event()
|
||||
release_first = asyncio.Event()
|
||||
second_finished = asyncio.Event()
|
||||
timestamps: dict[str, float] = {}
|
||||
|
||||
async def first_callback() -> None:
|
||||
timestamps["first"] = loop.time() - started_at
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
|
||||
async def second_callback() -> None:
|
||||
timestamps["second"] = loop.time() - started_at
|
||||
second_finished.set()
|
||||
|
||||
await asyncio.gather(
|
||||
first_queue.enqueue(interval, first_callback, description="first account"),
|
||||
second_queue.enqueue(interval, second_callback, description="second account"),
|
||||
)
|
||||
|
||||
await asyncio.wait_for(first_started.wait(), timeout=0.4)
|
||||
await asyncio.wait_for(second_finished.wait(), timeout=0.4)
|
||||
self.assertFalse(release_first.is_set())
|
||||
self.assertLess(abs(timestamps["first"] - timestamps["second"]), 0.05)
|
||||
|
||||
release_first.set()
|
||||
await self._wait_until_idle(first_queue)
|
||||
await self._wait_until_idle(second_queue)
|
||||
|
||||
async def test_stop_before_due_discards_pending_job(self):
|
||||
queue = await self._start_queue(account_id=301)
|
||||
callback_called = asyncio.Event()
|
||||
|
||||
async def callback() -> None:
|
||||
callback_called.set()
|
||||
|
||||
await queue.enqueue(0.12, callback, description="must be discarded")
|
||||
await asyncio.sleep(0.02)
|
||||
self.assertEqual(queue.pending_count, 1)
|
||||
|
||||
await queue.stop()
|
||||
|
||||
self.assertEqual(queue.pending_count, 0)
|
||||
await asyncio.sleep(0.13)
|
||||
self.assertFalse(callback_called.is_set())
|
||||
|
||||
async def test_callback_failure_does_not_block_following_job(self):
|
||||
errors: list[tuple[str, str]] = []
|
||||
|
||||
def on_error(description: str, exc: BaseException) -> None:
|
||||
errors.append((description, type(exc).__name__))
|
||||
|
||||
queue = await self._start_queue(account_id=401, on_error=on_error)
|
||||
calls: list[str] = []
|
||||
second_finished = asyncio.Event()
|
||||
|
||||
async def failing_callback() -> None:
|
||||
calls.append("first")
|
||||
raise ValueError("expected test failure")
|
||||
|
||||
async def following_callback() -> None:
|
||||
calls.append("second")
|
||||
second_finished.set()
|
||||
|
||||
with self.assertLogs("douyin_im.reply_queue", level="ERROR"):
|
||||
await queue.enqueue(0.03, failing_callback, description="first")
|
||||
await queue.enqueue(0.03, following_callback, description="second")
|
||||
await asyncio.wait_for(second_finished.wait(), timeout=0.5)
|
||||
|
||||
await self._wait_until_idle(queue)
|
||||
self.assertEqual(calls, ["first", "second"])
|
||||
self.assertEqual(errors, [("first", "ValueError")])
|
||||
|
||||
async def test_snapshot_exposes_waiting_job_details(self):
|
||||
queue = await self._start_queue(account_id=501)
|
||||
|
||||
async def callback() -> None:
|
||||
pass
|
||||
|
||||
await queue.enqueue(
|
||||
0.2,
|
||||
callback,
|
||||
description="回复 测试用户",
|
||||
details={
|
||||
"sender_name": "测试用户",
|
||||
"conversation_id": "conv-501",
|
||||
"incoming_content": "你好",
|
||||
"replies": ["您好"],
|
||||
},
|
||||
)
|
||||
items = await queue.snapshot()
|
||||
|
||||
self.assertEqual(len(items), 1)
|
||||
self.assertEqual(items[0]["position"], 1)
|
||||
self.assertEqual(items[0]["status"], "waiting")
|
||||
self.assertEqual(items[0]["sender_name"], "测试用户")
|
||||
self.assertEqual(items[0]["incoming_content"], "你好")
|
||||
self.assertEqual(items[0]["replies"], ["您好"])
|
||||
self.assertNotIn("callback", items[0])
|
||||
|
||||
async def test_send_now_wakes_first_job_and_moves_later_slots_forward(self):
|
||||
queue = await self._start_queue(account_id=502)
|
||||
interval = 0.2
|
||||
sent = asyncio.Event()
|
||||
|
||||
async def first_callback() -> None:
|
||||
sent.set()
|
||||
|
||||
async def noop() -> None:
|
||||
pass
|
||||
|
||||
await queue.enqueue(interval, first_callback, description="first")
|
||||
await queue.enqueue(interval, noop, description="second")
|
||||
await queue.enqueue(interval, noop, description="third")
|
||||
before = await queue.snapshot()
|
||||
|
||||
result = await queue.send_now(before[0]["job_id"])
|
||||
self.assertEqual(result["status"], "accepted")
|
||||
self.assertEqual(result["shifted_count"], 2)
|
||||
await asyncio.wait_for(sent.wait(), timeout=0.1)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
after = await queue.snapshot()
|
||||
self.assertEqual([item["description"] for item in after], ["second", "third"])
|
||||
for old, new in zip(before[1:], after):
|
||||
old_due = datetime.fromisoformat(old["scheduled_at"]).timestamp()
|
||||
new_due = datetime.fromisoformat(new["scheduled_at"]).timestamp()
|
||||
self.assertAlmostEqual(old_due - new_due, interval, delta=0.05)
|
||||
|
||||
async def test_send_now_middle_runs_first_and_only_shifts_jobs_behind_it(self):
|
||||
queue = await self._start_queue(account_id=503)
|
||||
interval = 0.12
|
||||
calls: list[str] = []
|
||||
all_sent = asyncio.Event()
|
||||
|
||||
def callback_for(name: str):
|
||||
async def callback() -> None:
|
||||
calls.append(name)
|
||||
if len(calls) == 3:
|
||||
all_sent.set()
|
||||
|
||||
return callback
|
||||
|
||||
for name in ("first", "middle", "last"):
|
||||
await queue.enqueue(interval, callback_for(name), description=name)
|
||||
before = await queue.snapshot()
|
||||
result = await queue.send_now(before[1]["job_id"])
|
||||
|
||||
self.assertEqual(result["shifted_count"], 1)
|
||||
await asyncio.wait_for(all_sent.wait(), timeout=0.6)
|
||||
self.assertEqual(calls, ["middle", "first", "last"])
|
||||
|
||||
first_before = datetime.fromisoformat(before[0]["scheduled_at"]).timestamp()
|
||||
# The first normal job keeps its original slot; this is also validated by execution order.
|
||||
self.assertGreater(first_before, datetime.now().timestamp() - 1)
|
||||
|
||||
async def test_send_now_does_not_overlap_active_send(self):
|
||||
queue = await self._start_queue(account_id=504)
|
||||
first_started = asyncio.Event()
|
||||
release_first = asyncio.Event()
|
||||
second_started = asyncio.Event()
|
||||
|
||||
async def first_callback() -> None:
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
|
||||
async def second_callback() -> None:
|
||||
second_started.set()
|
||||
|
||||
await queue.enqueue(0.01, first_callback, description="first")
|
||||
await queue.enqueue(0.2, second_callback, description="second")
|
||||
await asyncio.wait_for(first_started.wait(), timeout=0.2)
|
||||
items = await queue.snapshot()
|
||||
second = next(item for item in items if item["description"] == "second")
|
||||
|
||||
result = await queue.send_now(second["job_id"])
|
||||
self.assertEqual(result["status"], "accepted")
|
||||
await asyncio.sleep(0.03)
|
||||
self.assertFalse(second_started.is_set())
|
||||
|
||||
release_first.set()
|
||||
await asyncio.wait_for(second_started.wait(), timeout=0.2)
|
||||
|
||||
async def test_send_now_tail_releases_slot_for_next_enqueue(self):
|
||||
queue = await self._start_queue(account_id=505)
|
||||
interval = 0.2
|
||||
|
||||
async def noop() -> None:
|
||||
pass
|
||||
|
||||
for name in ("first", "second", "tail"):
|
||||
await queue.enqueue(interval, noop, description=name)
|
||||
before = await queue.snapshot()
|
||||
old_tail_due = datetime.fromisoformat(before[2]["scheduled_at"]).timestamp()
|
||||
|
||||
await queue.send_now(before[2]["job_id"])
|
||||
await queue.enqueue(interval, noop, description="new-tail")
|
||||
after = await queue.snapshot()
|
||||
new_tail = next(item for item in after if item["description"] == "new-tail")
|
||||
new_tail_due = datetime.fromisoformat(new_tail["scheduled_at"]).timestamp()
|
||||
|
||||
self.assertAlmostEqual(new_tail_due, old_tail_due, delta=0.05)
|
||||
|
||||
async def test_repeated_send_now_is_idempotent(self):
|
||||
queue = await self._start_queue(account_id=506)
|
||||
calls = 0
|
||||
finished = asyncio.Event()
|
||||
|
||||
async def callback() -> None:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
finished.set()
|
||||
|
||||
await queue.enqueue(0.2, callback, description="once")
|
||||
item = (await queue.snapshot())[0]
|
||||
first = await queue.send_now(item["job_id"])
|
||||
second = await queue.send_now(item["job_id"])
|
||||
|
||||
self.assertEqual(first["status"], "accepted")
|
||||
self.assertIn(second["status"], {"already_requested", "already_sending"})
|
||||
await asyncio.wait_for(finished.wait(), timeout=0.1)
|
||||
await asyncio.sleep(0.02)
|
||||
self.assertEqual(calls, 1)
|
||||
|
||||
async def test_merge_pending_keeps_one_job_slot_and_callback(self):
|
||||
queue = await self._start_queue(account_id=507)
|
||||
calls = 0
|
||||
finished = asyncio.Event()
|
||||
|
||||
async def callback() -> None:
|
||||
nonlocal calls
|
||||
calls += 1
|
||||
finished.set()
|
||||
|
||||
await queue.enqueue(
|
||||
0.3,
|
||||
callback,
|
||||
description="conversation",
|
||||
details={
|
||||
"incoming_content": "first",
|
||||
"incoming_contents": ["first"],
|
||||
"message_count": 1,
|
||||
"replies": ["one reply"],
|
||||
},
|
||||
merge_key="conv:507",
|
||||
)
|
||||
before = (await queue.snapshot())[0]
|
||||
|
||||
def append_second(details):
|
||||
details["incoming_content"] = "second"
|
||||
details["incoming_contents"].append("second")
|
||||
details["message_count"] = 2
|
||||
return details
|
||||
|
||||
result = await queue.merge_pending("conv:507", append_second)
|
||||
after = await queue.snapshot()
|
||||
|
||||
self.assertEqual(result["status"], "merged")
|
||||
self.assertEqual(result["position"], 1)
|
||||
self.assertEqual(len(after), 1)
|
||||
self.assertEqual(after[0]["job_id"], before["job_id"])
|
||||
self.assertEqual(after[0]["incoming_contents"], ["first", "second"])
|
||||
self.assertEqual(after[0]["replies"], ["one reply"])
|
||||
old_due = datetime.fromisoformat(before["scheduled_at"]).timestamp()
|
||||
new_due = datetime.fromisoformat(after[0]["scheduled_at"]).timestamp()
|
||||
self.assertAlmostEqual(old_due, new_due, delta=0.02)
|
||||
|
||||
# Management snapshots must not expose the queue's mutable nested list.
|
||||
after[0]["incoming_contents"].append("external mutation")
|
||||
self.assertEqual(
|
||||
(await queue.snapshot())[0]["incoming_contents"],
|
||||
["first", "second"],
|
||||
)
|
||||
|
||||
await queue.send_now(before["job_id"])
|
||||
await asyncio.wait_for(finished.wait(), timeout=0.2)
|
||||
self.assertEqual(calls, 1)
|
||||
|
||||
async def test_urgent_job_can_still_merge_before_sending(self):
|
||||
queue = await self._start_queue(account_id=508)
|
||||
blocker_started = asyncio.Event()
|
||||
release_blocker = asyncio.Event()
|
||||
merged_sent = asyncio.Event()
|
||||
merged_calls = 0
|
||||
|
||||
async def blocker() -> None:
|
||||
blocker_started.set()
|
||||
await release_blocker.wait()
|
||||
|
||||
async def merged_callback() -> None:
|
||||
nonlocal merged_calls
|
||||
merged_calls += 1
|
||||
merged_sent.set()
|
||||
|
||||
await queue.enqueue(0.01, blocker, description="blocker")
|
||||
await queue.enqueue(
|
||||
0.3,
|
||||
merged_callback,
|
||||
description="mergeable",
|
||||
details={"incoming_content": "first", "incoming_contents": ["first"]},
|
||||
merge_key="conv:508",
|
||||
)
|
||||
await asyncio.wait_for(blocker_started.wait(), timeout=0.2)
|
||||
mergeable = next(
|
||||
item for item in await queue.snapshot() if item["description"] == "mergeable"
|
||||
)
|
||||
await queue.send_now(mergeable["job_id"])
|
||||
|
||||
def append_second(details):
|
||||
details["incoming_contents"].append("second")
|
||||
details["message_count"] = 2
|
||||
return details
|
||||
|
||||
result = await queue.merge_pending("conv:508", append_second)
|
||||
self.assertEqual(result["status"], "merged")
|
||||
self.assertEqual(result["queue_status"], "ready")
|
||||
release_blocker.set()
|
||||
await asyncio.wait_for(merged_sent.wait(), timeout=0.2)
|
||||
self.assertEqual(merged_calls, 1)
|
||||
|
||||
async def test_active_job_is_sealed_against_merge(self):
|
||||
queue = await self._start_queue(account_id=509)
|
||||
started = asyncio.Event()
|
||||
release = asyncio.Event()
|
||||
|
||||
async def callback() -> None:
|
||||
started.set()
|
||||
await release.wait()
|
||||
|
||||
await queue.enqueue(
|
||||
0.01,
|
||||
callback,
|
||||
details={"incoming_content": "first", "incoming_contents": ["first"]},
|
||||
merge_key="conv:509",
|
||||
)
|
||||
await asyncio.wait_for(started.wait(), timeout=0.2)
|
||||
result = await queue.merge_pending("conv:509", lambda details: details)
|
||||
self.assertEqual(result["status"], "not_found")
|
||||
release.set()
|
||||
|
||||
async def test_merge_keys_bridge_partial_peer_and_conversation_ids(self):
|
||||
queue = await self._start_queue(account_id=510)
|
||||
|
||||
async def callback() -> None:
|
||||
pass
|
||||
|
||||
await queue.enqueue(
|
||||
0.3,
|
||||
callback,
|
||||
details={"incoming_contents": ["first"], "message_count": 1},
|
||||
merge_keys=("peer:123",),
|
||||
)
|
||||
|
||||
def append_second(details):
|
||||
details["incoming_contents"].append("second")
|
||||
details["message_count"] = 2
|
||||
return details
|
||||
|
||||
result = await queue.merge_pending(
|
||||
("conv:510", "peer:123"),
|
||||
append_second,
|
||||
)
|
||||
self.assertEqual(result["status"], "merged")
|
||||
self.assertEqual(queue.pending_count, 1)
|
||||
self.assertEqual(
|
||||
(await queue.snapshot())[0]["incoming_contents"],
|
||||
["first", "second"],
|
||||
)
|
||||
|
||||
async def test_different_conversation_ids_do_not_merge_on_shared_peer(self):
|
||||
queue = await self._start_queue(account_id=511)
|
||||
|
||||
async def callback() -> None:
|
||||
pass
|
||||
|
||||
await queue.enqueue(
|
||||
0.3,
|
||||
callback,
|
||||
details={"incoming_contents": ["first"]},
|
||||
merge_keys=("conv:first", "peer:123"),
|
||||
)
|
||||
result = await queue.merge_pending(
|
||||
("conv:second", "peer:123"),
|
||||
lambda details: details,
|
||||
)
|
||||
self.assertEqual(result["status"], "not_found")
|
||||
self.assertEqual(queue.pending_count, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,165 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
import main
|
||||
|
||||
|
||||
def _queue_item(account_id: int, job_id: str = "job-1") -> dict:
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"account_id": account_id,
|
||||
"position": 1,
|
||||
"status": "waiting",
|
||||
"expedited": False,
|
||||
"description": "回复 测试用户",
|
||||
"sender_name": "测试用户",
|
||||
"sender_id": "peer-1",
|
||||
"sender_avatar": None,
|
||||
"conversation_id": "conv-1",
|
||||
"incoming_content": "你好",
|
||||
"incoming_contents": ["你好", "第二条"],
|
||||
"message_count": 2,
|
||||
"replies": ["您好"],
|
||||
"interval_seconds": 60,
|
||||
"enqueued_at": "2026-07-20T10:00:00+00:00",
|
||||
"scheduled_at": "2026-07-20T10:01:00+00:00",
|
||||
"remaining_seconds": 60,
|
||||
}
|
||||
|
||||
|
||||
class _FakeService:
|
||||
def __init__(self, account_id: int, items=None, action=None):
|
||||
self.account_id = account_id
|
||||
self._running = True
|
||||
self.items = list(items or [])
|
||||
self.action = action or {
|
||||
"status": "accepted",
|
||||
"job_id": "job-1",
|
||||
"shifted_count": 2,
|
||||
}
|
||||
|
||||
async def get_reply_queue_snapshot(self):
|
||||
return list(self.items)
|
||||
|
||||
async def send_queued_reply_now(self, job_id: str):
|
||||
return {**self.action, "job_id": job_id}
|
||||
|
||||
|
||||
class ReplyQueueApiTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.original_workers = main.manager.workers
|
||||
main.manager.workers = {}
|
||||
|
||||
async def asyncTearDown(self):
|
||||
main.manager.workers = self.original_workers
|
||||
|
||||
async def test_summary_filters_accounts_by_owner_scope(self):
|
||||
service_one = _FakeService(1, [_queue_item(1)])
|
||||
service_two = _FakeService(2, [_queue_item(2, "job-2")])
|
||||
main.manager.workers = {
|
||||
1: SimpleNamespace(is_running=True, _im_service=service_one),
|
||||
2: SimpleNamespace(is_running=True, _im_service=service_two),
|
||||
}
|
||||
|
||||
with patch.object(main, "owned_account_ids", AsyncMock(return_value={1})):
|
||||
response = await main.get_reply_queue_summaries(
|
||||
db=object(),
|
||||
user=SimpleNamespace(id=10, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(response.total_pending, 1)
|
||||
self.assertEqual([item.account_id for item in response.items], [1])
|
||||
|
||||
async def test_offline_account_detail_returns_empty_snapshot(self):
|
||||
account = SimpleNamespace(id=1, reply_delay_seconds=0)
|
||||
with (
|
||||
patch.object(main, "get_owned_account", AsyncMock(return_value=account)),
|
||||
patch.object(main, "_global_reply_delay_seconds", return_value=0),
|
||||
):
|
||||
response = await main.get_account_reply_queue(
|
||||
account_id=1,
|
||||
db=object(),
|
||||
user=SimpleNamespace(id=10, role="operator"),
|
||||
)
|
||||
|
||||
self.assertFalse(response.running)
|
||||
self.assertEqual(response.pending_count, 0)
|
||||
self.assertEqual(response.items, [])
|
||||
|
||||
async def test_detail_preserves_merged_incoming_messages(self):
|
||||
account = SimpleNamespace(id=1, reply_delay_seconds=60)
|
||||
service = _FakeService(1, [_queue_item(1)])
|
||||
main.manager.workers = {
|
||||
1: SimpleNamespace(is_running=True, _im_service=service),
|
||||
}
|
||||
with (
|
||||
patch.object(main, "get_owned_account", AsyncMock(return_value=account)),
|
||||
patch.object(main, "_resolve_effective_reply_delay", return_value=60),
|
||||
):
|
||||
response = await main.get_account_reply_queue(
|
||||
account_id=1,
|
||||
db=object(),
|
||||
user=SimpleNamespace(id=10, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(response.pending_count, 1)
|
||||
self.assertEqual(response.items[0].incoming_contents, ["你好", "第二条"])
|
||||
self.assertEqual(response.items[0].message_count, 2)
|
||||
self.assertEqual(response.items[0].replies, ["您好"])
|
||||
|
||||
async def test_send_now_returns_shifted_count(self):
|
||||
service = _FakeService(1)
|
||||
main.manager.workers = {
|
||||
1: SimpleNamespace(is_running=True, _im_service=service),
|
||||
}
|
||||
with (
|
||||
patch.object(main, "get_owned_account", AsyncMock(return_value=SimpleNamespace(id=1))),
|
||||
patch.object(main.system_logger, "record"),
|
||||
):
|
||||
response = await main.send_account_queued_reply_now(
|
||||
account_id=1,
|
||||
job_id="job-1",
|
||||
db=object(),
|
||||
user=SimpleNamespace(id=10, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(response.status, "accepted")
|
||||
self.assertEqual(response.shifted_count, 2)
|
||||
|
||||
async def test_cross_account_or_finished_job_returns_not_found(self):
|
||||
service = _FakeService(1, action={"status": "not_found"})
|
||||
main.manager.workers = {
|
||||
1: SimpleNamespace(is_running=True, _im_service=service),
|
||||
}
|
||||
with patch.object(
|
||||
main,
|
||||
"get_owned_account",
|
||||
AsyncMock(return_value=SimpleNamespace(id=1)),
|
||||
):
|
||||
with self.assertRaises(main.HTTPException) as caught:
|
||||
await main.send_account_queued_reply_now(
|
||||
account_id=1,
|
||||
job_id="other-account-job",
|
||||
db=object(),
|
||||
user=SimpleNamespace(id=10, role="operator"),
|
||||
)
|
||||
|
||||
self.assertEqual(caught.exception.status_code, 404)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,421 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from auth.system_settings import SystemSettingsData, set_cached_settings
|
||||
from rpa_engine.douyin_im.service import DouyinImService
|
||||
from rpa_engine.playwright_worker import DouyinWorker
|
||||
|
||||
|
||||
class _RecordingQueue:
|
||||
def __init__(self) -> None:
|
||||
self.jobs: list[tuple[float, object, str, dict, frozenset[str]]] = []
|
||||
|
||||
async def enqueue(
|
||||
self,
|
||||
delay_seconds,
|
||||
callback,
|
||||
description="",
|
||||
details=None,
|
||||
merge_key="",
|
||||
merge_keys=None,
|
||||
) -> int:
|
||||
keys = merge_keys if merge_keys is not None else [merge_key]
|
||||
normalized_keys = frozenset(str(key) for key in keys if str(key or "").strip())
|
||||
self.jobs.append(
|
||||
(delay_seconds, callback, description, dict(details or {}), normalized_keys)
|
||||
)
|
||||
return len(self.jobs)
|
||||
|
||||
async def merge_pending(self, merge_key, details_merger):
|
||||
incoming_keys = (
|
||||
frozenset([merge_key])
|
||||
if isinstance(merge_key, str)
|
||||
else frozenset(merge_key)
|
||||
)
|
||||
for index, job in enumerate(self.jobs):
|
||||
existing_conversations = {
|
||||
key for key in job[4] if key.startswith("conv:")
|
||||
}
|
||||
incoming_conversations = {
|
||||
key for key in incoming_keys if key.startswith("conv:")
|
||||
}
|
||||
same_conversation = bool(
|
||||
existing_conversations & incoming_conversations
|
||||
)
|
||||
different_explicit_conversations = bool(
|
||||
existing_conversations
|
||||
and incoming_conversations
|
||||
and not same_conversation
|
||||
)
|
||||
same_peer = bool(
|
||||
{key for key in job[4] if key.startswith("peer:")}
|
||||
& {key for key in incoming_keys if key.startswith("peer:")}
|
||||
)
|
||||
if not same_conversation and (different_explicit_conversations or not same_peer):
|
||||
continue
|
||||
merged_details = details_merger(dict(job[3]))
|
||||
self.jobs[index] = (
|
||||
job[0],
|
||||
job[1],
|
||||
job[2],
|
||||
merged_details,
|
||||
frozenset(job[4] | incoming_keys),
|
||||
)
|
||||
return {
|
||||
"status": "merged",
|
||||
"job_id": f"job-{index + 1}",
|
||||
"position": index + 1,
|
||||
"message_count": merged_details.get("message_count", 1),
|
||||
}
|
||||
return {"status": "not_found"}
|
||||
|
||||
|
||||
def _build_service(delay_seconds: int = 60):
|
||||
match_reply = AsyncMock(return_value=["自动回复"])
|
||||
log_fn = AsyncMock()
|
||||
service = DouyinImService(
|
||||
session=SimpleNamespace(my_uid="999"),
|
||||
match_reply=match_reply,
|
||||
log_fn=log_fn,
|
||||
account_id=1,
|
||||
)
|
||||
service._running = True
|
||||
service._reply_queue = _RecordingQueue()
|
||||
service._resolve_peer_profile = AsyncMock(return_value=("张三", "", "123"))
|
||||
service._resolve_cooldown_seconds = AsyncMock(return_value=0)
|
||||
service._resolve_reply_delay_seconds = AsyncMock(return_value=delay_seconds)
|
||||
return service, match_reply, log_fn
|
||||
|
||||
|
||||
class ReplyQueueIntegrationTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_same_message_from_ws_and_poll_is_queued_once(self):
|
||||
service, match_reply, _ = _build_service()
|
||||
message = {
|
||||
"conversation_id": "conv-1",
|
||||
"sender_uid": "123",
|
||||
"sender_name": "张三",
|
||||
"content": "你好",
|
||||
"server_message_id": "mid-1",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await asyncio.gather(
|
||||
service._handle_incoming(dict(message)),
|
||||
service._handle_incoming(dict(message)),
|
||||
)
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 1)
|
||||
self.assertEqual(match_reply.await_count, 1)
|
||||
details = service._reply_queue.jobs[0][3]
|
||||
self.assertEqual(details["sender_name"], "张三")
|
||||
self.assertEqual(details["conversation_id"], "conv-1")
|
||||
self.assertEqual(details["incoming_content"], "你好")
|
||||
self.assertEqual(details["replies"], ["自动回复"])
|
||||
|
||||
async def test_same_conversation_with_different_message_ids_merges(self):
|
||||
service, match_reply, _ = _build_service()
|
||||
base = {
|
||||
"conversation_id": "conv-1",
|
||||
"sender_uid": "123",
|
||||
"sender_name": "张三",
|
||||
"content": "你好",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming({**base, "server_message_id": "mid-1"})
|
||||
await service._handle_incoming({**base, "server_message_id": "mid-2"})
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 1)
|
||||
self.assertEqual(match_reply.await_count, 1)
|
||||
details = service._reply_queue.jobs[0][3]
|
||||
self.assertEqual(details["incoming_contents"], ["你好", "你好"])
|
||||
self.assertEqual(details["message_count"], 2)
|
||||
self.assertEqual(details["replies"], ["自动回复"])
|
||||
|
||||
async def test_pending_conversation_merges_before_cooldown_filter(self):
|
||||
service, match_reply, _ = _build_service()
|
||||
service._resolve_cooldown_seconds = AsyncMock(return_value=120)
|
||||
base = {
|
||||
"conversation_id": "conv-1",
|
||||
"sender_uid": "123",
|
||||
"sender_name": "张三",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming(
|
||||
{**base, "content": "first", "server_message_id": "mid-1"}
|
||||
)
|
||||
await service._handle_incoming(
|
||||
{
|
||||
**base,
|
||||
"conversation_id": "conv-1",
|
||||
"content": "second",
|
||||
"server_message_id": "mid-2",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 1)
|
||||
self.assertEqual(match_reply.await_count, 1)
|
||||
details = service._reply_queue.jobs[0][3]
|
||||
self.assertEqual(details["incoming_contents"], ["first", "second"])
|
||||
self.assertEqual(details["message_count"], 2)
|
||||
|
||||
async def test_http_previews_from_different_conversations_do_not_collide(self):
|
||||
service, _, _ = _build_service()
|
||||
base = {
|
||||
"sender_uid": "",
|
||||
"sender_name": "同名用户",
|
||||
"content": "相同内容",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming({**base, "conversation_id": "conv-a"})
|
||||
await service._handle_incoming({**base, "conversation_id": "conv-b"})
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 2)
|
||||
|
||||
async def test_missing_conversation_id_uses_peer_uid_for_merge(self):
|
||||
service, match_reply, _ = _build_service()
|
||||
base = {
|
||||
"conversation_id": "",
|
||||
"sender_uid": "123",
|
||||
"sender_name": "张三",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming(
|
||||
{**base, "content": "first", "server_message_id": "mid-1"}
|
||||
)
|
||||
await service._handle_incoming(
|
||||
{
|
||||
**base,
|
||||
"conversation_id": "conv-1",
|
||||
"content": "second",
|
||||
"server_message_id": "mid-2",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 1)
|
||||
self.assertEqual(match_reply.await_count, 1)
|
||||
self.assertEqual(
|
||||
service._reply_queue.jobs[0][3]["incoming_contents"],
|
||||
["first", "second"],
|
||||
)
|
||||
|
||||
async def test_peer_alias_added_later_merges_with_conversation_only_job(self):
|
||||
service, match_reply, _ = _build_service()
|
||||
|
||||
async def resolve_profile(conv_id, sender_uid, sender, sender_avatar):
|
||||
return "张三", "", str(sender_uid or "")
|
||||
|
||||
service._resolve_peer_profile = AsyncMock(side_effect=resolve_profile)
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming(
|
||||
{
|
||||
"conversation_id": "conv-1",
|
||||
"sender_uid": "",
|
||||
"sender_name": "张三",
|
||||
"content": "first",
|
||||
"server_message_id": "mid-1",
|
||||
}
|
||||
)
|
||||
await service._handle_incoming(
|
||||
{
|
||||
"conversation_id": "conv-1",
|
||||
"sender_uid": "123",
|
||||
"sender_name": "张三",
|
||||
"content": "second",
|
||||
"server_message_id": "mid-2",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 1)
|
||||
self.assertEqual(match_reply.await_count, 1)
|
||||
self.assertEqual(
|
||||
service._reply_queue.jobs[0][3]["incoming_contents"],
|
||||
["first", "second"],
|
||||
)
|
||||
|
||||
async def test_missing_conversation_and_peer_ids_do_not_merge_by_name(self):
|
||||
service, match_reply, _ = _build_service()
|
||||
service._resolve_peer_profile = AsyncMock(return_value=("同名用户", "", ""))
|
||||
base = {
|
||||
"conversation_id": "",
|
||||
"sender_uid": "",
|
||||
"sender_name": "同名用户",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming(
|
||||
{**base, "content": "first", "server_message_id": "mid-1"}
|
||||
)
|
||||
await service._handle_incoming(
|
||||
{**base, "content": "second", "server_message_id": "mid-2"}
|
||||
)
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 2)
|
||||
self.assertEqual(match_reply.await_count, 2)
|
||||
|
||||
async def test_slow_first_message_keeps_arrival_order(self):
|
||||
service, _, _ = _build_service()
|
||||
|
||||
async def resolve_profile(conv_id, sender_uid, sender, sender_avatar):
|
||||
if conv_id == "conv-first":
|
||||
await asyncio.sleep(0.03)
|
||||
return "先到", "", "101"
|
||||
return "后到", "", "102"
|
||||
|
||||
service._resolve_peer_profile = AsyncMock(side_effect=resolve_profile)
|
||||
first = {
|
||||
"conversation_id": "conv-first",
|
||||
"sender_uid": "101",
|
||||
"sender_name": "先到",
|
||||
"content": "第一条",
|
||||
"server_message_id": "mid-first",
|
||||
}
|
||||
second = {
|
||||
"conversation_id": "conv-second",
|
||||
"sender_uid": "102",
|
||||
"sender_name": "后到",
|
||||
"content": "第二条",
|
||||
"server_message_id": "mid-second",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
first_task = asyncio.create_task(service._handle_incoming(first))
|
||||
await asyncio.sleep(0)
|
||||
second_task = asyncio.create_task(service._handle_incoming(second))
|
||||
await asyncio.gather(first_task, second_task)
|
||||
|
||||
descriptions = [job[2] for job in service._reply_queue.jobs]
|
||||
self.assertEqual(descriptions, ["回复 先到", "回复 后到"])
|
||||
|
||||
async def test_zero_effective_delay_skips_queue_and_sends_immediately(self):
|
||||
service, _, _ = _build_service(delay_seconds=0)
|
||||
service._send_auto_reply = AsyncMock()
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming(
|
||||
{
|
||||
"conversation_id": "conv-1",
|
||||
"sender_uid": "123",
|
||||
"sender_name": "张三",
|
||||
"content": "你好",
|
||||
"server_message_id": "mid-1",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 0)
|
||||
service._send_auto_reply.assert_awaited_once()
|
||||
|
||||
async def test_zero_delay_keeps_separate_immediate_replies(self):
|
||||
service, _, _ = _build_service(delay_seconds=0)
|
||||
service._send_auto_reply = AsyncMock()
|
||||
base = {
|
||||
"conversation_id": "conv-1",
|
||||
"sender_uid": "123",
|
||||
"sender_name": "张三",
|
||||
}
|
||||
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._handle_incoming(
|
||||
{**base, "content": "first", "server_message_id": "mid-1"}
|
||||
)
|
||||
await service._handle_incoming(
|
||||
{**base, "content": "second", "server_message_id": "mid-2"}
|
||||
)
|
||||
|
||||
self.assertEqual(len(service._reply_queue.jobs), 0)
|
||||
self.assertEqual(service._send_auto_reply.await_count, 2)
|
||||
|
||||
async def test_session_stop_during_first_payload_cancels_remaining_payloads(self):
|
||||
service, _, log_fn = _build_service()
|
||||
|
||||
async def fail_and_stop(*args, **kwargs):
|
||||
service.last_error = "INVALID_REQUEST"
|
||||
service._running = False
|
||||
return False, None
|
||||
|
||||
service._send_text = AsyncMock(side_effect=fail_and_stop)
|
||||
with unittest.mock.patch(
|
||||
"rpa_engine.douyin_im.service.system_logger.record", Mock()
|
||||
):
|
||||
await service._send_auto_reply(
|
||||
sender="张三",
|
||||
content="你好",
|
||||
conv_id="conv-1",
|
||||
replies=["第一段", "第二段"],
|
||||
peer_key="123",
|
||||
cooldown=0,
|
||||
log_kwargs={
|
||||
"sender_name": "张三",
|
||||
"sender_id": "123",
|
||||
"sender_avatar": None,
|
||||
"message": "你好",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(service._send_text.await_count, 1)
|
||||
log_fn.assert_awaited_once()
|
||||
|
||||
|
||||
class ReplyDelayResolutionTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncSetUp(self):
|
||||
self.worker = DouyinWorker(account_id=1)
|
||||
|
||||
async def test_account_value_overrides_system_default(self):
|
||||
set_cached_settings(SystemSettingsData(auto_reply_delay_seconds=60))
|
||||
self.worker.get_reply_delay = AsyncMock(return_value=15)
|
||||
|
||||
self.assertEqual(await self.worker.resolve_reply_delay_seconds(), 15)
|
||||
|
||||
async def test_unset_account_uses_system_default(self):
|
||||
set_cached_settings(SystemSettingsData(auto_reply_delay_seconds=60))
|
||||
self.worker.get_reply_delay = AsyncMock(return_value=None)
|
||||
|
||||
self.assertEqual(await self.worker.resolve_reply_delay_seconds(), 60)
|
||||
|
||||
async def test_both_unset_skip_queue_rule(self):
|
||||
set_cached_settings(SystemSettingsData(auto_reply_delay_seconds=0))
|
||||
self.worker.get_reply_delay = AsyncMock(return_value=None)
|
||||
|
||||
self.assertEqual(await self.worker.resolve_reply_delay_seconds(), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,676 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from rpa_engine.douyin_im.session import DouyinImSession
|
||||
from rpa_engine import account_profile as account_profile_module
|
||||
from rpa_engine.playwright_worker import DouyinWorker
|
||||
|
||||
|
||||
class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_im_direct_missing_sec_user_id_exits_before_online_state(self):
|
||||
worker = DouyinWorker(account_id=301, login_mode="im_direct")
|
||||
worker._require_sec_user_id = AsyncMock(return_value=False)
|
||||
worker._load_user_agent = AsyncMock(return_value="test-agent")
|
||||
im_session = DouyinImSession(
|
||||
cookies={"sessionid": "test-session"},
|
||||
my_uid=30101,
|
||||
)
|
||||
worker._build_im_session_from_storage = AsyncMock(return_value=im_session)
|
||||
worker._persist_im_session = AsyncMock()
|
||||
worker._run_im_direct_service = AsyncMock()
|
||||
|
||||
with patch(
|
||||
"rpa_engine.playwright_worker.validate_im_session",
|
||||
new=AsyncMock(return_value=(True, "ready")),
|
||||
) as validate:
|
||||
started, reason = await worker._try_cookie_only_im_start(
|
||||
{"cookies": [{"name": "sessionid", "value": "test-session"}]}
|
||||
)
|
||||
|
||||
self.assertFalse(started)
|
||||
self.assertIn("sec_user_id", reason)
|
||||
worker._require_sec_user_id.assert_awaited_once()
|
||||
require_call = worker._require_sec_user_id.await_args
|
||||
self.assertTrue(require_call.kwargs["refresh_if_missing"])
|
||||
self.assertTrue(require_call.kwargs["refresh_if_stale"])
|
||||
worker._build_im_session_from_storage.assert_awaited_once()
|
||||
validate.assert_awaited_once_with(im_session)
|
||||
worker._persist_im_session.assert_not_awaited()
|
||||
worker._run_im_direct_service.assert_not_awaited()
|
||||
|
||||
async def test_running_guard_precedes_disabled_follow_welcome_setting(self):
|
||||
worker = DouyinWorker(account_id=302, login_mode="im_direct")
|
||||
worker.is_running = True
|
||||
worker._im_service = SimpleNamespace(_running=True)
|
||||
worker._require_sec_user_id = AsyncMock(return_value=False)
|
||||
worker.get_db = AsyncMock()
|
||||
|
||||
await worker.follow_welcome_tick()
|
||||
|
||||
worker._require_sec_user_id.assert_awaited_once()
|
||||
require_call = worker._require_sec_user_id.await_args
|
||||
self.assertFalse(require_call.kwargs.get("refresh_if_missing", False))
|
||||
# The identity guard must run before the account's follow-welcome flag
|
||||
# is queried. Otherwise accounts with that feature disabled could stay
|
||||
# hosted indefinitely without a sec_user_id.
|
||||
worker.get_db.assert_not_awaited()
|
||||
|
||||
async def test_blank_sec_user_id_is_missing_and_stops_hosting(self):
|
||||
worker = DouyinWorker(account_id=303, login_mode="im_direct")
|
||||
worker._load_sec_user_id = AsyncMock(return_value=" ")
|
||||
worker._refresh_sec_user_id = AsyncMock(return_value="\t")
|
||||
worker._stop_for_missing_sec_user_id = AsyncMock()
|
||||
|
||||
accepted = await worker._require_sec_user_id(
|
||||
"直连启动前",
|
||||
refresh_if_missing=True,
|
||||
)
|
||||
|
||||
self.assertFalse(accepted)
|
||||
worker._load_sec_user_id.assert_awaited()
|
||||
worker._refresh_sec_user_id.assert_awaited_once_with()
|
||||
worker._stop_for_missing_sec_user_id.assert_awaited_once_with("直连启动前")
|
||||
|
||||
async def test_existing_sec_user_id_does_not_refresh_or_stop(self):
|
||||
worker = DouyinWorker(account_id=304, login_mode="im_direct")
|
||||
worker._load_sec_user_id = AsyncMock(return_value="MS4wLjABAAAA-valid")
|
||||
worker._refresh_sec_user_id = AsyncMock()
|
||||
worker._stop_for_missing_sec_user_id = AsyncMock()
|
||||
|
||||
accepted = await worker._require_sec_user_id(
|
||||
"运行中",
|
||||
refresh_if_missing=True,
|
||||
)
|
||||
|
||||
self.assertTrue(accepted)
|
||||
worker._load_sec_user_id.assert_awaited_once_with()
|
||||
worker._refresh_sec_user_id.assert_not_awaited()
|
||||
worker._stop_for_missing_sec_user_id.assert_not_awaited()
|
||||
|
||||
async def test_identity_read_error_does_not_claim_field_is_missing(self):
|
||||
worker = DouyinWorker(account_id=307, login_mode="im_direct")
|
||||
worker._load_sec_user_id = AsyncMock(
|
||||
side_effect=RuntimeError("database temporarily unavailable")
|
||||
)
|
||||
worker._refresh_sec_user_id = AsyncMock()
|
||||
worker._stop_for_missing_sec_user_id = AsyncMock()
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "database temporarily unavailable"):
|
||||
await worker._require_sec_user_id(
|
||||
"托管运行中",
|
||||
refresh_if_missing=True,
|
||||
)
|
||||
|
||||
worker._refresh_sec_user_id.assert_not_awaited()
|
||||
worker._stop_for_missing_sec_user_id.assert_not_awaited()
|
||||
|
||||
async def test_unknown_profile_refresh_error_does_not_stop_as_missing(self):
|
||||
worker = DouyinWorker(account_id=308, login_mode="im_direct")
|
||||
worker._load_sec_user_id = AsyncMock(return_value="")
|
||||
worker._refresh_sec_user_id = AsyncMock(
|
||||
side_effect=RuntimeError("暂时无法核验 sec_user_id,请稍后重试")
|
||||
)
|
||||
worker._stop_for_missing_sec_user_id = AsyncMock()
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "暂时无法核验"):
|
||||
await worker._require_sec_user_id(
|
||||
"启动托管时",
|
||||
refresh_if_missing=True,
|
||||
)
|
||||
|
||||
worker._refresh_sec_user_id.assert_awaited_once_with()
|
||||
worker._stop_for_missing_sec_user_id.assert_not_awaited()
|
||||
|
||||
async def test_stale_cookie_forces_refresh_instead_of_using_cached_identity(self):
|
||||
worker = DouyinWorker(account_id=309, login_mode="im_direct")
|
||||
worker._load_sec_user_id = AsyncMock(return_value="cached-sec-user-id")
|
||||
worker._sec_user_id_is_stale = AsyncMock(return_value=True)
|
||||
worker._refresh_sec_user_id = AsyncMock(return_value="fresh-sec-user-id")
|
||||
worker._stop_for_missing_sec_user_id = AsyncMock()
|
||||
|
||||
resolved = await worker._require_sec_user_id(
|
||||
"启动托管时",
|
||||
refresh_if_missing=True,
|
||||
refresh_if_stale=True,
|
||||
)
|
||||
|
||||
self.assertEqual(resolved, "fresh-sec-user-id")
|
||||
worker._sec_user_id_is_stale.assert_awaited_once_with()
|
||||
worker._refresh_sec_user_id.assert_awaited_once_with()
|
||||
worker._stop_for_missing_sec_user_id.assert_not_awaited()
|
||||
|
||||
async def test_profile_helper_uses_uid_fallback_for_sec_user_id(self):
|
||||
fetch_detail = AsyncMock(
|
||||
return_value={
|
||||
"uid": "123456789",
|
||||
"nickname": "fallback-user",
|
||||
"sec_user_id": "",
|
||||
"fetched": True,
|
||||
"message": "",
|
||||
}
|
||||
)
|
||||
run_in_thread = AsyncMock(return_value=("MS4wLjABAAAA-fallback", True))
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"fetch_douyin_profile_detail",
|
||||
fetch_detail,
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module.asyncio,
|
||||
"to_thread",
|
||||
run_in_thread,
|
||||
),
|
||||
):
|
||||
detail = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
)
|
||||
|
||||
self.assertEqual(detail["sec_user_id"], "MS4wLjABAAAA-fallback")
|
||||
self.assertEqual(detail["sec_user_id_status"], "found")
|
||||
fetch_detail.assert_awaited_once_with("cookie-json", "test-agent")
|
||||
run_in_thread.assert_awaited_once_with(
|
||||
account_profile_module._resolve_sec_user_id_by_uid_sync,
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
"123456789",
|
||||
)
|
||||
|
||||
async def test_profile_transport_failure_remains_unknown_not_missing(self):
|
||||
fetch_detail = AsyncMock(
|
||||
return_value={
|
||||
"uid": "",
|
||||
"sec_user_id": "",
|
||||
"fetched": False,
|
||||
"message": "profile request timed out",
|
||||
}
|
||||
)
|
||||
run_in_thread = AsyncMock()
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"fetch_douyin_profile_detail",
|
||||
fetch_detail,
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module.asyncio,
|
||||
"to_thread",
|
||||
run_in_thread,
|
||||
),
|
||||
):
|
||||
detail = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
)
|
||||
|
||||
self.assertEqual(detail["sec_user_id_status"], "unknown")
|
||||
run_in_thread.assert_not_awaited()
|
||||
|
||||
async def test_confirmed_empty_uid_fallback_is_missing(self):
|
||||
fetch_detail = AsyncMock(
|
||||
return_value={
|
||||
"uid": "987654321",
|
||||
"sec_user_id": "",
|
||||
"fetched": True,
|
||||
"message": "",
|
||||
}
|
||||
)
|
||||
run_in_thread = AsyncMock(return_value=("", True))
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"fetch_douyin_profile_detail",
|
||||
fetch_detail,
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module.asyncio,
|
||||
"to_thread",
|
||||
run_in_thread,
|
||||
),
|
||||
):
|
||||
detail = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
)
|
||||
|
||||
self.assertEqual(detail["sec_user_id"], "")
|
||||
self.assertEqual(detail["sec_user_id_status"], "missing")
|
||||
run_in_thread.assert_awaited_once()
|
||||
|
||||
async def test_stop_for_missing_sec_user_id_is_offline_and_idempotent(self):
|
||||
worker = DouyinWorker(account_id=310, login_mode="im_direct")
|
||||
worker.is_running = True
|
||||
worker.stopping = False
|
||||
service = SimpleNamespace(_running=True)
|
||||
worker._im_service = service
|
||||
worker.update_account_status = AsyncMock()
|
||||
|
||||
with patch(
|
||||
"rpa_engine.playwright_worker.system_logger.record"
|
||||
) as record_system_log:
|
||||
await worker._stop_for_missing_sec_user_id("托管运行中")
|
||||
await worker._stop_for_missing_sec_user_id("重复检查")
|
||||
|
||||
self.assertTrue(worker.stopping)
|
||||
self.assertFalse(worker.is_running)
|
||||
self.assertFalse(service._running)
|
||||
worker.update_account_status.assert_awaited_once()
|
||||
status_call = worker.update_account_status.await_args
|
||||
self.assertEqual(status_call.args[0], "offline")
|
||||
self.assertIn("sec_user_id", status_call.kwargs["error_msg"])
|
||||
self.assertIn("托管已自动退出", status_call.kwargs["error_msg"])
|
||||
record_system_log.assert_called_once()
|
||||
|
||||
async def test_cookie_uid_without_valid_profile_payload_stays_unknown(self):
|
||||
auth = SimpleNamespace(
|
||||
cookie={},
|
||||
msToken="test-ms-token",
|
||||
get_uid=lambda: "cookie-only-uid",
|
||||
)
|
||||
payloads = [
|
||||
{"status_code": 0, "data": {"status": "ok"}},
|
||||
{"status_code": 0, "message": "success"},
|
||||
{"status_code": 0, "data": {}},
|
||||
]
|
||||
responses = [SimpleNamespace(json=lambda value=value: value) for value in payloads]
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"_build_auth",
|
||||
return_value=(auth, "test-agent"),
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module.requests,
|
||||
"get",
|
||||
side_effect=responses,
|
||||
) as request_get,
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"generate_a_bogus",
|
||||
return_value="a-bogus",
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"generate_webid",
|
||||
return_value="web-id",
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"_requests_proxies",
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
raw_detail = account_profile_module.fetch_douyin_profile_detail_sync(
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
)
|
||||
|
||||
self.assertEqual(request_get.call_count, 3)
|
||||
self.assertEqual(raw_detail["uid"], "cookie-only-uid")
|
||||
self.assertFalse(raw_detail["fetched"])
|
||||
self.assertFalse(raw_detail["profile_response_valid"])
|
||||
|
||||
with patch.object(
|
||||
account_profile_module,
|
||||
"fetch_douyin_profile_detail",
|
||||
new=AsyncMock(return_value=raw_detail),
|
||||
):
|
||||
checked = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
)
|
||||
|
||||
self.assertEqual(checked["sec_user_id_status"], "unknown")
|
||||
|
||||
async def test_invalid_uid_fallback_payloads_stay_unknown(self):
|
||||
auth = SimpleNamespace(cookie={}, msToken="test-ms-token")
|
||||
cases = (
|
||||
(
|
||||
"nonzero-status",
|
||||
{
|
||||
"status_code": 10007,
|
||||
"data": {
|
||||
"user": {
|
||||
"uid": "fallback-uid",
|
||||
"sec_uid": "must-not-be-used",
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
"no-user-node",
|
||||
{"status_code": 0, "data": {"status": "ok"}},
|
||||
),
|
||||
)
|
||||
|
||||
for label, payload in cases:
|
||||
with self.subTest(label=label):
|
||||
with (
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"_build_auth",
|
||||
return_value=(auth, "test-agent"),
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"_douyin_get_json",
|
||||
return_value=payload,
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"generate_webid",
|
||||
return_value="web-id",
|
||||
),
|
||||
):
|
||||
resolved, request_completed = (
|
||||
account_profile_module._resolve_sec_user_id_by_uid_sync(
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
"fallback-uid",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(resolved, "")
|
||||
self.assertFalse(request_completed)
|
||||
|
||||
fetch_detail = AsyncMock(
|
||||
return_value={
|
||||
"uid": "fallback-uid",
|
||||
"sec_user_id": "",
|
||||
"fetched": True,
|
||||
"message": "",
|
||||
}
|
||||
)
|
||||
run_in_thread = AsyncMock(
|
||||
return_value=(resolved, request_completed)
|
||||
)
|
||||
with (
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"fetch_douyin_profile_detail",
|
||||
fetch_detail,
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module.asyncio,
|
||||
"to_thread",
|
||||
run_in_thread,
|
||||
),
|
||||
):
|
||||
checked = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
|
||||
"cookie-json",
|
||||
"test-agent",
|
||||
)
|
||||
|
||||
self.assertEqual(checked["sec_user_id_status"], "unknown")
|
||||
|
||||
async def test_unknown_profile_sync_preserves_cached_identity_and_videos(self):
|
||||
old_synced_at = object()
|
||||
profile = SimpleNamespace(
|
||||
sec_user_id="cached-sec-user-id",
|
||||
synced_at=old_synced_at,
|
||||
sync_message="previous message",
|
||||
)
|
||||
account = SimpleNamespace(
|
||||
id=411,
|
||||
user_agent="test-agent",
|
||||
douyin_uid="cached-uid",
|
||||
username="cached-user",
|
||||
avatar_url="cached-avatar",
|
||||
)
|
||||
scalar_result = SimpleNamespace(scalar_one_or_none=lambda: profile)
|
||||
db = SimpleNamespace(
|
||||
execute=AsyncMock(return_value=scalar_result),
|
||||
commit=AsyncMock(),
|
||||
add=MagicMock(),
|
||||
)
|
||||
unknown_detail = {
|
||||
"uid": "cookie-uid",
|
||||
"sec_user_id": "",
|
||||
"sec_user_id_status": "unknown",
|
||||
"fetched": False,
|
||||
"message": "profile endpoint temporarily unavailable",
|
||||
}
|
||||
cached_response = {
|
||||
"account_id": 411,
|
||||
"sec_user_id": "cached-sec-user-id",
|
||||
"synced_at": "cached-time",
|
||||
"videos": [{"aweme_id": "cached-video"}],
|
||||
}
|
||||
fetch_videos = AsyncMock()
|
||||
apply_profile = AsyncMock()
|
||||
load_cached = AsyncMock(return_value=dict(cached_response))
|
||||
|
||||
with (
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"fetch_douyin_profile_detail_with_sec_user_id",
|
||||
new=AsyncMock(return_value=unknown_detail),
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"fetch_douyin_user_videos",
|
||||
fetch_videos,
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"apply_douyin_profile",
|
||||
apply_profile,
|
||||
),
|
||||
patch.object(
|
||||
account_profile_module,
|
||||
"load_account_profile_from_db",
|
||||
load_cached,
|
||||
),
|
||||
):
|
||||
result = await account_profile_module.sync_account_profile_to_db(
|
||||
db,
|
||||
account,
|
||||
"cookie-json",
|
||||
)
|
||||
|
||||
self.assertEqual(profile.sec_user_id, "cached-sec-user-id")
|
||||
self.assertIs(profile.synced_at, old_synced_at)
|
||||
fetch_videos.assert_not_awaited()
|
||||
apply_profile.assert_not_awaited()
|
||||
load_cached.assert_awaited_once_with(db, account)
|
||||
self.assertEqual(db.execute.await_count, 1)
|
||||
self.assertNotIn("delete", str(db.execute.await_args.args[0]).lower())
|
||||
self.assertEqual(result["videos"], cached_response["videos"])
|
||||
self.assertEqual(
|
||||
result["message"],
|
||||
"profile endpoint temporarily unavailable",
|
||||
)
|
||||
|
||||
async def test_browser_login_missing_sec_user_id_closes_browser_and_skips_im(self):
|
||||
worker = DouyinWorker(account_id=305, login_mode="browser")
|
||||
worker.is_running = True
|
||||
events: list[str] = []
|
||||
|
||||
async def reject_identity(*_args, **_kwargs):
|
||||
events.append("require-sec-user-id")
|
||||
return False
|
||||
|
||||
worker._load_user_agent = AsyncMock(return_value="test-agent")
|
||||
worker._probe_existing_login = AsyncMock(return_value=True)
|
||||
worker._finalize_login_session = AsyncMock()
|
||||
worker._require_sec_user_id = AsyncMock(side_effect=reject_identity)
|
||||
worker._setup_im_network_listener = AsyncMock()
|
||||
worker._navigate_to_message_center = AsyncMock()
|
||||
worker._harvest_im_credentials = AsyncMock()
|
||||
worker._persist_cookies = AsyncMock()
|
||||
im_session = DouyinImSession(
|
||||
cookies={"sessionid": "test-session"},
|
||||
my_uid=30501,
|
||||
)
|
||||
worker._build_im_session = AsyncMock(return_value=im_session)
|
||||
worker._persist_im_session = AsyncMock()
|
||||
worker._run_im_direct_service = AsyncMock()
|
||||
worker._close_browser_only = AsyncMock()
|
||||
|
||||
page = SimpleNamespace()
|
||||
context = SimpleNamespace(
|
||||
add_init_script=AsyncMock(),
|
||||
new_page=AsyncMock(return_value=page),
|
||||
)
|
||||
browser = SimpleNamespace(new_context=AsyncMock(return_value=context))
|
||||
playwright = SimpleNamespace()
|
||||
playwright_starter = SimpleNamespace(
|
||||
start=AsyncMock(return_value=playwright),
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def browser_slot(account_id: int, description: str):
|
||||
self.assertEqual(account_id, 305)
|
||||
self.assertEqual(description, "browser credential harvest")
|
||||
events.append("browser-acquired")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.append("browser-released")
|
||||
|
||||
controller = SimpleNamespace(browser_slot=browser_slot)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"rpa_engine.playwright_worker.async_playwright",
|
||||
return_value=playwright_starter,
|
||||
),
|
||||
patch(
|
||||
"rpa_engine.playwright_worker._launch_chromium",
|
||||
new=AsyncMock(return_value=browser),
|
||||
),
|
||||
patch(
|
||||
"rpa_engine.playwright_worker.get_traffic_controller",
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"rpa_engine.playwright_worker.validate_im_session",
|
||||
new=AsyncMock(return_value=(True, "ready")),
|
||||
),
|
||||
):
|
||||
await worker._run_browser_im_flow(
|
||||
storage_state=None,
|
||||
cookie_info={
|
||||
"cookie_valid": False,
|
||||
"has_sessionid": False,
|
||||
"reason": "no saved login",
|
||||
},
|
||||
)
|
||||
|
||||
worker._finalize_login_session.assert_awaited_once_with()
|
||||
worker._require_sec_user_id.assert_awaited_once()
|
||||
require_call = worker._require_sec_user_id.await_args
|
||||
self.assertTrue(require_call.kwargs["force_refresh"])
|
||||
self.assertEqual(
|
||||
events,
|
||||
["browser-acquired", "browser-released", "require-sec-user-id"],
|
||||
)
|
||||
worker._setup_im_network_listener.assert_awaited_once_with()
|
||||
worker._navigate_to_message_center.assert_awaited_once_with()
|
||||
worker._harvest_im_credentials.assert_awaited_once_with(timeout=25)
|
||||
worker._persist_cookies.assert_awaited_once_with()
|
||||
worker._build_im_session.assert_awaited_once_with()
|
||||
worker._persist_im_session.assert_not_awaited()
|
||||
worker._run_im_direct_service.assert_not_awaited()
|
||||
worker._close_browser_only.assert_awaited_once_with()
|
||||
|
||||
async def test_browser_login_with_sec_user_id_continues_to_im(self):
|
||||
worker = DouyinWorker(account_id=306, login_mode="browser")
|
||||
worker.is_running = True
|
||||
worker._load_user_agent = AsyncMock(return_value="test-agent")
|
||||
worker._probe_existing_login = AsyncMock(return_value=True)
|
||||
worker._finalize_login_session = AsyncMock()
|
||||
worker._require_sec_user_id = AsyncMock(return_value=True)
|
||||
worker._setup_im_network_listener = AsyncMock()
|
||||
worker._navigate_to_message_center = AsyncMock()
|
||||
worker._harvest_im_credentials = AsyncMock()
|
||||
worker._persist_cookies = AsyncMock()
|
||||
im_session = DouyinImSession(
|
||||
cookies={"sessionid": "test-session"},
|
||||
my_uid=30601,
|
||||
)
|
||||
worker._build_im_session = AsyncMock(return_value=im_session)
|
||||
worker._persist_im_session = AsyncMock()
|
||||
worker._run_im_direct_service = AsyncMock()
|
||||
worker._close_browser_only = AsyncMock()
|
||||
|
||||
page = SimpleNamespace()
|
||||
context = SimpleNamespace(
|
||||
add_init_script=AsyncMock(),
|
||||
new_page=AsyncMock(return_value=page),
|
||||
)
|
||||
browser = SimpleNamespace(new_context=AsyncMock(return_value=context))
|
||||
playwright_starter = SimpleNamespace(
|
||||
start=AsyncMock(return_value=SimpleNamespace()),
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def browser_slot(_account_id: int, _description: str):
|
||||
yield
|
||||
|
||||
controller = SimpleNamespace(browser_slot=browser_slot)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"rpa_engine.playwright_worker.async_playwright",
|
||||
return_value=playwright_starter,
|
||||
),
|
||||
patch(
|
||||
"rpa_engine.playwright_worker._launch_chromium",
|
||||
new=AsyncMock(return_value=browser),
|
||||
),
|
||||
patch(
|
||||
"rpa_engine.playwright_worker.get_traffic_controller",
|
||||
return_value=controller,
|
||||
),
|
||||
patch(
|
||||
"rpa_engine.playwright_worker.validate_im_session",
|
||||
new=AsyncMock(return_value=(True, "ready")),
|
||||
),
|
||||
):
|
||||
await worker._run_browser_im_flow(
|
||||
storage_state=None,
|
||||
cookie_info={
|
||||
"cookie_valid": False,
|
||||
"has_sessionid": False,
|
||||
"reason": "no saved login",
|
||||
},
|
||||
)
|
||||
|
||||
worker._require_sec_user_id.assert_awaited_once()
|
||||
self.assertTrue(worker._require_sec_user_id.await_args.kwargs["force_refresh"])
|
||||
worker._setup_im_network_listener.assert_awaited_once_with()
|
||||
worker._navigate_to_message_center.assert_awaited_once_with()
|
||||
worker._harvest_im_credentials.assert_awaited_once_with(timeout=25)
|
||||
worker._build_im_session.assert_awaited_once_with()
|
||||
worker._persist_im_session.assert_awaited_once_with(
|
||||
im_session,
|
||||
status="online",
|
||||
clear_error=True,
|
||||
)
|
||||
worker._run_im_direct_service.assert_awaited_once_with(im_session)
|
||||
worker._close_browser_only.assert_awaited()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,284 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
os.environ["KEFU_DB_TYPE"] = "sqlite"
|
||||
os.environ["KEFU_DATABASE_URL"] = ""
|
||||
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from rpa_engine.douyin_im import http_client as http_client_module
|
||||
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
|
||||
from rpa_engine.douyin_im.session import DouyinImSession
|
||||
from rpa_engine.playwright_worker import DouyinWorker
|
||||
|
||||
|
||||
class SendTextMessageEntryTests(unittest.IsolatedAsyncioTestCase):
|
||||
def _make_client(self, account_id: int = 41) -> DouyinImHttpClient:
|
||||
session = DouyinImSession(
|
||||
cookies={"sessionid": "test-session"},
|
||||
my_uid=10001,
|
||||
conv_meta={"existing": {"ticket": "old-ticket"}},
|
||||
)
|
||||
return DouyinImHttpClient(session, account_id=account_id)
|
||||
|
||||
async def test_public_entry_submits_the_whole_send_to_outbound_queue(self):
|
||||
client = self._make_client(account_id=73)
|
||||
submit = AsyncMock(return_value=True)
|
||||
|
||||
with patch(
|
||||
"rpa_engine.douyin_im.traffic_control.submit_outbound",
|
||||
submit,
|
||||
):
|
||||
sent = await client.send_text_message(
|
||||
"0:1:10001:20002",
|
||||
"hello",
|
||||
conversation_short_id="short-1",
|
||||
)
|
||||
|
||||
self.assertTrue(sent)
|
||||
submit.assert_awaited_once()
|
||||
account_id, operation = submit.await_args.args[:2]
|
||||
self.assertEqual(account_id, 73)
|
||||
self.assertTrue(callable(operation))
|
||||
self.assertIn("20002", submit.await_args.kwargs["description"])
|
||||
|
||||
async def test_bypass_path_does_not_submit_again(self):
|
||||
client = self._make_client()
|
||||
client._resolve_authoritative_uid = MagicMock(return_value=0)
|
||||
submit = AsyncMock(return_value=True)
|
||||
|
||||
with patch(
|
||||
"rpa_engine.douyin_im.traffic_control.submit_outbound",
|
||||
submit,
|
||||
):
|
||||
sent = await client.send_text_message(
|
||||
"0:1:10001:20002",
|
||||
"hello",
|
||||
_bypass_global_queue=True,
|
||||
)
|
||||
|
||||
self.assertFalse(sent)
|
||||
submit.assert_not_awaited()
|
||||
client._resolve_authoritative_uid.assert_called_once()
|
||||
|
||||
async def test_queued_client_result_state_is_copied_to_calling_client(self):
|
||||
client = self._make_client(account_id=88)
|
||||
queued_meta = {
|
||||
"0:1:10001:20002": {
|
||||
"ticket": "fresh-ticket",
|
||||
"conversation_short_id": "fresh-short-id",
|
||||
}
|
||||
}
|
||||
queued_client = SimpleNamespace(
|
||||
send_text_message=AsyncMock(return_value=False),
|
||||
last_send_meta=queued_meta,
|
||||
last_error="credential expired",
|
||||
last_send_needs_refresh=True,
|
||||
last_request_debug="response status=401",
|
||||
)
|
||||
queued_context = MagicMock()
|
||||
queued_context.__aenter__ = AsyncMock(return_value=queued_client)
|
||||
queued_context.__aexit__ = AsyncMock(return_value=None)
|
||||
queued_factory = MagicMock(return_value=queued_context)
|
||||
|
||||
async def execute_submission(account_id, operation, description=""):
|
||||
self.assertEqual(account_id, 88)
|
||||
self.assertIn("20002", description)
|
||||
return await operation()
|
||||
|
||||
submit = AsyncMock(side_effect=execute_submission)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"rpa_engine.douyin_im.traffic_control.submit_outbound",
|
||||
submit,
|
||||
),
|
||||
patch.object(
|
||||
http_client_module,
|
||||
"DouyinImHttpClient",
|
||||
queued_factory,
|
||||
),
|
||||
):
|
||||
sent = await client.send_text_message(
|
||||
"0:1:10001:20002",
|
||||
"queued hello",
|
||||
conversation_short_id="short-before-send",
|
||||
)
|
||||
|
||||
self.assertFalse(sent)
|
||||
submit.assert_awaited_once()
|
||||
queued_factory.assert_called_once_with(client.session, account_id=88)
|
||||
queued_client.send_text_message.assert_awaited_once_with(
|
||||
"0:1:10001:20002",
|
||||
"queued hello",
|
||||
conversation_short_id="short-before-send",
|
||||
_bypass_global_queue=True,
|
||||
)
|
||||
self.assertEqual(client.last_send_meta, queued_meta)
|
||||
self.assertIsNot(client.last_send_meta, queued_meta)
|
||||
self.assertEqual(client.last_error, "credential expired")
|
||||
self.assertTrue(client.last_send_needs_refresh)
|
||||
self.assertEqual(client.last_request_debug, "response status=401")
|
||||
|
||||
|
||||
class WorkerLifecycleTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_start_saves_task_and_stop_waits_until_it_is_done(self):
|
||||
worker = DouyinWorker(account_id=919, login_mode="im_direct")
|
||||
loop_started = asyncio.Event()
|
||||
loop_finished = asyncio.Event()
|
||||
never_finishes_without_cancellation = asyncio.Event()
|
||||
|
||||
async def controlled_run_loop() -> None:
|
||||
loop_started.set()
|
||||
try:
|
||||
await never_finishes_without_cancellation.wait()
|
||||
finally:
|
||||
loop_finished.set()
|
||||
|
||||
worker._run_loop = controlled_run_loop
|
||||
worker.update_account_status = AsyncMock()
|
||||
worker.cleanup = AsyncMock()
|
||||
|
||||
await worker.start()
|
||||
task = worker._task
|
||||
|
||||
self.assertIsInstance(task, asyncio.Task)
|
||||
self.assertEqual(task.get_name(), "douyin-worker-919")
|
||||
self.assertFalse(task.done())
|
||||
await asyncio.wait_for(loop_started.wait(), timeout=0.2)
|
||||
|
||||
# Starting an already-running worker must retain the same owned task.
|
||||
await worker.start()
|
||||
self.assertIs(worker._task, task)
|
||||
|
||||
await asyncio.wait_for(worker.stop(), timeout=0.5)
|
||||
|
||||
self.assertTrue(task.done())
|
||||
self.assertTrue(task.cancelled())
|
||||
self.assertTrue(loop_finished.is_set())
|
||||
self.assertIsNone(worker._task)
|
||||
self.assertTrue(worker.stopping)
|
||||
self.assertFalse(worker.is_running)
|
||||
worker.update_account_status.assert_awaited_once_with("offline")
|
||||
worker.cleanup.assert_awaited_once_with()
|
||||
leaked = [
|
||||
running
|
||||
for running in asyncio.all_tasks()
|
||||
if running is not asyncio.current_task()
|
||||
and running.get_name() == "douyin-worker-919"
|
||||
and not running.done()
|
||||
]
|
||||
self.assertEqual(leaked, [])
|
||||
|
||||
async def test_browser_flow_closes_resources_before_releasing_slot_on_error(self):
|
||||
worker = DouyinWorker(account_id=920, login_mode="browser")
|
||||
events: list[str] = []
|
||||
|
||||
# A truthy browser resource makes the flow's finally block responsible
|
||||
# for cleanup even though credential preparation fails midway.
|
||||
worker.page = object()
|
||||
|
||||
async def failing_prepare(storage_state, cookie_info):
|
||||
events.append("prepare")
|
||||
raise RuntimeError("credential harvest failed")
|
||||
|
||||
async def close_browser_only():
|
||||
events.append("close")
|
||||
worker.page = None
|
||||
|
||||
@asynccontextmanager
|
||||
async def browser_slot(account_id, description):
|
||||
self.assertEqual(account_id, 920)
|
||||
self.assertEqual(description, "browser credential harvest")
|
||||
events.append("acquire")
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
events.append("release")
|
||||
|
||||
controller = SimpleNamespace(browser_slot=browser_slot)
|
||||
worker._prepare_browser_im_flow = AsyncMock(side_effect=failing_prepare)
|
||||
worker._close_browser_only = AsyncMock(side_effect=close_browser_only)
|
||||
|
||||
with patch(
|
||||
"rpa_engine.playwright_worker.get_traffic_controller",
|
||||
return_value=controller,
|
||||
):
|
||||
with self.assertRaisesRegex(RuntimeError, "credential harvest failed"):
|
||||
await worker._run_browser_im_flow(
|
||||
storage_state={"cookies": []},
|
||||
cookie_info={"has_sessionid": True},
|
||||
)
|
||||
|
||||
self.assertEqual(events, ["acquire", "prepare", "close", "release"])
|
||||
worker._prepare_browser_im_flow.assert_awaited_once_with(
|
||||
{"cookies": []},
|
||||
{"has_sessionid": True},
|
||||
)
|
||||
worker._close_browser_only.assert_awaited_once_with()
|
||||
|
||||
async def test_close_browser_only_closes_every_resource_and_clears_references(self):
|
||||
worker = DouyinWorker(account_id=921, login_mode="browser")
|
||||
events: list[str] = []
|
||||
|
||||
async def record(name: str, *, fail: bool = False) -> None:
|
||||
events.append(name)
|
||||
if fail:
|
||||
raise RuntimeError(f"{name} close failed")
|
||||
|
||||
async def close_page() -> None:
|
||||
await record("page")
|
||||
|
||||
async def close_context() -> None:
|
||||
await record("context", fail=True)
|
||||
|
||||
async def close_browser() -> None:
|
||||
await record("browser")
|
||||
|
||||
async def stop_playwright() -> None:
|
||||
await record("playwright")
|
||||
|
||||
page = SimpleNamespace(
|
||||
close=AsyncMock(side_effect=close_page),
|
||||
)
|
||||
context = SimpleNamespace(
|
||||
# One failed close must not keep the remaining processes alive.
|
||||
close=AsyncMock(side_effect=close_context),
|
||||
)
|
||||
browser = SimpleNamespace(
|
||||
close=AsyncMock(side_effect=close_browser),
|
||||
)
|
||||
playwright = SimpleNamespace(
|
||||
stop=AsyncMock(side_effect=stop_playwright),
|
||||
)
|
||||
worker.page = page
|
||||
worker.context = context
|
||||
worker.browser = browser
|
||||
worker.playwright = playwright
|
||||
|
||||
await worker._close_browser_only()
|
||||
|
||||
self.assertEqual(events, ["page", "context", "browser", "playwright"])
|
||||
page.close.assert_awaited_once_with()
|
||||
context.close.assert_awaited_once_with()
|
||||
browser.close.assert_awaited_once_with()
|
||||
playwright.stop.assert_awaited_once_with()
|
||||
self.assertIsNone(worker.page)
|
||||
self.assertIsNone(worker.context)
|
||||
self.assertIsNone(worker.browser)
|
||||
self.assertIsNone(worker.playwright)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,495 @@
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
BACKEND_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(BACKEND_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(BACKEND_DIR))
|
||||
|
||||
from rpa_engine.douyin_im.traffic_control import (
|
||||
GlobalSendQueue,
|
||||
TrafficController,
|
||||
get_traffic_controller,
|
||||
)
|
||||
|
||||
|
||||
class GlobalSendQueueTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def _make_queue(self, interval_seconds: float = 0.0) -> GlobalSendQueue:
|
||||
queue = GlobalSendQueue(interval_seconds=interval_seconds)
|
||||
self.addAsyncCleanup(queue.stop)
|
||||
return queue
|
||||
|
||||
async def _wait_for_queued_count(
|
||||
self,
|
||||
queue: GlobalSendQueue,
|
||||
expected: int,
|
||||
timeout: float = 0.5,
|
||||
) -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
while loop.time() < deadline:
|
||||
if (await queue.snapshot())["queued_count"] == expected:
|
||||
return
|
||||
await asyncio.sleep(0.001)
|
||||
self.assertEqual((await queue.snapshot())["queued_count"], expected)
|
||||
|
||||
async def test_global_send_concurrency_is_one_across_accounts(self):
|
||||
queue = await self._make_queue()
|
||||
active = 0
|
||||
maximum_active = 0
|
||||
|
||||
async def operation(index: int) -> int:
|
||||
nonlocal active, maximum_active
|
||||
active += 1
|
||||
maximum_active = max(maximum_active, active)
|
||||
await asyncio.sleep(0.015)
|
||||
active -= 1
|
||||
return index
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(
|
||||
queue.submit(
|
||||
account_id=(index % 3) + 1,
|
||||
operation=lambda index=index: operation(index),
|
||||
description=f"send-{index}",
|
||||
)
|
||||
)
|
||||
for index in range(9)
|
||||
]
|
||||
|
||||
results = await asyncio.wait_for(asyncio.gather(*tasks), timeout=1.0)
|
||||
|
||||
self.assertEqual(sorted(results), list(range(9)))
|
||||
self.assertEqual(maximum_active, 1)
|
||||
self.assertEqual(active, 0)
|
||||
|
||||
async def test_same_account_keeps_fifo_order(self):
|
||||
queue = await self._make_queue()
|
||||
calls: list[int] = []
|
||||
|
||||
async def operation(index: int) -> int:
|
||||
calls.append(index)
|
||||
await asyncio.sleep(0)
|
||||
return index
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(
|
||||
queue.submit(
|
||||
account_id=101,
|
||||
operation=lambda index=index: operation(index),
|
||||
description=str(index),
|
||||
)
|
||||
)
|
||||
for index in range(6)
|
||||
]
|
||||
|
||||
results = await asyncio.wait_for(asyncio.gather(*tasks), timeout=0.5)
|
||||
|
||||
self.assertEqual(calls, list(range(6)))
|
||||
self.assertEqual(results, list(range(6)))
|
||||
|
||||
async def test_busy_accounts_are_dispatched_round_robin(self):
|
||||
queue = await self._make_queue()
|
||||
calls: list[str] = []
|
||||
first_started = asyncio.Event()
|
||||
release_first = asyncio.Event()
|
||||
|
||||
async def first_operation() -> str:
|
||||
calls.append("a1")
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
return "a1"
|
||||
|
||||
async def operation(name: str) -> str:
|
||||
calls.append(name)
|
||||
await asyncio.sleep(0)
|
||||
return name
|
||||
|
||||
first = asyncio.create_task(queue.submit(1, first_operation, "a1"))
|
||||
await asyncio.wait_for(first_started.wait(), timeout=0.2)
|
||||
|
||||
# Account 1 is deliberately noisy. Accounts 2 and 3 must each get a
|
||||
# turn before account 1 is allowed to dispatch its next waiting job.
|
||||
remaining_specs = [
|
||||
(1, "a2"),
|
||||
(1, "a3"),
|
||||
(2, "b1"),
|
||||
(2, "b2"),
|
||||
(3, "c1"),
|
||||
]
|
||||
remaining = [
|
||||
asyncio.create_task(
|
||||
queue.submit(
|
||||
account_id,
|
||||
lambda name=name: operation(name),
|
||||
description=name,
|
||||
)
|
||||
)
|
||||
for account_id, name in remaining_specs
|
||||
]
|
||||
await self._wait_for_queued_count(queue, expected=len(remaining))
|
||||
|
||||
release_first.set()
|
||||
results = await asyncio.wait_for(
|
||||
asyncio.gather(first, *remaining),
|
||||
timeout=0.5,
|
||||
)
|
||||
|
||||
self.assertEqual(calls, ["a1", "b1", "c1", "a2", "b2", "a3"])
|
||||
self.assertCountEqual(results, ["a1", "a2", "a3", "b1", "b2", "c1"])
|
||||
|
||||
async def test_send_starts_are_separated_by_configured_interval(self):
|
||||
interval = 0.08
|
||||
queue = await self._make_queue(interval_seconds=interval)
|
||||
loop = asyncio.get_running_loop()
|
||||
started_at: list[float] = []
|
||||
|
||||
async def operation() -> None:
|
||||
started_at.append(loop.time())
|
||||
|
||||
tasks = [
|
||||
asyncio.create_task(queue.submit(index + 1, operation, str(index)))
|
||||
for index in range(3)
|
||||
]
|
||||
await asyncio.wait_for(asyncio.gather(*tasks), timeout=0.7)
|
||||
|
||||
self.assertEqual(len(started_at), 3)
|
||||
gaps = [later - earlier for earlier, later in zip(started_at, started_at[1:])]
|
||||
for gap in gaps:
|
||||
# The Windows selector clock may wake one ~15.6 ms tick early.
|
||||
self.assertGreaterEqual(gap, interval - 0.025)
|
||||
|
||||
async def test_operation_failure_does_not_block_later_job(self):
|
||||
queue = await self._make_queue()
|
||||
calls: list[str] = []
|
||||
|
||||
async def failing_operation() -> None:
|
||||
calls.append("failed")
|
||||
raise ValueError("expected failure")
|
||||
|
||||
async def following_operation() -> str:
|
||||
calls.append("continued")
|
||||
return "ok"
|
||||
|
||||
failed = asyncio.create_task(queue.submit(1, failing_operation, "failed"))
|
||||
continued = asyncio.create_task(queue.submit(2, following_operation, "continued"))
|
||||
|
||||
results = await asyncio.wait_for(
|
||||
asyncio.gather(failed, continued, return_exceptions=True),
|
||||
timeout=0.2,
|
||||
)
|
||||
|
||||
self.assertIsInstance(results[0], ValueError)
|
||||
self.assertEqual(str(results[0]), "expected failure")
|
||||
self.assertEqual(results[1], "ok")
|
||||
self.assertEqual(calls, ["failed", "continued"])
|
||||
|
||||
async def test_cancelling_waiting_submit_prevents_operation(self):
|
||||
queue = await self._make_queue()
|
||||
first_started = asyncio.Event()
|
||||
release_first = asyncio.Event()
|
||||
cancelled_operation_called = asyncio.Event()
|
||||
|
||||
async def blocking_operation() -> None:
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
|
||||
async def must_not_run() -> None:
|
||||
cancelled_operation_called.set()
|
||||
|
||||
blocker = asyncio.create_task(queue.submit(1, blocking_operation, "blocker"))
|
||||
await asyncio.wait_for(first_started.wait(), timeout=0.2)
|
||||
waiting = asyncio.create_task(queue.submit(2, must_not_run, "cancelled"))
|
||||
await self._wait_for_queued_count(queue, expected=1)
|
||||
|
||||
waiting.cancel()
|
||||
with self.assertRaises(asyncio.CancelledError):
|
||||
await waiting
|
||||
release_first.set()
|
||||
await asyncio.wait_for(blocker, timeout=0.2)
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
self.assertFalse(cancelled_operation_called.is_set())
|
||||
self.assertEqual((await queue.snapshot())["pending_count"], 0)
|
||||
|
||||
async def test_cancelling_active_submit_waits_for_real_result_without_overlap(self):
|
||||
queue = await self._make_queue()
|
||||
active = 0
|
||||
maximum_active = 0
|
||||
first_started = asyncio.Event()
|
||||
release_first = asyncio.Event()
|
||||
second_started = asyncio.Event()
|
||||
|
||||
async def first_operation() -> str:
|
||||
nonlocal active, maximum_active
|
||||
active += 1
|
||||
maximum_active = max(maximum_active, active)
|
||||
first_started.set()
|
||||
await release_first.wait()
|
||||
active -= 1
|
||||
return "delivered"
|
||||
|
||||
async def second_operation() -> str:
|
||||
nonlocal active, maximum_active
|
||||
active += 1
|
||||
maximum_active = max(maximum_active, active)
|
||||
second_started.set()
|
||||
await asyncio.sleep(0)
|
||||
active -= 1
|
||||
return "next"
|
||||
|
||||
first = asyncio.create_task(queue.submit(1, first_operation, "active"))
|
||||
await asyncio.wait_for(first_started.wait(), timeout=0.2)
|
||||
second = asyncio.create_task(queue.submit(2, second_operation, "next"))
|
||||
await self._wait_for_queued_count(queue, expected=1)
|
||||
|
||||
first.cancel()
|
||||
await asyncio.sleep(0.02)
|
||||
|
||||
# Once the network operation has begun, cancelling its caller must not
|
||||
# release the single lane or report a false cancellation to the caller.
|
||||
self.assertFalse(first.done())
|
||||
self.assertFalse(second_started.is_set())
|
||||
self.assertEqual(active, 1)
|
||||
|
||||
release_first.set()
|
||||
self.assertEqual(await asyncio.wait_for(first, timeout=0.2), "delivered")
|
||||
self.assertEqual(await asyncio.wait_for(second, timeout=0.2), "next")
|
||||
self.assertTrue(second_started.is_set())
|
||||
self.assertEqual(maximum_active, 1)
|
||||
self.assertEqual(active, 0)
|
||||
|
||||
async def test_cancel_account_removes_waiting_jobs_without_running_them(self):
|
||||
queue = await self._make_queue()
|
||||
blocker_started = asyncio.Event()
|
||||
release_blocker = asyncio.Event()
|
||||
removed_calls: list[str] = []
|
||||
|
||||
async def blocking_operation() -> None:
|
||||
blocker_started.set()
|
||||
await release_blocker.wait()
|
||||
|
||||
async def removed_operation(name: str) -> None:
|
||||
removed_calls.append(name)
|
||||
|
||||
blocker = asyncio.create_task(queue.submit(1, blocking_operation, "blocker"))
|
||||
await asyncio.wait_for(blocker_started.wait(), timeout=0.2)
|
||||
removed = [
|
||||
asyncio.create_task(
|
||||
queue.submit(
|
||||
77,
|
||||
lambda name=name: removed_operation(name),
|
||||
name,
|
||||
)
|
||||
)
|
||||
for name in ("waiting-1", "waiting-2")
|
||||
]
|
||||
await self._wait_for_queued_count(queue, expected=2)
|
||||
|
||||
self.assertEqual(await queue.cancel_account(77), 2)
|
||||
results = await asyncio.wait_for(
|
||||
asyncio.gather(*removed, return_exceptions=True),
|
||||
timeout=0.1,
|
||||
)
|
||||
snapshot = await queue.snapshot()
|
||||
|
||||
self.assertTrue(all(isinstance(result, asyncio.CancelledError) for result in results))
|
||||
self.assertEqual(removed_calls, [])
|
||||
self.assertNotIn(77, snapshot["per_account"])
|
||||
self.assertEqual(snapshot["queued_count"], 0)
|
||||
self.assertEqual(snapshot["active_account_id"], 1)
|
||||
|
||||
release_blocker.set()
|
||||
await asyncio.wait_for(blocker, timeout=0.2)
|
||||
await asyncio.sleep(0)
|
||||
self.assertEqual((await queue.snapshot())["pending_count"], 0)
|
||||
|
||||
async def test_cancel_account_releases_active_caller_but_drains_lane(self):
|
||||
queue = await self._make_queue()
|
||||
active = 0
|
||||
maximum_active = 0
|
||||
active_started = asyncio.Event()
|
||||
release_active = asyncio.Event()
|
||||
active_drained = asyncio.Event()
|
||||
next_started = asyncio.Event()
|
||||
calls: list[str] = []
|
||||
|
||||
async def active_operation() -> str:
|
||||
nonlocal active, maximum_active
|
||||
active += 1
|
||||
maximum_active = max(maximum_active, active)
|
||||
calls.append("active-start")
|
||||
active_started.set()
|
||||
await release_active.wait()
|
||||
calls.append("active-drained")
|
||||
active -= 1
|
||||
active_drained.set()
|
||||
return "discarded-result"
|
||||
|
||||
async def next_operation() -> str:
|
||||
nonlocal active, maximum_active
|
||||
active += 1
|
||||
maximum_active = max(maximum_active, active)
|
||||
calls.append("next")
|
||||
next_started.set()
|
||||
await asyncio.sleep(0)
|
||||
active -= 1
|
||||
return "next-result"
|
||||
|
||||
active_submit = asyncio.create_task(
|
||||
queue.submit(88, active_operation, "active-account")
|
||||
)
|
||||
await asyncio.wait_for(active_started.wait(), timeout=0.2)
|
||||
next_submit = asyncio.create_task(queue.submit(89, next_operation, "other-account"))
|
||||
await self._wait_for_queued_count(queue, expected=1)
|
||||
|
||||
self.assertEqual(await queue.cancel_account(88), 1)
|
||||
cancelled_result = await asyncio.wait_for(
|
||||
asyncio.gather(active_submit, return_exceptions=True),
|
||||
timeout=0.1,
|
||||
)
|
||||
|
||||
self.assertIsInstance(cancelled_result[0], asyncio.CancelledError)
|
||||
self.assertTrue(active_submit.cancelled())
|
||||
self.assertFalse(active_drained.is_set())
|
||||
self.assertFalse(next_started.is_set())
|
||||
self.assertEqual(active, 1)
|
||||
|
||||
# The cancelled account's network operation still owns the lane until
|
||||
# it actually drains; only then can a different account begin.
|
||||
release_active.set()
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(next_submit, timeout=0.2),
|
||||
"next-result",
|
||||
)
|
||||
self.assertTrue(active_drained.is_set())
|
||||
self.assertEqual(calls, ["active-start", "active-drained", "next"])
|
||||
self.assertEqual(maximum_active, 1)
|
||||
self.assertEqual(active, 0)
|
||||
|
||||
# A fresh submission proves the dispatcher remains usable after the
|
||||
# cancelled active job and its queued successor have both completed.
|
||||
async def recovered_operation() -> str:
|
||||
calls.append("recovered")
|
||||
return "recovered-result"
|
||||
|
||||
self.assertEqual(
|
||||
await asyncio.wait_for(
|
||||
queue.submit(90, recovered_operation, "recovered"),
|
||||
timeout=0.2,
|
||||
),
|
||||
"recovered-result",
|
||||
)
|
||||
self.assertEqual(calls[-1], "recovered")
|
||||
self.assertEqual((await queue.snapshot())["pending_count"], 0)
|
||||
|
||||
|
||||
class BackgroundTrafficLimitTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_background_slot_respects_configured_concurrency_limit(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{"KEFU_BACKGROUND_NETWORK_CONCURRENCY": "2"},
|
||||
):
|
||||
controller = TrafficController()
|
||||
self.addAsyncCleanup(controller.stop)
|
||||
|
||||
active = 0
|
||||
maximum_active = 0
|
||||
entered = 0
|
||||
all_finished = asyncio.Event()
|
||||
|
||||
async def request(account_id: int) -> None:
|
||||
nonlocal active, maximum_active, entered
|
||||
async with controller.background_slot(account_id, "poll"):
|
||||
active += 1
|
||||
entered += 1
|
||||
maximum_active = max(maximum_active, active)
|
||||
await asyncio.sleep(0.02)
|
||||
active -= 1
|
||||
if entered == 8 and active == 0:
|
||||
all_finished.set()
|
||||
|
||||
tasks = [asyncio.create_task(request(index + 1)) for index in range(8)]
|
||||
await asyncio.wait_for(asyncio.gather(*tasks), timeout=0.5)
|
||||
|
||||
self.assertEqual(entered, 8)
|
||||
self.assertEqual(maximum_active, 2)
|
||||
self.assertEqual(active, 0)
|
||||
self.assertEqual(controller.background_active, 0)
|
||||
self.assertEqual(controller.background_waiting, 0)
|
||||
|
||||
async def test_nested_background_slot_in_same_task_is_reentrant(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
{"KEFU_BACKGROUND_NETWORK_CONCURRENCY": "1"},
|
||||
):
|
||||
controller = TrafficController()
|
||||
self.addAsyncCleanup(controller.stop)
|
||||
|
||||
tasks_seen: list[asyncio.Task] = []
|
||||
|
||||
async def nested_request() -> None:
|
||||
tasks_seen.append(asyncio.current_task())
|
||||
async with controller.background_slot(1, "outer"):
|
||||
self.assertEqual(controller.background_active, 1)
|
||||
self.assertEqual(controller.background_waiting, 0)
|
||||
self.assertEqual(controller._background._value, 0)
|
||||
|
||||
tasks_seen.append(asyncio.current_task())
|
||||
async with controller.background_slot(1, "inner"):
|
||||
tasks_seen.append(asyncio.current_task())
|
||||
self.assertEqual(controller.background_active, 1)
|
||||
self.assertEqual(controller.background_waiting, 0)
|
||||
self.assertEqual(controller._background._value, 0)
|
||||
|
||||
self.assertEqual(controller.background_active, 1)
|
||||
self.assertEqual(controller._background._value, 0)
|
||||
|
||||
self.assertEqual(controller.background_active, 0)
|
||||
self.assertEqual(controller.background_waiting, 0)
|
||||
self.assertEqual(controller._background._value, 1)
|
||||
|
||||
await asyncio.wait_for(nested_request(), timeout=0.2)
|
||||
|
||||
self.assertEqual(len(tasks_seen), 3)
|
||||
self.assertTrue(all(task is tasks_seen[0] for task in tasks_seen))
|
||||
|
||||
|
||||
class TrafficControllerLoopIsolationTests(unittest.TestCase):
|
||||
def test_get_traffic_controller_does_not_reuse_asyncio_primitives(self):
|
||||
async def capture_controller_state():
|
||||
controller = get_traffic_controller()
|
||||
return {
|
||||
"loop": asyncio.get_running_loop(),
|
||||
"controller": controller,
|
||||
"send_lock": controller.send_queue._state_lock,
|
||||
"send_wake": controller.send_queue._wake,
|
||||
"background": controller._background,
|
||||
"browser": controller._browser,
|
||||
}
|
||||
|
||||
# IsolatedAsyncioTestCase uses a fresh asyncio.Runner per test. Keep
|
||||
# both closed loop objects alive so the WeakKeyDictionary is exercised
|
||||
# with two distinct keys rather than relying on garbage collection.
|
||||
with asyncio.Runner() as first_runner:
|
||||
first = first_runner.run(capture_controller_state())
|
||||
with asyncio.Runner() as second_runner:
|
||||
second = second_runner.run(capture_controller_state())
|
||||
|
||||
for key in (
|
||||
"loop",
|
||||
"controller",
|
||||
"send_lock",
|
||||
"send_wake",
|
||||
"background",
|
||||
"browser",
|
||||
):
|
||||
self.assertIsNot(first[key], second[key], key)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user