This commit is contained in:
Your Name
2026-07-23 17:56:25 +08:00
parent a05dae8412
commit 4970d8f8d3
4262 changed files with 735221 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
+57
View File
@@ -0,0 +1,57 @@
"""用户可添加抖音账号数量限制(限制功能已移除,保留接口兼容)。"""
from __future__ import annotations
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from models.models import Account, User
from .roles import is_admin
UNLIMITED_ACCOUNTS = -1
def normalize_max_accounts(value: int | None, role: str = "operator") -> int:
if is_admin(role):
return UNLIMITED_ACCOUNTS
if value is None:
return 3
try:
parsed = int(value)
except (TypeError, ValueError):
return 3
if parsed < 0:
return UNLIMITED_ACCOUNTS
return parsed
def account_limit_for_user(user: User) -> int | None:
"""账号数量/并发限制已移除,始终不限制。"""
return None
async def count_user_accounts(db: AsyncSession, user_id: int) -> int:
result = await db.execute(
select(func.count()).select_from(Account).where(Account.owner_id == user_id)
)
return int(result.scalar() or 0)
async def count_user_account_breakdown(db: AsyncSession, user_id: int) -> dict[str, int]:
"""统计用户名下托管账号:总数 / 可用 / 额度停用。"""
total = await count_user_accounts(db, user_id)
if total <= 0:
return {"total": 0, "active": 0, "disabled": 0}
disabled_result = await db.execute(
select(func.count())
.select_from(Account)
.where(Account.owner_id == user_id, Account.quota_disabled.is_(True))
)
disabled = int(disabled_result.scalar() or 0)
active = max(0, total - disabled)
return {"total": total, "active": active, "disabled": disabled}
async def ensure_can_add_account(db: AsyncSession, user: User) -> None:
"""账号数量限制已移除,任何用户可添加任意数量账号。"""
return
+68
View File
@@ -0,0 +1,68 @@
"""账号额度与 quota_disabled 同步。"""
from __future__ import annotations
from collections.abc import Awaitable, Callable
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models.models import Account, User
from .account_limits import account_limit_for_user
QUOTA_DISABLED_MESSAGE = "账号额度不足,该账号已被系统停用"
async def sync_user_account_quota(
db: AsyncSession,
user: User,
*,
stop_worker: Callable[[int], Awaitable[None]] | None = None,
) -> dict[str, Any]:
"""按 max_accounts 同步停用状态:保留最早创建的账号,超额账号禁用并停止托管。"""
limit = account_limit_for_user(user)
result = await db.execute(
select(Account)
.where(Account.owner_id == user.id)
.order_by(Account.created_at.asc(), Account.id.asc())
)
accounts = list(result.scalars().all())
disabled_ids: list[int] = []
enabled_ids: list[int] = []
if limit is None:
for acc in accounts:
if acc.quota_disabled:
acc.quota_disabled = False
if (acc.error_message or "").strip() == QUOTA_DISABLED_MESSAGE:
acc.error_message = None
enabled_ids.append(acc.id)
return {"disabled_ids": disabled_ids, "enabled_ids": enabled_ids, "limit": None}
for idx, acc in enumerate(accounts):
if idx < limit:
if acc.quota_disabled:
acc.quota_disabled = False
if (acc.error_message or "").strip() == QUOTA_DISABLED_MESSAGE:
acc.error_message = None
enabled_ids.append(acc.id)
else:
if not acc.quota_disabled:
acc.quota_disabled = True
acc.error_message = QUOTA_DISABLED_MESSAGE
acc.status = "offline"
acc.qr_code_base64 = None
disabled_ids.append(acc.id)
if stop_worker:
await stop_worker(acc.id)
return {"disabled_ids": disabled_ids, "enabled_ids": enabled_ids, "limit": limit}
async def default_stop_worker(account_id: int) -> None:
from main import manager
await manager.stop_worker(account_id)
+50
View File
@@ -0,0 +1,50 @@
from fastapi import Depends, HTTPException, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models.database import get_db
from models.models import User
from .jwt_utils import decode_access_token
from .roles import can_manage_users, can_write, is_admin
bearer_scheme = HTTPBearer(auto_error=False)
async def get_current_user(
credentials: HTTPAuthorizationCredentials = Depends(bearer_scheme),
db: AsyncSession = Depends(get_db),
) -> User:
if not credentials or not credentials.credentials:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="未登录或令牌缺失")
payload = decode_access_token(credentials.credentials)
if not payload or not payload.get("sub"):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="登录已过期,请重新登录")
try:
user_id = int(payload["sub"])
except (TypeError, ValueError):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="无效令牌")
result = await db.execute(select(User).where(User.id == user_id))
user = result.scalar_one_or_none()
if not user or not user.is_active:
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户不存在或已禁用")
return user
async def require_admin(user: User = Depends(get_current_user)) -> User:
if not is_admin(user.role):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="需要管理员权限")
return user
async def require_write(user: User = Depends(get_current_user)) -> User:
if not can_write(user.role):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="当前角色只读,无法执行此操作")
return user
async def require_user_manager(user: User = Depends(get_current_user)) -> User:
if not can_manage_users(user.role):
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="需要管理员权限")
return user
+378
View File
@@ -0,0 +1,378 @@
"""发送邮箱验证邮件(标准库 SMTP)。"""
from __future__ import annotations
import asyncio
import logging
import smtplib
import ssl
from dataclasses import asdict
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import Any
from .email_templates import build_password_reset_content, build_test_email_content, build_verify_email_content
from .system_settings import SystemSettingsData
logger = logging.getLogger("auth.email")
SMTP_TIMEOUT_SECONDS = 45
# 阿里企业邮自定义域名 SMTP(如 smtp.xxiaw.com)常出现 535,官方地址可正常认证
ALIBABA_SMTP_OFFICIAL = "smtp.mxhichina.com"
_SSL_CONTEXT = ssl.create_default_context()
def _ssl_context(*, insecure: bool = False) -> ssl.SSLContext:
if not insecure:
return _SSL_CONTEXT
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return ctx
def _connect_ssl(host: str, port: int) -> smtplib.SMTP:
try:
return smtplib.SMTP_SSL(
host,
port,
timeout=SMTP_TIMEOUT_SECONDS,
context=_ssl_context(insecure=False),
)
except ssl.SSLCertVerificationError:
logger.warning(
"SMTP SSL certificate hostname mismatch for %s, retry without verify",
host,
)
return smtplib.SMTP_SSL(
host,
port,
timeout=SMTP_TIMEOUT_SECONDS,
context=_ssl_context(insecure=True),
)
def _starttls(server: smtplib.SMTP, host: str) -> None:
try:
server.starttls(context=_ssl_context(insecure=False))
except ssl.SSLCertVerificationError:
logger.warning(
"SMTP STARTTLS certificate hostname mismatch for %s, retry without verify",
host,
)
server.starttls(context=_ssl_context(insecure=True))
def build_verification_link(token: str, settings: SystemSettingsData) -> str:
return f"{settings.app_url_normalized()}/#/login?verify_token={token}"
def build_password_reset_link(token: str, settings: SystemSettingsData) -> str:
return f"{settings.app_url_normalized()}/#/login?reset_token={token}"
def resolve_smtp_mode(settings: SystemSettingsData) -> tuple[int, bool, bool]:
"""返回 (port, use_ssl, use_starttls)。"""
port = int(settings.smtp_port or 587)
use_ssl = bool(settings.smtp_use_ssl) or port == 465
# 465 只能 implicit SSL,绝不能 plain + STARTTLS
if port == 465:
use_ssl = True
use_starttls = False
elif port == 587:
use_ssl = False
use_starttls = bool(settings.smtp_use_tls)
else:
use_ssl = bool(settings.smtp_use_ssl)
use_starttls = bool(settings.smtp_use_tls) and not use_ssl
return port, use_ssl, use_starttls
def _open_smtp(settings: SystemSettingsData) -> smtplib.SMTP:
host = settings.smtp_host.strip()
port, use_ssl, use_starttls = resolve_smtp_mode(settings)
logger.info(
"SMTP connect host=%s port=%s ssl=%s starttls=%s user=%s",
host,
port,
use_ssl,
use_starttls,
settings.smtp_user or "",
)
if use_ssl:
server = _connect_ssl(host, port)
server.ehlo()
return server
server = smtplib.SMTP(host, port, timeout=SMTP_TIMEOUT_SECONDS)
server.ehlo()
if use_starttls:
_starttls(server, host)
server.ehlo()
return server
def _close_smtp(server: smtplib.SMTP) -> None:
try:
server.quit()
except Exception:
try:
server.close()
except Exception:
pass
def _login_smtp(server: smtplib.SMTP, settings: SystemSettingsData) -> tuple[smtplib.SMTP, bool]:
"""登录 SMTP,返回 (server, 是否使用了阿里官方地址回退)。"""
user = (settings.smtp_user or "").strip()
pwd = settings.smtp_password
if not user:
return server, False
if not pwd:
raise RuntimeError("未填写 SMTP 密码,请在设置中填写登录密码或客户端安全密码后重试")
try:
server.login(user, pwd)
return server, False
except smtplib.SMTPAuthenticationError as exc:
host = (settings.smtp_host or "").strip().lower()
if host == ALIBABA_SMTP_OFFICIAL:
raise RuntimeError(_format_login_error(exc, settings)) from exc
logger.warning(
"SMTP auth failed on %s (%s), retrying via %s",
host,
exc,
ALIBABA_SMTP_OFFICIAL,
)
_close_smtp(server)
fallback = SystemSettingsData(**{**asdict(settings), "smtp_host": ALIBABA_SMTP_OFFICIAL})
fallback_server = _open_smtp(fallback)
try:
fallback_server.login(user, pwd)
except Exception as retry_exc:
_close_smtp(fallback_server)
raise RuntimeError(_format_login_error(exc, settings)) from retry_exc
return fallback_server, True
def _format_login_error(exc: Exception, settings: SystemSettingsData) -> str:
if isinstance(exc, smtplib.SMTPAuthenticationError):
return _format_smtp_error(exc, settings)
if isinstance(exc, smtplib.SMTPServerDisconnected):
return (
"SMTP 服务器在登录时断开连接,通常不是加密方式问题,而是密码错误或账号未授权 SMTP。"
f"请确认用户名 {settings.smtp_user or ''} 与密码/客户端安全密码正确,"
"并在阿里企业邮后台开启「允许使用第三方客户端」。"
"若使用自定义域名 SMTP(如 smtp.xxiaw.com)仍失败,可改用 smtp.mxhichina.com 后重试。"
)
return _format_smtp_error(exc, settings)
def _format_smtp_error(exc: Exception, settings: SystemSettingsData) -> str:
msg = str(exc).strip() or exc.__class__.__name__
port, use_ssl, use_starttls = resolve_smtp_mode(settings)
hints: list[str] = []
if isinstance(exc, ssl.SSLCertVerificationError):
hints.append(
f"SSL 证书域名与 {settings.smtp_host} 不匹配(阿里企业邮自定义域名常见),"
"系统已尝试跳过证书校验;若仍失败请改用 smtp.mxhichina.com"
)
return f"{msg}。{' '.join(hints)}"
if isinstance(exc, smtplib.SMTPAuthenticationError):
host = (settings.smtp_host or "").strip().lower()
if host != ALIBABA_SMTP_OFFICIAL and host.startswith("smtp."):
hints.append(
f"若使用自定义域名 SMTP({settings.smtp_host}),阿里企业邮常返回 535;"
f"请将 SMTP 服务器改为 {ALIBABA_SMTP_OFFICIAL} 后重试"
)
hints.append(
"SMTP 登录失败:请确认登录用户名为完整邮箱地址,"
"密码为邮箱登录密码或阿里企业邮「客户端安全密码」(非网页登录密码时需在邮箱设置中单独生成)"
)
hints.append("请在阿里企业邮管理后台确认已开启「允许使用第三方客户端」")
return f"{msg}。{' '.join(hints)}"
lower = msg.lower()
if "connection unexpectedly closed" in lower and port == 465 and not use_ssl:
hints.append("465 端口必须使用 SSL 加密,不能勾选 STARTTLS")
elif "connection unexpectedly closed" in lower and port == 465:
if use_ssl:
hints.append(
"连接已建立但在后续步骤失败:请重点检查 SMTP 密码/客户端安全密码,"
"或尝试将服务器改为 smtp.mxhichina.com"
)
else:
hints.append("465 端口必须使用 SSL 加密,请选择「SSL / TLS(端口 465)」")
if "timed out" in lower or "timeout" in lower or "10060" in lower:
if port == 587:
hints.append(
f"587 端口 STARTTLS 连接 {settings.smtp_host} 超时:"
"该服务器可能仅支持 465 SSL,请将端口改为 465 并选择 SSL 加密"
)
else:
hints.append(
f"无法在 {SMTP_TIMEOUT_SECONDS}s 内连接 {settings.smtp_host}:{port},"
"请检查服务器地址、端口、防火墙/安全组是否放行出站 SMTP"
)
if "authentication" in lower or "535" in lower:
hints.append("认证失败:请检查 SMTP 密码/客户端安全密码是否正确")
if hints:
return f"{msg}。{' '.join(hints)}"
return msg
def diagnose_smtp(settings: SystemSettingsData) -> dict[str, Any]:
"""分步检测 SMTP 连接,便于定位配置问题。"""
host = (settings.smtp_host or "").strip()
port, use_ssl, use_starttls = resolve_smtp_mode(settings)
result: dict[str, Any] = {
"host": host,
"port": port,
"mode": "ssl" if use_ssl else ("starttls" if use_starttls else "plain"),
"steps": [],
"ok": False,
}
if not host:
result["steps"].append({"step": "config", "ok": False, "message": "未填写 SMTP 服务器"})
return result
server: smtplib.SMTP | None = None
try:
server = _open_smtp(settings)
result["steps"].append({"step": "connect", "ok": True, "message": "连接成功"})
except Exception as exc:
result["steps"].append(
{"step": "connect", "ok": False, "message": _format_smtp_error(exc, settings)}
)
return result
try:
if settings.smtp_user:
if not settings.smtp_password:
raise RuntimeError("未填写 SMTP 密码,请在设置中填写登录密码或客户端安全密码后重试")
host_before = (settings.smtp_host or "").strip().lower()
server, used_fallback = _login_smtp(server, settings)
login_msg = "登录成功"
if used_fallback and host_before != ALIBABA_SMTP_OFFICIAL:
login_msg = (
f"登录成功(已通过 {ALIBABA_SMTP_OFFICIAL} 认证,"
f"建议将 SMTP 服务器改为 {ALIBABA_SMTP_OFFICIAL})"
)
result["steps"].append({"step": "login", "ok": True, "message": login_msg})
else:
result["steps"].append({"step": "login", "ok": True, "message": "未配置用户名,跳过登录"})
result["ok"] = True
except Exception as exc:
result["steps"].append(
{"step": "login", "ok": False, "message": _format_login_error(exc, settings)}
)
finally:
if server:
try:
server.quit()
except Exception:
server.close()
return result
def _send_sync(
settings: SystemSettingsData,
to_email: str,
subject: str,
html_body: str,
text_body: str,
) -> None:
if not settings.smtp_configured():
raise RuntimeError("SMTP 未配置,无法发送邮件")
smtp_from = (settings.smtp_from or settings.smtp_user or "").strip()
msg = MIMEMultipart("alternative")
msg["Subject"] = subject
msg["From"] = smtp_from
msg["To"] = to_email
msg.attach(MIMEText(text_body, "plain", "utf-8"))
msg.attach(MIMEText(html_body, "html", "utf-8"))
server = _open_smtp(settings)
try:
if settings.smtp_user:
try:
server, _ = _login_smtp(server, settings)
except RuntimeError:
raise
except Exception as exc:
raise RuntimeError(_format_login_error(exc, settings)) from exc
server.sendmail(smtp_from, [to_email], msg.as_string())
except RuntimeError:
raise
except Exception as exc:
raise RuntimeError(_format_smtp_error(exc, settings)) from exc
finally:
try:
server.quit()
except Exception:
server.close()
async def send_password_reset_email(
to_email: str,
username: str,
token: str,
settings: SystemSettingsData,
) -> str:
"""发送密码重置邮件,返回重置链接。"""
link = build_password_reset_link(token, settings)
expire_hours = int(settings.email_verify_token_hours or 24)
subject, text_body, html_body = build_password_reset_content(
settings, username, to_email, link, expire_hours
)
if settings.smtp_configured():
await asyncio.to_thread(_send_sync, settings, to_email, subject, html_body, text_body)
logger.info("Password reset email sent to %s", to_email)
else:
logger.warning(
"SMTP 未配置,密码重置链接: user=%s email=%s link=%s",
username,
to_email,
link,
)
return link
async def send_verification_email(
to_email: str,
username: str,
token: str,
settings: SystemSettingsData,
) -> str:
"""发送验证邮件,返回验证链接。"""
link = build_verification_link(token, settings)
subject, text_body, html_body = build_verify_email_content(
settings, username, to_email, link
)
if settings.smtp_configured():
await asyncio.to_thread(_send_sync, settings, to_email, subject, html_body, text_body)
logger.info("Verification email sent to %s", to_email)
else:
logger.warning(
"SMTP 未配置,验证链接: user=%s email=%s link=%s",
username,
to_email,
link,
)
return link
async def send_test_email(to_email: str, settings: SystemSettingsData) -> None:
diagnosis = await asyncio.to_thread(diagnose_smtp, settings)
if not diagnosis.get("ok"):
failed = next((s for s in diagnosis.get("steps", []) if not s.get("ok")), None)
raise RuntimeError(failed["message"] if failed else "SMTP 连接检测失败")
subject, text_body, html_body = build_test_email_content(settings, to_email)
await asyncio.to_thread(_send_sync, settings, to_email, subject, html_body, text_body)
+103
View File
@@ -0,0 +1,103 @@
"""邮件模板渲染。"""
from __future__ import annotations
import html
import re
from .system_settings import (
APP_NAME,
DEFAULT_EMAIL_TEST_BODY,
DEFAULT_EMAIL_TEST_HTML,
DEFAULT_EMAIL_TEST_SUBJECT,
DEFAULT_EMAIL_VERIFY_BODY,
DEFAULT_EMAIL_VERIFY_HTML,
DEFAULT_EMAIL_VERIFY_SUBJECT,
DEFAULT_PASSWORD_RESET_BODY,
DEFAULT_PASSWORD_RESET_HTML,
DEFAULT_PASSWORD_RESET_SUBJECT,
SystemSettingsData,
)
_TEMPLATE_VAR_PATTERN = re.compile(r"\{(\w+)\}")
def render_email_template(template: str, **context: str) -> str:
"""安全渲染邮件模板,未知占位符保留原样。"""
value = template or ""
def replacer(match: re.Match) -> str:
key = match.group(1)
return context.get(key, match.group(0))
return _TEMPLATE_VAR_PATTERN.sub(replacer, value)
def _text_to_html(text: str) -> str:
escaped = html.escape(text or "")
return f'<pre style="font-family:sans-serif;white-space:pre-wrap;margin:0;">{escaped}</pre>'
def build_verify_email_content(
settings: SystemSettingsData,
username: str,
to_email: str,
link: str,
) -> tuple[str, str, str]:
context = {
"username": username,
"email": to_email,
"link": link,
"app_name": APP_NAME,
}
subject_tpl = settings.email_verify_subject or DEFAULT_EMAIL_VERIFY_SUBJECT
body_tpl = settings.email_verify_body or DEFAULT_EMAIL_VERIFY_BODY
html_tpl = settings.email_verify_html or ""
subject = render_email_template(subject_tpl, **context)
text_body = render_email_template(body_tpl, **context)
if html_tpl.strip():
html_body = render_email_template(html_tpl, **context)
else:
html_body = _text_to_html(text_body)
return subject, text_body, html_body
def build_password_reset_content(
settings: SystemSettingsData,
username: str,
to_email: str,
link: str,
expire_hours: int,
) -> tuple[str, str, str]:
context = {
"username": username,
"email": to_email,
"link": link,
"app_name": APP_NAME,
"expire_hours": str(expire_hours),
}
subject = render_email_template(DEFAULT_PASSWORD_RESET_SUBJECT, **context)
text_body = render_email_template(DEFAULT_PASSWORD_RESET_BODY, **context)
html_body = render_email_template(DEFAULT_PASSWORD_RESET_HTML, **context)
return subject, text_body, html_body
def build_test_email_content(settings: SystemSettingsData, to_email: str) -> tuple[str, str, str]:
context = {
"username": "测试用户",
"email": to_email,
"link": settings.app_url_normalized(),
"app_name": APP_NAME,
}
subject_tpl = settings.email_test_subject or DEFAULT_EMAIL_TEST_SUBJECT
body_tpl = settings.email_test_body or DEFAULT_EMAIL_TEST_BODY
html_tpl = settings.email_test_html or ""
subject = render_email_template(subject_tpl, **context)
text_body = render_email_template(body_tpl, **context)
if html_tpl.strip():
html_body = render_email_template(html_tpl, **context)
else:
html_body = _text_to_html(text_body)
return subject, text_body, html_body
+78
View File
@@ -0,0 +1,78 @@
"""邮箱验证令牌创建与校验。"""
from __future__ import annotations
import secrets
from datetime import datetime, timedelta
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession
from models.models import EmailVerificationToken, User
def _token_expires_at(hours: int) -> datetime:
return datetime.utcnow() + timedelta(hours=max(1, min(168, int(hours or 24))))
async def invalidate_user_tokens(db: AsyncSession, user_id: int) -> None:
await db.execute(
delete(EmailVerificationToken).where(EmailVerificationToken.user_id == user_id)
)
async def create_verification_token(
db: AsyncSession,
user: User,
expire_hours: int = 24,
) -> str:
await invalidate_user_tokens(db, user.id)
token = secrets.token_urlsafe(32)
db.add(
EmailVerificationToken(
user_id=user.id,
token=token,
expires_at=_token_expires_at(expire_hours),
)
)
await db.flush()
return token
async def verify_email_token(db: AsyncSession, token: str) -> User | None:
value = (token or "").strip()
if not value:
return None
result = await db.execute(
select(EmailVerificationToken, User)
.join(User, User.id == EmailVerificationToken.user_id)
.where(EmailVerificationToken.token == value)
)
row = result.first()
if not row:
return None
record, user = row
if record.expires_at < datetime.utcnow():
await db.delete(record)
await db.flush()
return None
user.email_verified = True
user.email_verified_at = datetime.utcnow()
await db.delete(record)
await db.flush()
return user
def mask_email(email: str) -> str:
value = (email or "").strip()
if "@" not in value:
return value
local, domain = value.split("@", 1)
if len(local) <= 2:
masked_local = local[0] + "*"
else:
masked_local = local[0] + "*" * (len(local) - 2) + local[-1]
return f"{masked_local}@{domain}"
+24
View File
@@ -0,0 +1,24 @@
import os
from datetime import datetime, timedelta
from typing import Any, Optional
from jose import JWTError, jwt
SECRET_KEY = os.getenv("KEFU_SECRET_KEY", "kefu-dev-secret-change-in-production")
ALGORITHM = "HS256"
ACCESS_TOKEN_EXPIRE_MINUTES = int(os.getenv("KEFU_TOKEN_EXPIRE_MINUTES", str(60 * 24)))
def create_access_token(subject: str, extra: Optional[dict[str, Any]] = None) -> str:
expire = datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
payload = {"sub": subject, "exp": expire}
if extra:
payload.update(extra)
return jwt.encode(payload, SECRET_KEY, algorithm=ALGORITHM)
def decode_access_token(token: str) -> Optional[dict[str, Any]]:
try:
return jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
except JWTError:
return None
+64
View File
@@ -0,0 +1,64 @@
"""密码重置令牌创建与校验。"""
from __future__ import annotations
import secrets
from datetime import datetime, timedelta
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession
from models.models import PasswordResetToken, User
def _token_expires_at(hours: int) -> datetime:
return datetime.utcnow() + timedelta(hours=max(1, min(168, int(hours or 24))))
async def invalidate_user_reset_tokens(db: AsyncSession, user_id: int) -> None:
await db.execute(
delete(PasswordResetToken).where(PasswordResetToken.user_id == user_id)
)
async def create_password_reset_token(
db: AsyncSession,
user: User,
expire_hours: int = 24,
) -> str:
await invalidate_user_reset_tokens(db, user.id)
token = secrets.token_urlsafe(32)
db.add(
PasswordResetToken(
user_id=user.id,
token=token,
expires_at=_token_expires_at(expire_hours),
)
)
await db.flush()
return token
async def verify_password_reset_token(db: AsyncSession, token: str) -> User | None:
value = (token or "").strip()
if not value:
return None
result = await db.execute(
select(PasswordResetToken, User)
.join(User, User.id == PasswordResetToken.user_id)
.where(PasswordResetToken.token == value)
)
row = result.first()
if not row:
return None
record, user = row
if record.expires_at < datetime.utcnow():
await db.delete(record)
await db.flush()
return None
await db.delete(record)
await db.flush()
return user
+11
View File
@@ -0,0 +1,11 @@
from passlib.context import CryptContext
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
def hash_password(password: str) -> str:
return pwd_context.hash(password)
def verify_password(plain: str, hashed: str) -> bool:
return pwd_context.verify(plain, hashed)
+31
View File
@@ -0,0 +1,31 @@
from typing import Iterable
ROLE_ADMIN = "admin"
ROLE_OPERATOR = "operator"
ROLE_VIEWER = "viewer"
ALL_ROLES = (ROLE_ADMIN, ROLE_OPERATOR, ROLE_VIEWER)
ROLE_LABELS = {
ROLE_ADMIN: "管理员",
ROLE_OPERATOR: "运营",
ROLE_VIEWER: "只读",
}
def is_admin(role: str) -> bool:
return role == ROLE_ADMIN
def can_write(role: str) -> bool:
return role in (ROLE_ADMIN, ROLE_OPERATOR)
def can_manage_users(role: str) -> bool:
return role == ROLE_ADMIN
def ensure_role(role: str) -> str:
if role not in ALL_ROLES:
raise ValueError(f"无效角色: {role}")
return role
+511
View File
@@ -0,0 +1,511 @@
import logging
from datetime import datetime
from fastapi import APIRouter, Depends, HTTPException, Query, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models.database import get_db
from models.models import User
from .account_limits import (
UNLIMITED_ACCOUNTS,
count_user_account_breakdown,
normalize_max_accounts,
)
from .account_quota import default_stop_worker, sync_user_account_quota
from .dependencies import get_current_user, require_user_manager
from .email_service import (
build_password_reset_link,
build_verification_link,
send_password_reset_email,
send_verification_email,
)
from .email_verification import create_verification_token, mask_email, verify_email_token
from .password_reset import create_password_reset_token, verify_password_reset_token
from .jwt_utils import create_access_token
from .passwords import hash_password, verify_password
from .roles import ALL_ROLES, ROLE_LABELS, ROLE_OPERATOR, ensure_role, is_admin
from .schemas import (
LoginRequest,
MessageResponse,
ForgotPasswordRequest,
ForgotPasswordResponse,
RegisterRequest,
RegisterResponse,
ResendVerificationRequest,
ResetPasswordRequest,
RolesResponse,
RoleInfo,
TokenResponse,
UserCreate,
UserResponse,
UserUpdate,
VerifyEmailRequest,
)
from .system_settings import SystemSettingsData, load_settings
logger = logging.getLogger("auth.router")
router = APIRouter(prefix="/api/auth", tags=["auth"])
async def _build_user_response(db: AsyncSession, user: User, with_count: bool = False) -> UserResponse:
payload = UserResponse.model_validate(user)
if with_count:
breakdown = await count_user_account_breakdown(db, user.id)
payload.account_count = breakdown["total"]
payload.active_account_count = breakdown["active"]
payload.disabled_account_count = breakdown["disabled"]
return payload
async def _ensure_email_available(
db: AsyncSession,
email: str | None,
exclude_user_id: int | None = None,
) -> str | None:
value = (str(email).strip().lower() if email else "") or None
if not value:
return None
stmt = select(User).where(User.email == value)
if exclude_user_id:
stmt = stmt.where(User.id != exclude_user_id)
exists = await db.execute(stmt)
if exists.scalar_one_or_none():
raise HTTPException(status_code=400, detail="邮箱已被其他用户使用")
return value
def _apply_email_verified(user: User, verified: bool | None) -> None:
if verified is None:
return
user.email_verified = verified
if verified:
if not user.email_verified_at:
user.email_verified_at = datetime.utcnow()
else:
user.email_verified_at = None
def _email_not_verified_detail(user: User) -> dict:
return {
"code": "email_not_verified",
"message": "邮箱尚未验证,请先完成邮箱验证后再登录",
"email": mask_email(user.email or ""),
}
def _email_not_bound_detail() -> dict:
return {
"code": "email_not_bound",
"message": "该账号未绑定邮箱,请联系管理员绑定邮箱后再登录",
}
def _require_email_access(user: User, settings: SystemSettingsData) -> None:
if is_admin(user.role):
return
if settings.email_binding_required and not user.email:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=_email_not_bound_detail(),
)
if not settings.email_verification_required:
return
if user.email_verified:
return
if user.email:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=_email_not_verified_detail(user),
)
async def _issue_token(user: User) -> TokenResponse:
token = create_access_token(str(user.id), {"role": user.role, "username": user.username})
return TokenResponse(access_token=token)
async def _send_user_verification(
db: AsyncSession,
user: User,
settings: SystemSettingsData,
) -> tuple[bool, str | None]:
if not user.email:
raise HTTPException(status_code=400, detail="该账号未绑定邮箱")
token = await create_verification_token(
db, user, expire_hours=settings.email_verify_token_hours
)
await db.commit()
link = build_verification_link(token, settings)
if not settings.smtp_configured():
logger.warning(
"SMTP 未配置,验证链接: user=%s email=%s link=%s",
user.username,
user.email,
link,
)
dev_url = link if settings.debug_show_verify_link else None
return False, dev_url
try:
await send_verification_email(user.email, user.username, token, settings)
return True, None
except Exception as exc:
logger.exception("Send verification email failed for user=%s", user.username)
dev_url = link if settings.debug_show_verify_link else None
if dev_url:
return False, dev_url
raise HTTPException(status_code=400, detail=f"邮件发送失败:{exc}") from exc
@router.post("/login", response_model=TokenResponse)
async def login(body: LoginRequest, db: AsyncSession = Depends(get_db)):
settings = await load_settings(db)
result = await db.execute(select(User).where(User.username == body.username.strip()))
user = result.scalar_one_or_none()
if not user or not verify_password(body.password, user.password_hash):
raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="用户名或密码错误")
if not user.is_active:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="账号已禁用")
_require_email_access(user, settings)
return await _issue_token(user)
@router.post("/register", response_model=RegisterResponse)
async def register(body: RegisterRequest, db: AsyncSession = Depends(get_db)):
settings = await load_settings(db)
if not settings.registration_enabled:
raise HTTPException(status_code=403, detail="当前未开放用户注册")
username = body.username.strip()
email = str(body.email).strip().lower()
exists = await db.execute(select(User).where(User.username == username))
if exists.scalar_one_or_none():
raise HTTPException(status_code=400, detail="用户名已存在")
exists = await db.execute(select(User).where(User.email == email))
if exists.scalar_one_or_none():
raise HTTPException(status_code=400, detail="邮箱已被注册")
verification_required = settings.email_verification_required
user = User(
username=username,
email=email,
password_hash=hash_password(body.password),
display_name=body.display_name or username,
role=ROLE_OPERATOR,
is_active=True,
email_verified=not verification_required,
email_verified_at=datetime.utcnow() if not verification_required else None,
max_accounts=max(0, int(settings.default_register_max_accounts or 3)),
)
db.add(user)
await db.flush()
if not verification_required:
await db.commit()
await db.refresh(user)
return RegisterResponse(
message="注册成功,可直接登录",
email=mask_email(user.email),
verification_sent=False,
verification_required=False,
)
verification_sent, dev_url = await _send_user_verification(db, user, settings)
await db.refresh(user)
message = "注册成功,验证邮件已发送,请查收并完成验证后再登录"
if not verification_sent:
message = (
"注册成功。验证邮件未能发出,请使用下方链接或联系管理员检查 SMTP 配置"
if dev_url
else "注册成功。邮件服务未配置或发送失败,请联系管理员"
)
return RegisterResponse(
message=message,
email=mask_email(user.email),
verification_sent=verification_sent,
verification_required=True,
dev_verify_url=dev_url,
)
@router.post("/verify-email", response_model=MessageResponse)
async def verify_email(body: VerifyEmailRequest, db: AsyncSession = Depends(get_db)):
user = await verify_email_token(db, body.token.strip())
if not user:
raise HTTPException(status_code=400, detail="验证链接无效或已过期")
await db.commit()
return MessageResponse(message="邮箱验证成功,现在可以登录了")
@router.get("/verify-email", response_model=MessageResponse)
async def verify_email_get(token: str = Query(..., min_length=8), db: AsyncSession = Depends(get_db)):
user = await verify_email_token(db, token.strip())
if not user:
raise HTTPException(status_code=400, detail="验证链接无效或已过期")
await db.commit()
return MessageResponse(message="邮箱验证成功,现在可以登录了")
@router.post("/forgot-password", response_model=ForgotPasswordResponse)
async def forgot_password(body: ForgotPasswordRequest, db: AsyncSession = Depends(get_db)):
settings = await load_settings(db)
if not settings.smtp_configured():
raise HTTPException(
status_code=400,
detail="邮件服务未配置,请联系管理员重置密码",
)
email = str(body.email).strip().lower() if body.email else None
username = body.username.strip() if body.username else None
if not email and not username:
raise HTTPException(status_code=400, detail="请提供邮箱或用户名")
user = None
if email:
result = await db.execute(select(User).where(User.email == email))
user = result.scalar_one_or_none()
if not user and username:
result = await db.execute(select(User).where(User.username == username))
user = result.scalar_one_or_none()
generic = ForgotPasswordResponse(
message="若账号存在且已绑定邮箱,重置邮件将发送到注册邮箱",
email=mask_email(user.email if user and user.email else (email or "")),
reset_sent=True,
)
if not user or not user.email or not user.is_active:
return generic
token = await create_password_reset_token(
db, user, expire_hours=settings.email_verify_token_hours
)
await db.commit()
link = build_password_reset_link(token, settings)
try:
await send_password_reset_email(user.email, user.username, token, settings)
except Exception as exc:
logger.exception("Send password reset email failed for user=%s", user.username)
dev_url = link if settings.debug_show_verify_link else None
if dev_url:
return ForgotPasswordResponse(
message="邮件发送失败,请使用下方开发重置链接",
email=mask_email(user.email),
reset_sent=False,
dev_reset_url=dev_url,
)
raise HTTPException(status_code=400, detail=f"邮件发送失败:{exc}") from exc
return ForgotPasswordResponse(
message="重置邮件已发送,请查收并按邮件说明设置新密码",
email=mask_email(user.email),
reset_sent=True,
dev_reset_url=None,
)
@router.post("/reset-password", response_model=MessageResponse)
async def reset_password(body: ResetPasswordRequest, db: AsyncSession = Depends(get_db)):
user = await verify_password_reset_token(db, body.token.strip())
if not user:
raise HTTPException(status_code=400, detail="重置链接无效或已过期")
if not user.is_active:
raise HTTPException(status_code=400, detail="账号已禁用,请联系管理员")
user.password_hash = hash_password(body.password)
await db.commit()
return MessageResponse(message="密码已重置,请使用新密码登录")
@router.post("/resend-verification", response_model=RegisterResponse)
async def resend_verification(body: ResendVerificationRequest, db: AsyncSession = Depends(get_db)):
settings = await load_settings(db)
if not settings.email_verification_required:
raise HTTPException(status_code=400, detail="当前系统未开启邮箱验证")
email = str(body.email).strip().lower() if body.email else None
username = body.username.strip() if body.username else None
if not email and not username:
raise HTTPException(status_code=400, detail="请提供邮箱或用户名")
user = None
if email:
result = await db.execute(select(User).where(User.email == email))
user = result.scalar_one_or_none()
if not user and username:
result = await db.execute(select(User).where(User.username == username))
user = result.scalar_one_or_none()
if not user:
return RegisterResponse(
message="若账号存在且未验证,验证邮件将发送到注册邮箱",
email=mask_email(email or ""),
verification_sent=True,
verification_required=True,
)
if user.email_verified:
raise HTTPException(status_code=400, detail="该账号邮箱已验证,可直接登录")
verification_sent, dev_url = await _send_user_verification(db, user, settings)
message = (
"验证邮件已重新发送,请查收"
if verification_sent
else "邮件未能发出,请使用下方验证链接或联系管理员检查 SMTP 配置"
)
return RegisterResponse(
message=message,
email=mask_email(user.email or ""),
verification_sent=verification_sent,
verification_required=True,
dev_verify_url=dev_url,
)
@router.get("/me", response_model=UserResponse)
async def get_me(user: User = Depends(get_current_user), db: AsyncSession = Depends(get_db)):
return await _build_user_response(db, user, with_count=True)
@router.get("/roles", response_model=RolesResponse)
async def list_roles(_: User = Depends(get_current_user)):
return RolesResponse(
roles=[RoleInfo(value=r, label=ROLE_LABELS.get(r, r)) for r in ALL_ROLES]
)
users_router = APIRouter(prefix="/api/users", tags=["users"])
@users_router.get("", response_model=list[UserResponse])
async def list_users(
db: AsyncSession = Depends(get_db),
_: User = Depends(require_user_manager),
):
result = await db.execute(select(User).order_by(User.id.asc()))
users = result.scalars().all()
responses = []
for user in users:
responses.append(await _build_user_response(db, user, with_count=True))
return responses
@users_router.post("", response_model=UserResponse)
async def create_user(
body: UserCreate,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_user_manager),
):
settings = await load_settings(db)
exists = await db.execute(select(User).where(User.username == body.username))
if exists.scalar_one_or_none():
raise HTTPException(status_code=400, detail="用户名已存在")
try:
role = ensure_role(body.role)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
email = await _ensure_email_available(db, str(body.email) if body.email else None)
if settings.email_binding_required and not is_admin(role) and not email:
raise HTTPException(status_code=400, detail="系统已开启「登录必须绑定邮箱」,请填写邮箱")
if email:
email_verified = body.email_verified
else:
email_verified = not settings.email_binding_required
user = User(
username=body.username.strip(),
password_hash=hash_password(body.password),
display_name=body.display_name or body.username,
role=role,
is_active=True,
email=email,
email_verified=email_verified,
email_verified_at=datetime.utcnow() if email and email_verified else None,
max_accounts=normalize_max_accounts(body.max_accounts, role),
)
db.add(user)
await db.commit()
await db.refresh(user)
return await _build_user_response(db, user, with_count=True)
@users_router.put("/{user_id}", response_model=UserResponse)
async def update_user(
user_id: int,
body: UserUpdate,
db: AsyncSession = Depends(get_db),
current: User = Depends(require_user_manager),
):
result = await db.execute(select(User).where(User.id == user_id))
user = result.scalar_one_or_none()
if not user:
raise HTTPException(status_code=404, detail="用户不存在")
settings = await load_settings(db)
if user.id == current.id and body.is_active is False:
raise HTTPException(status_code=400, detail="不能禁用当前登录账号")
if body.display_name is not None:
user.display_name = body.display_name
if body.role is not None:
prev_role = user.role
try:
user.role = ensure_role(body.role)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
if is_admin(user.role):
user.max_accounts = UNLIMITED_ACCOUNTS
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
elif is_admin(prev_role) and not is_admin(user.role):
settings = await load_settings(db)
user.max_accounts = max(0, int(settings.default_register_max_accounts or 3))
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
if body.is_active is not None:
user.is_active = body.is_active
if body.password:
user.password_hash = hash_password(body.password)
updates = body.model_dump(exclude_unset=True)
if "max_accounts" in updates and not is_admin(user.role):
user.max_accounts = normalize_max_accounts(updates["max_accounts"], user.role)
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
if "email" in updates:
raw_email = updates.get("email")
user.email = await _ensure_email_available(
db,
str(raw_email) if raw_email else None,
exclude_user_id=user.id,
)
if not user.email:
user.email_verified = not (
settings.email_binding_required and not is_admin(user.role)
)
user.email_verified_at = None
if "email_verified" in updates:
if not user.email:
raise HTTPException(status_code=400, detail="未绑定邮箱时无法设置验证状态")
_apply_email_verified(user, updates["email_verified"])
if settings.email_binding_required and not is_admin(user.role) and not user.email:
raise HTTPException(status_code=400, detail="系统已开启「登录必须绑定邮箱」,该用户需绑定邮箱")
await db.commit()
await db.refresh(user)
return await _build_user_response(db, user, with_count=True)
@users_router.delete("/{user_id}")
async def delete_user(
user_id: int,
db: AsyncSession = Depends(get_db),
current: User = Depends(require_user_manager),
):
if user_id == current.id:
raise HTTPException(status_code=400, detail="不能删除当前登录账号")
result = await db.execute(select(User).where(User.id == user_id))
user = result.scalar_one_or_none()
if not user:
raise HTTPException(status_code=404, detail="用户不存在")
await db.delete(user)
await db.commit()
return {"message": "用户已删除"}
+108
View File
@@ -0,0 +1,108 @@
from datetime import datetime
from typing import Any, Optional
from pydantic import BaseModel, EmailStr, Field
from .roles import ALL_ROLES
class LoginRequest(BaseModel):
username: str
password: str
class RegisterRequest(BaseModel):
username: str = Field(min_length=2, max_length=50)
email: EmailStr
password: str = Field(min_length=6, max_length=128)
display_name: Optional[str] = None
class VerifyEmailRequest(BaseModel):
token: str = Field(min_length=8, max_length=128)
class ResendVerificationRequest(BaseModel):
email: Optional[EmailStr] = None
username: Optional[str] = None
class ForgotPasswordRequest(BaseModel):
email: Optional[EmailStr] = None
username: Optional[str] = None
class ResetPasswordRequest(BaseModel):
token: str = Field(min_length=8, max_length=128)
password: str = Field(min_length=6, max_length=128)
class ForgotPasswordResponse(BaseModel):
message: str
email: str
reset_sent: bool
dev_reset_url: Optional[str] = None
class TokenResponse(BaseModel):
access_token: str
token_type: str = "bearer"
class RegisterResponse(BaseModel):
message: str
email: str
verification_sent: bool
verification_required: bool = True
dev_verify_url: Optional[str] = None
class MessageResponse(BaseModel):
message: str
class UserResponse(BaseModel):
id: int
username: str
email: Optional[str] = None
display_name: Optional[str] = None
role: str
is_active: bool
email_verified: bool = False
max_accounts: int = 3
account_count: Optional[int] = None
active_account_count: Optional[int] = None
disabled_account_count: Optional[int] = None
created_at: datetime
class Config:
from_attributes = True
class UserCreate(BaseModel):
username: str = Field(min_length=2, max_length=50)
password: str = Field(min_length=6, max_length=128)
display_name: Optional[str] = None
role: str = "operator"
email: Optional[EmailStr] = None
email_verified: bool = True
max_accounts: int = Field(default=3, ge=0, le=999)
class UserUpdate(BaseModel):
display_name: Optional[str] = None
role: Optional[str] = None
is_active: Optional[bool] = None
password: Optional[str] = Field(default=None, min_length=6, max_length=128)
email: Optional[EmailStr] = None
email_verified: Optional[bool] = None
max_accounts: Optional[int] = Field(default=None, ge=0, le=999)
class RoleInfo(BaseModel):
value: str
label: str
class RolesResponse(BaseModel):
roles: list[RoleInfo]
+109
View File
@@ -0,0 +1,109 @@
from typing import Optional
from fastapi import HTTPException, status
from sqlalchemy import or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from models.models import Account, AutoReplyRule, MessageLog, ReceivedMessageLog, SystemLog, User
from .roles import is_admin
async def get_owned_account(
db: AsyncSession,
user: User,
account_id: int,
*,
write: bool = False,
) -> Account:
result = await db.execute(select(Account).where(Account.id == account_id))
account = result.scalar_one_or_none()
if not account:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="账号不存在")
if is_admin(user.role):
return account
if account.owner_id != user.id:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="无权访问该账号")
if write and user.role == "viewer":
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="只读用户无法修改")
return account
def accounts_for_user(user: User):
stmt = select(Account)
if not is_admin(user.role):
stmt = stmt.where(Account.owner_id == user.id)
return stmt
async def owned_account_ids(db: AsyncSession, user: User) -> Optional[set[int]]:
if is_admin(user.role):
return None
result = await db.execute(select(Account.id).where(Account.owner_id == user.id))
return {row[0] for row in result.all()}
def logs_for_user(user: User, account_id: Optional[int] = None):
stmt = select(MessageLog)
if account_id is not None:
stmt = stmt.where(MessageLog.account_id == account_id)
if not is_admin(user.role):
owned = select(Account.id).where(Account.owner_id == user.id)
stmt = stmt.where(MessageLog.account_id.in_(owned))
return stmt
def received_logs_for_user(user: User, account_id: Optional[int] = None):
stmt = select(ReceivedMessageLog)
if account_id is not None:
stmt = stmt.where(ReceivedMessageLog.account_id == account_id)
if not is_admin(user.role):
owned = select(Account.id).where(Account.owner_id == user.id)
stmt = stmt.where(ReceivedMessageLog.account_id.in_(owned))
return stmt
def rules_for_user(user: User, account_id: Optional[int] = None):
stmt = select(AutoReplyRule)
if account_id is not None:
stmt = stmt.where(AutoReplyRule.account_id == account_id)
if is_admin(user.role):
return stmt
owned = select(Account.id).where(Account.owner_id == user.id)
return stmt.where(AutoReplyRule.account_id.in_(owned))
async def get_accessible_rule(db: AsyncSession, user: User, rule_id: int, *, write: bool = False) -> AutoReplyRule:
result = await db.execute(select(AutoReplyRule).where(AutoReplyRule.id == rule_id))
rule = result.scalar_one_or_none()
if not rule:
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="规则不存在")
if is_admin(user.role):
if write and user.role == "viewer":
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="只读用户无法修改")
return rule
if rule.account_id is None:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="无权访问全局规则")
account = await get_owned_account(db, user, rule.account_id, write=write)
if rule.owner_id and rule.owner_id != user.id and account.owner_id != user.id:
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="无权访问该规则")
if write and user.role == "viewer":
raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="只读用户无法修改")
return rule
def system_logs_for_user(user: User, account_id: Optional[int] = None):
stmt = select(SystemLog)
if account_id is not None:
stmt = stmt.where(SystemLog.account_id == account_id)
if not is_admin(user.role):
owned = select(Account.id).where(Account.owner_id == user.id)
stmt = stmt.where(
or_(
SystemLog.account_id.in_(owned),
SystemLog.account_id.is_(None),
)
)
return stmt
+346
View File
@@ -0,0 +1,346 @@
from pydantic import BaseModel, EmailStr, Field
from dataclasses import asdict
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from models.database import get_db
from models.db_config import (
PASSWORD_PLACEHOLDER as DB_PASSWORD_PLACEHOLDER,
database_config_to_response,
persist_database_config,
test_database_connection,
)
from models.db_transfer import inspect_sqlite_source, migrate_sqlite_to_target
from models.models import User
from .dependencies import require_admin
from .email_service import send_test_email
from .system_settings import (
PASSWORD_PLACEHOLDER,
SystemSettingsData,
load_settings,
save_settings,
settings_to_admin_response,
settings_to_payment_response,
settings_to_public,
)
router = APIRouter(prefix="/api/settings", tags=["settings"])
class PublicSettingsResponse(BaseModel):
registration_enabled: bool
email_verification_required: bool
email_binding_required: bool
class SystemSettingsResponse(BaseModel):
registration_enabled: bool
email_verification_required: bool
email_binding_required: bool = False
email_verify_token_hours: int
auto_reply_delay_seconds: int = 0
auto_reply_cooldown_seconds: int = 60
app_url: str
default_register_max_accounts: int = 3
smtp_host: str
smtp_port: int
smtp_user: str
smtp_password: str = ""
smtp_password_configured: bool = False
smtp_from: str
smtp_use_tls: bool
smtp_use_ssl: bool = False
debug_show_verify_link: bool
email_verify_subject: str
email_verify_body: str
email_verify_html: str
email_test_subject: str
email_test_body: str
email_test_html: str
class SystemSettingsUpdate(BaseModel):
registration_enabled: bool | None = None
email_verification_required: bool | None = None
email_binding_required: bool | None = None
email_verify_token_hours: int | None = Field(default=None, ge=1, le=168)
auto_reply_delay_seconds: int | None = Field(default=None, ge=0, le=86400)
auto_reply_cooldown_seconds: int | None = Field(default=None, ge=0, le=86400)
app_url: str | None = None
default_register_max_accounts: int | None = Field(default=None, ge=0, le=999)
smtp_host: str | None = None
smtp_port: int | None = Field(default=None, ge=1, le=65535)
smtp_user: str | None = None
smtp_password: str | None = None
smtp_from: str | None = None
smtp_use_tls: bool | None = None
smtp_use_ssl: bool | None = None
debug_show_verify_link: bool | None = None
email_verify_subject: str | None = None
email_verify_body: str | None = None
email_verify_html: str | None = None
email_test_subject: str | None = None
email_test_body: str | None = None
email_test_html: str | None = None
class TestEmailRequest(BaseModel):
to_email: EmailStr
smtp_host: str | None = None
smtp_port: int | None = Field(default=None, ge=1, le=65535)
smtp_user: str | None = None
smtp_password: str | None = None
smtp_from: str | None = None
smtp_use_tls: bool | None = None
smtp_use_ssl: bool | None = None
class MessageResponse(BaseModel):
message: str
class DatabaseSettingsResponse(BaseModel):
db_type: str
db_host: str = ""
db_port: int = 0
db_user: str = ""
db_password: str = ""
db_password_configured: bool = False
db_name: str = ""
db_path: str = ""
database_url_display: str = ""
supported_types: list[str] = []
class DatabaseSettingsUpdate(BaseModel):
db_type: str
db_host: str | None = None
db_port: int | None = Field(default=None, ge=1, le=65535)
db_user: str | None = None
db_password: str | None = None
db_name: str | None = None
db_path: str | None = None
class DatabaseTestRequest(BaseModel):
db_type: str
db_host: str | None = None
db_port: int | None = Field(default=None, ge=1, le=65535)
db_user: str | None = None
db_password: str | None = None
db_name: str | None = None
db_path: str | None = None
class DatabaseMigratePreviewResponse(BaseModel):
source_path: str
exists: bool
tables: dict[str, int]
total_rows: int
class DatabaseMigrateRequest(DatabaseTestRequest):
source_db_path: str | None = None
clear_target: bool = False
class DatabaseMigrateResponse(BaseModel):
message: str
source_path: str
target_type: str
target_url: str
tables: dict[str, int]
total_rows: int
class PaymentSettingsResponse(BaseModel):
app_url: str
payment_enabled: bool = False
payment_demo_mode: bool = True
wechat_pay_enabled: bool = False
alipay_pay_enabled: bool = False
account_slot_unit_price: float = 9.9
account_slot_purchase_min: int = 1
account_slot_purchase_max: int = 20
wechat_app_id: str = ""
wechat_mch_id: str = ""
wechat_api_v3_key: str = ""
wechat_api_v3_key_configured: bool = False
wechat_cert_serial: str = ""
wechat_private_key: str = ""
wechat_private_key_configured: bool = False
wechat_pay_configured: bool = False
alipay_app_id: str = ""
alipay_private_key: str = ""
alipay_private_key_configured: bool = False
alipay_public_key: str = ""
alipay_sandbox: bool = False
alipay_configured: bool = False
class PaymentSettingsUpdate(BaseModel):
payment_enabled: bool | None = None
payment_demo_mode: bool | None = None
wechat_pay_enabled: bool | None = None
alipay_pay_enabled: bool | None = None
account_slot_unit_price: float | None = Field(default=None, ge=0.01, le=99999)
account_slot_purchase_min: int | None = Field(default=None, ge=1, le=100)
account_slot_purchase_max: int | None = Field(default=None, ge=1, le=100)
wechat_app_id: str | None = None
wechat_mch_id: str | None = None
wechat_api_v3_key: str | None = None
wechat_cert_serial: str | None = None
wechat_private_key: str | None = None
alipay_app_id: str | None = None
alipay_private_key: str | None = None
alipay_public_key: str | None = None
alipay_sandbox: bool | None = None
@router.get("/public", response_model=PublicSettingsResponse)
async def get_public_settings(db: AsyncSession = Depends(get_db)):
data = await load_settings(db)
return PublicSettingsResponse(**settings_to_public(data))
@router.get("", response_model=SystemSettingsResponse)
async def get_system_settings(
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
data = await load_settings(db)
return SystemSettingsResponse(**settings_to_admin_response(data))
@router.put("", response_model=SystemSettingsResponse)
async def update_system_settings(
body: SystemSettingsUpdate,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
updates = body.model_dump(exclude_unset=True)
if "app_url" in updates and updates["app_url"]:
updates["app_url"] = updates["app_url"].strip().rstrip("/")
data = await save_settings(db, updates)
return SystemSettingsResponse(**settings_to_admin_response(data))
@router.get("/payment", response_model=PaymentSettingsResponse)
async def get_payment_settings(
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
data = await load_settings(db)
return PaymentSettingsResponse(**settings_to_payment_response(data))
@router.put("/payment", response_model=PaymentSettingsResponse)
async def update_payment_settings(
body: PaymentSettingsUpdate,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
updates = body.model_dump(exclude_unset=True)
data = await save_settings(db, updates)
return PaymentSettingsResponse(**settings_to_payment_response(data))
@router.post("/test-email", response_model=MessageResponse)
async def test_smtp_email(
body: TestEmailRequest,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
data = await load_settings(db)
overrides = body.model_dump(exclude_unset=True, exclude={"to_email"})
if overrides:
merged = asdict(data)
for key, value in overrides.items():
if value is None:
continue
if key == "smtp_password":
pwd = str(value).strip()
if not pwd or pwd == PASSWORD_PLACEHOLDER:
continue
merged[key] = value
data = SystemSettingsData(**merged)
if not data.smtp_configured():
raise HTTPException(status_code=400, detail="请先完整配置 SMTP 服务器与发件人")
try:
await send_test_email(str(body.to_email), data)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return MessageResponse(message=f"测试邮件已发送至 {body.to_email}")
@router.get("/database", response_model=DatabaseSettingsResponse)
async def get_database_settings(_: User = Depends(require_admin)):
return DatabaseSettingsResponse(**database_config_to_response())
@router.put("/database", response_model=MessageResponse)
async def update_database_settings(
body: DatabaseSettingsUpdate,
_: User = Depends(require_admin),
):
payload = body.model_dump(exclude_unset=True)
if payload.get("db_password") in (None, "", DB_PASSWORD_PLACEHOLDER):
payload.pop("db_password", None)
try:
await test_database_connection(payload)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"连接测试失败: {exc}") from exc
try:
persist_database_config(payload)
except Exception as exc:
raise HTTPException(status_code=500, detail=f"写入配置失败: {exc}") from exc
return MessageResponse(message="数据库配置已保存至 .env,请重启后端服务后生效")
@router.post("/database/test", response_model=MessageResponse)
async def test_database_settings(
body: DatabaseTestRequest,
_: User = Depends(require_admin),
):
payload = body.model_dump(exclude_unset=True)
if payload.get("db_password") in (None, "", DB_PASSWORD_PLACEHOLDER):
payload.pop("db_password", None)
try:
await test_database_connection(payload)
except Exception as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
return MessageResponse(message="数据库连接测试成功")
@router.get("/database/migrate/preview", response_model=DatabaseMigratePreviewResponse)
async def preview_database_migration(
source_db_path: str | None = None,
_: User = Depends(require_admin),
):
return DatabaseMigratePreviewResponse(**await inspect_sqlite_source(source_db_path))
@router.post("/database/migrate", response_model=DatabaseMigrateResponse)
async def migrate_database_data(
body: DatabaseMigrateRequest,
_: User = Depends(require_admin),
):
payload = body.model_dump(exclude_unset=True)
clear_target = bool(payload.pop("clear_target", False))
if payload.get("db_password") in (None, "", DB_PASSWORD_PLACEHOLDER):
payload.pop("db_password", None)
try:
await test_database_connection(payload)
except Exception as exc:
raise HTTPException(status_code=400, detail=f"目标库连接失败: {exc}") from exc
try:
result = await migrate_sqlite_to_target(payload, clear_target=clear_target)
except FileNotFoundError as exc:
raise HTTPException(status_code=404, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except Exception as exc:
raise HTTPException(status_code=500, detail=f"迁移失败: {exc}") from exc
return DatabaseMigrateResponse(**result)
+366
View File
@@ -0,0 +1,366 @@
"""系统功能配置(数据库存储 + 内存缓存)。"""
from __future__ import annotations
import json
from dataclasses import asdict, dataclass, fields
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models.models import AppConfig
CONFIG_ROW_ID = 1
PASSWORD_PLACEHOLDER = "******"
APP_NAME = "抖音回复助手"
DEFAULT_EMAIL_VERIFY_SUBJECT = f"【{APP_NAME}】请验证您的邮箱"
DEFAULT_EMAIL_VERIFY_BODY = (
"您好 {username},\n\n"
"感谢注册{app_name}。请点击以下链接完成邮箱验证:\n"
"{link}\n\n"
"如非本人操作,请忽略此邮件。\n"
)
DEFAULT_EMAIL_VERIFY_HTML = (
"<p>您好 <strong>{username}</strong>,</p>"
"<p>感谢注册{app_name}。请点击下方按钮完成邮箱验证:</p>"
'<p><a href="{link}" style="display:inline-block;padding:10px 18px;'
'background:#aa3bff;color:#fff;text-decoration:none;border-radius:6px;">'
"验证邮箱</a></p>"
'<p>或复制链接到浏览器:<br><a href="{link}">{link}</a></p>'
'<p style="color:#888;font-size:12px;">如非本人操作,请忽略此邮件。</p>'
)
DEFAULT_EMAIL_TEST_SUBJECT = f"【{APP_NAME}】SMTP 测试邮件"
DEFAULT_EMAIL_TEST_BODY = "这是一封 SMTP 配置测试邮件。若您收到此邮件,说明邮件服务已配置正确。"
DEFAULT_EMAIL_TEST_HTML = (
"<p>这是一封 <strong>SMTP 配置测试</strong> 邮件。</p>"
"<p>若您收到此邮件,说明邮件服务已配置正确。</p>"
)
DEFAULT_PASSWORD_RESET_SUBJECT = f"【{APP_NAME}】重置您的登录密码"
DEFAULT_PASSWORD_RESET_BODY = (
"您好 {username},\n\n"
"我们收到了重置 {app_name} 账号密码的请求。请点击以下链接设置新密码:\n"
"{link}\n\n"
"链接有效期 {expire_hours} 小时。如非本人操作,请忽略此邮件。\n"
)
DEFAULT_PASSWORD_RESET_HTML = (
"<p>您好 <strong>{username}</strong>,</p>"
"<p>我们收到了重置 {app_name} 账号密码的请求。请点击下方按钮设置新密码:</p>"
'<p><a href="{link}" style="display:inline-block;padding:10px 18px;'
'background:#aa3bff;color:#fff;text-decoration:none;border-radius:6px;">'
"重置密码</a></p>"
'<p>或复制链接到浏览器:<br><a href="{link}">{link}</a></p>'
'<p style="color:#888;font-size:12px;">链接有效期 {expire_hours} 小时。如非本人操作,请忽略此邮件。</p>'
)
@dataclass
class SystemSettingsData:
registration_enabled: bool = True
email_verification_required: bool = True
email_binding_required: bool = False
email_verify_token_hours: int = 24
auto_reply_delay_seconds: int = 0
auto_reply_cooldown_seconds: int = 60
app_url: str = "http://localhost:8800"
smtp_host: str = ""
smtp_port: int = 587
smtp_user: str = ""
smtp_password: str = ""
smtp_from: str = ""
smtp_use_tls: bool = True
smtp_use_ssl: bool = False
debug_show_verify_link: bool = False
default_register_max_accounts: int = 3
payment_enabled: bool = False
payment_demo_mode: bool = True
wechat_pay_enabled: bool = False
alipay_pay_enabled: bool = False
account_slot_unit_price: float = 9.9
account_slot_purchase_min: int = 1
account_slot_purchase_max: int = 20
wechat_app_id: str = ""
wechat_mch_id: str = ""
wechat_api_v3_key: str = ""
wechat_cert_serial: str = ""
wechat_private_key: str = ""
alipay_app_id: str = ""
alipay_private_key: str = ""
alipay_public_key: str = ""
alipay_sandbox: bool = False
email_verify_subject: str = DEFAULT_EMAIL_VERIFY_SUBJECT
email_verify_body: str = DEFAULT_EMAIL_VERIFY_BODY
email_verify_html: str = DEFAULT_EMAIL_VERIFY_HTML
email_test_subject: str = DEFAULT_EMAIL_TEST_SUBJECT
email_test_body: str = DEFAULT_EMAIL_TEST_BODY
email_test_html: str = DEFAULT_EMAIL_TEST_HTML
def smtp_configured(self) -> bool:
sender = (self.smtp_from or self.smtp_user or "").strip()
return bool(self.smtp_host.strip() and sender)
def app_url_normalized(self) -> str:
return (self.app_url or "http://localhost:8800").rstrip("/")
def wechat_pay_configured(self) -> bool:
return bool(
self.wechat_app_id.strip()
and self.wechat_mch_id.strip()
and self.wechat_api_v3_key.strip()
and self.wechat_cert_serial.strip()
and self.wechat_private_key.strip()
)
def alipay_configured(self) -> bool:
return bool(
self.alipay_app_id.strip()
and self.alipay_private_key.strip()
and self.alipay_public_key.strip()
)
def payment_channel_available(self, channel: str) -> bool:
if channel == "wechat":
return self.wechat_pay_enabled and self.wechat_pay_configured()
if channel == "alipay":
return self.alipay_pay_enabled and self.alipay_configured()
return False
def payment_channel_selectable(self, channel: str) -> bool:
"""用户可选的支付渠道(含演示模式)。"""
if channel == "wechat":
if not self.wechat_pay_enabled:
return False
return self.wechat_pay_configured() or (
self.payment_demo_mode and self.payment_enabled
)
if channel == "alipay":
if not self.alipay_pay_enabled:
return False
return self.alipay_configured() or (
self.payment_demo_mode and self.payment_enabled
)
return False
def available_payment_channels(self) -> list[str]:
channels = []
if self.payment_channel_selectable("wechat"):
channels.append("wechat")
if self.payment_channel_selectable("alipay"):
channels.append("alipay")
return channels
_settings_cache: SystemSettingsData | None = None
def _coerce_bool(value: Any, default: bool) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.lower() in ("1", "true", "yes", "on")
if value is None:
return default
return bool(value)
def _parse_settings(raw: dict[str, Any]) -> SystemSettingsData:
base = SystemSettingsData()
allowed = {f.name for f in fields(SystemSettingsData)}
merged: dict[str, Any] = {}
for key in allowed:
if key in raw:
merged[key] = raw[key]
if "smtp_port" in merged:
try:
merged["smtp_port"] = int(merged["smtp_port"])
except (TypeError, ValueError):
merged["smtp_port"] = base.smtp_port
if "email_verify_token_hours" in merged:
try:
merged["email_verify_token_hours"] = max(
1, min(168, int(merged["email_verify_token_hours"]))
)
except (TypeError, ValueError):
merged["email_verify_token_hours"] = base.email_verify_token_hours
if "auto_reply_cooldown_seconds" in merged:
try:
merged["auto_reply_cooldown_seconds"] = max(
0, min(86400, int(merged["auto_reply_cooldown_seconds"]))
)
except (TypeError, ValueError):
merged["auto_reply_cooldown_seconds"] = base.auto_reply_cooldown_seconds
if "auto_reply_delay_seconds" in merged:
try:
merged["auto_reply_delay_seconds"] = max(
0, min(86400, int(merged["auto_reply_delay_seconds"]))
)
except (TypeError, ValueError):
merged["auto_reply_delay_seconds"] = base.auto_reply_delay_seconds
if "default_register_max_accounts" in merged:
try:
merged["default_register_max_accounts"] = max(
0, min(999, int(merged["default_register_max_accounts"]))
)
except (TypeError, ValueError):
merged["default_register_max_accounts"] = base.default_register_max_accounts
if "account_slot_unit_price" in merged:
try:
merged["account_slot_unit_price"] = max(
0.01, min(99999.0, float(merged["account_slot_unit_price"]))
)
except (TypeError, ValueError):
merged["account_slot_unit_price"] = base.account_slot_unit_price
for int_key, lo, hi in (
("account_slot_purchase_min", 1, 100),
("account_slot_purchase_max", 1, 100),
):
if int_key in merged:
try:
merged[int_key] = max(lo, min(hi, int(merged[int_key])))
except (TypeError, ValueError):
merged[int_key] = getattr(base, int_key)
for bool_key in (
"registration_enabled",
"email_verification_required",
"email_binding_required",
"smtp_use_tls",
"smtp_use_ssl",
"debug_show_verify_link",
"payment_enabled",
"payment_demo_mode",
"wechat_pay_enabled",
"alipay_pay_enabled",
"alipay_sandbox",
):
if bool_key in merged:
merged[bool_key] = _coerce_bool(merged[bool_key], getattr(base, bool_key))
return SystemSettingsData(**{**asdict(base), **merged})
def get_cached_settings() -> SystemSettingsData:
global _settings_cache
if _settings_cache is None:
_settings_cache = SystemSettingsData()
return _settings_cache
def set_cached_settings(data: SystemSettingsData) -> None:
global _settings_cache
_settings_cache = data
async def load_settings(db: AsyncSession) -> SystemSettingsData:
result = await db.execute(select(AppConfig).where(AppConfig.id == CONFIG_ROW_ID))
row = result.scalar_one_or_none()
if not row or not row.data:
data = SystemSettingsData()
set_cached_settings(data)
return data
try:
payload = json.loads(row.data)
except json.JSONDecodeError:
payload = {}
data = _parse_settings(payload if isinstance(payload, dict) else {})
set_cached_settings(data)
return data
async def ensure_default_settings(db: AsyncSession) -> SystemSettingsData:
result = await db.execute(select(AppConfig).where(AppConfig.id == CONFIG_ROW_ID))
row = result.scalar_one_or_none()
if row:
return await load_settings(db)
data = SystemSettingsData()
row = AppConfig(
id=CONFIG_ROW_ID,
data=json.dumps(asdict(data), ensure_ascii=False),
updated_at=datetime.utcnow(),
)
db.add(row)
await db.commit()
set_cached_settings(data)
return data
async def save_settings(db: AsyncSession, updates: dict[str, Any]) -> SystemSettingsData:
current = await load_settings(db)
merged = asdict(current)
for key, value in updates.items():
if key not in merged or value is None:
continue
if key == "smtp_password":
pwd = str(value).strip()
if not pwd or pwd == PASSWORD_PLACEHOLDER:
continue
merged[key] = pwd
continue
if key in ("wechat_api_v3_key", "wechat_private_key", "alipay_private_key"):
secret = str(value).strip()
if not secret or secret == PASSWORD_PLACEHOLDER:
continue
merged[key] = secret
continue
merged[key] = value
data = _parse_settings(merged)
result = await db.execute(select(AppConfig).where(AppConfig.id == CONFIG_ROW_ID))
row = result.scalar_one_or_none()
if not row:
row = AppConfig(id=CONFIG_ROW_ID, data="{}", updated_at=datetime.utcnow())
db.add(row)
row.data = json.dumps(asdict(data), ensure_ascii=False)
row.updated_at = datetime.utcnow()
await db.commit()
set_cached_settings(data)
return data
def settings_to_public(data: SystemSettingsData) -> dict[str, bool]:
return {
"registration_enabled": data.registration_enabled,
"email_verification_required": data.email_verification_required,
"email_binding_required": data.email_binding_required,
}
def settings_to_admin_response(data: SystemSettingsData) -> dict[str, Any]:
payload = asdict(data)
payload["smtp_password"] = PASSWORD_PLACEHOLDER if data.smtp_password else ""
payload["smtp_password_configured"] = bool(data.smtp_password)
return payload
PAYMENT_SETTING_KEYS = (
"payment_enabled",
"payment_demo_mode",
"wechat_pay_enabled",
"alipay_pay_enabled",
"account_slot_unit_price",
"account_slot_purchase_min",
"account_slot_purchase_max",
"wechat_app_id",
"wechat_mch_id",
"wechat_api_v3_key",
"wechat_cert_serial",
"wechat_private_key",
"alipay_app_id",
"alipay_private_key",
"alipay_public_key",
"alipay_sandbox",
)
def settings_to_payment_response(data: SystemSettingsData) -> dict[str, Any]:
payload = {key: getattr(data, key) for key in PAYMENT_SETTING_KEYS}
payload["app_url"] = data.app_url
payload["wechat_api_v3_key"] = PASSWORD_PLACEHOLDER if data.wechat_api_v3_key else ""
payload["wechat_api_v3_key_configured"] = bool(data.wechat_api_v3_key)
payload["wechat_private_key"] = PASSWORD_PLACEHOLDER if data.wechat_private_key else ""
payload["wechat_private_key_configured"] = bool(data.wechat_private_key)
payload["alipay_private_key"] = PASSWORD_PLACEHOLDER if data.alipay_private_key else ""
payload["alipay_private_key_configured"] = bool(data.alipay_private_key)
payload["wechat_pay_configured"] = data.wechat_pay_configured()
payload["alipay_configured"] = data.alipay_configured()
return payload
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env bash
# 宝塔一键修复:权限 + 安装依赖 + 提示正确启动命令
# SSH 执行: bash /www/wwwroot/douyin/backend/baota_fix.sh
set -euo pipefail
BACKEND="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$BACKEND/.." && pwd)"
# shellcheck source=lib/resolve_python.sh
source "$BACKEND/lib/resolve_python.sh"
echo "========================================"
echo " 宝塔环境修复"
echo "========================================"
# 权限
chown -R www:www "$ROOT" 2>/dev/null || true
chmod +x "$BACKEND/baota_start.sh" "$BACKEND/baota_init.sh" 2>/dev/null || true
chmod +x "$ROOT/install.sh" "$ROOT/start_web.sh" "$ROOT/start_backend.sh" 2>/dev/null || true
if [ -d "$BACKEND/.venv/bin" ]; then
chmod -R 755 "$BACKEND/.venv"
chmod +x "$BACKEND/.venv/bin/"* 2>/dev/null || true
fi
echo "[OK] 权限已修复"
if [ -n "${KEFU_PYTHON:-}" ] && [ -x "$KEFU_PYTHON" ]; then
PY="$KEFU_PYTHON"
if ! "$PY" -c "import ssl" 2>/dev/null; then
echo "[错误] KEFU_PYTHON 无 SSL: $KEFU_PYTHON"
exit 1
fi
else
PY="$(resolve_python "$BACKEND" "$ROOT")" || {
echo
echo "请先编译可用 Python:"
echo " AUTO_BUILD_PYTHON=1 bash $ROOT/install.sh"
exit 1
}
fi
echo "[OK] Python: $PY ($("$PY" -V 2>&1))"
if ! "$PY" -c "import ssl" 2>/dev/null; then
echo "[错误] 该 Python 无 SSL,请换环境或执行: AUTO_BUILD_PYTHON=1 $ROOT/install.sh"
exit 1
fi
echo "[1/3] pip install requirements..."
"$PY" -m pip install -U pip
"$PY" -m pip install -r "$BACKEND/requirements.txt"
echo "[OK] Python 依赖已安装"
echo "[2/3] 校验 uvicorn..."
"$PY" -c "import uvicorn, fastapi; print(' uvicorn + fastapi OK')"
echo "[3/3] 可选:完整初始化 Playwright/前端"
echo " bash $BACKEND/baota_init.sh"
echo
echo "========================================"
echo " 请在宝塔面板修改以下配置:"
echo
echo " 项目路径: $BACKEND"
echo " 启动命令: bash $BACKEND/baota_start.sh"
echo " 初始化命令: bash $BACKEND/baota_init.sh"
echo " 依赖包: $BACKEND/requirements.txt"
echo
echo " 不要用:"
echo " backend/.venv/bin/uvicorn (不存在)"
echo " start_web.bat (Windows 脚本,Linux 不能运行)"
echo "========================================"
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env bash
# 宝塔 Python 项目 - 项目初始化命令
# 在面板填写: /www/wwwroot/douyin/backend/baota_init.sh
# 执行时机: 宝塔创建虚拟环境并 pip install requirements.txt 之后
set -euo pipefail
BACKEND="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$BACKEND/.." && pwd)"
# shellcheck source=lib/resolve_python.sh
source "$BACKEND/lib/resolve_python.sh"
PY="$(resolve_python "$BACKEND" "$ROOT")"
ensure_uvicorn "$PY" "$BACKEND"
echo "========================================"
echo " 抖音客服 - 宝塔项目初始化"
echo " Python: $("$PY" -V 2>&1) @ $PY"
echo "========================================"
# 校验 Python
"$PY" -c "import ssl; print('[OK] OpenSSL:', ssl.OPENSSL_VERSION)"
echo "[1/4] Playwright Chromium..."
KEFU_PYTHON="$PY" bash "$BACKEND/baota_playwright.sh"
echo "[2/4] IM 签名 Node 依赖..."
if command -v npm &>/dev/null; then
cd "$BACKEND/rpa_engine/douyin_im/static"
npm install --no-fund --no-audit
else
echo "[警告] 未找到 npm,请宝塔安装 Node 18+ 后重新运行本脚本"
fi
echo "[3/4] 前端依赖与构建..."
if command -v npm &>/dev/null && [ -f "$ROOT/frontend/package.json" ]; then
cd "$ROOT/frontend"
npm install --no-fund --no-audit
npm run build
else
echo "[警告] 跳过前端构建"
fi
echo "[4/4] 校验..."
cd "$BACKEND"
"$PY" -c "import fastapi, uvicorn, playwright; print(' 核心包 OK')"
[ -f "$ROOT/frontend/dist/index.html" ] && echo " 前端静态 OK" || echo "[警告] 未找到 frontend/dist"
echo
echo "========================================"
echo " 宝塔初始化完成"
echo " 启动命令请用: bash $BACKEND/baota_start.sh"
echo "========================================"
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env bash
# 安装 Playwright Chromium(Linux 宝塔/无桌面服务器)
# 用法: bash /www/wwwroot/douyin/backend/baota_playwright.sh
set -euo pipefail
BACKEND="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$BACKEND/.." && pwd)"
# shellcheck source=lib/resolve_python.sh
source "$BACKEND/lib/resolve_python.sh"
PY="${KEFU_PYTHON:-}"
if [ -z "$PY" ]; then
PY="$(resolve_python "$BACKEND" "$ROOT")"
fi
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$ROOT/playwright-browsers}"
mkdir -p "$PLAYWRIGHT_BROWSERS_PATH"
chown -R www:www "$PLAYWRIGHT_BROWSERS_PATH" 2>/dev/null || true
echo "========================================"
echo " Playwright Chromium 安装"
echo " Python: $PY"
echo " 目录: $PLAYWRIGHT_BROWSERS_PATH"
echo "========================================"
echo "[1/4] 安装系统依赖(需 root)..."
if [ "$(uname -s)" = "Linux" ]; then
if [ "$(id -u)" -eq 0 ]; then
# 优先用 dnf(阿里云 Linux 3 / Anolis / RHEL 8+),回退到 yum
DNF=""
if command -v dnf &>/dev/null; then
DNF="dnf"
elif command -v yum &>/dev/null; then
DNF="yum"
fi
if [ -n "$DNF" ]; then
# Chromium 运行库 + Xvfb 虚拟显示(无桌面服务器跑有头浏览器扫码登录必需)
"$DNF" install -y atk at-spi2-atk cups-libs libdrm libXcomposite libXdamage \
libXrandr mesa-libgbm pango alsa-lib nss nspr libxkbcommon \
libXScrnSaver gtk3 xorg-x11-server-Xvfb 2>/dev/null || true
elif command -v apt-get &>/dev/null; then
apt-get update
apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 \
libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 \
libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2 \
libatspi2.0-0 libxshmfence1 xvfb 2>/dev/null || true
fi
"$PY" -m playwright install-deps chromium || true
else
echo " 非 root,跳过 install-deps / Xvfb(若启动失败请 sudo 执行本脚本)"
fi
fi
echo "[2/4] 安装 pyvirtualdisplay(自动拉起 Xvfb 虚拟显示)..."
# requirements.txt 已含 pyvirtualdisplay;这里兜底确保 venv 内已安装
"$PY" -m pip show pyvirtualdisplay &>/dev/null \
|| "$PY" -m pip install "pyvirtualdisplay==3.0" || true
echo "[3/4] 下载 Chromium..."
"$PY" -m playwright install chromium
echo "[4/4] 校验..."
CHROME=$(find "$PLAYWRIGHT_BROWSERS_PATH" -name chrome -type f 2>/dev/null | head -1)
if [ -n "$CHROME" ] && [ -x "$CHROME" ]; then
echo "[OK] 浏览器: $CHROME"
chmod +x "$CHROME" 2>/dev/null || true
chown -R www:www "$PLAYWRIGHT_BROWSERS_PATH" 2>/dev/null || true
else
echo "[错误] 未找到 chrome 可执行文件,请检查网络或磁盘空间"
ls -la "$PLAYWRIGHT_BROWSERS_PATH" 2>/dev/null || true
exit 1
fi
echo
# 校验虚拟显示链路:无桌面 Linux 上扫码登录依赖 Xvfb + pyvirtualdisplay
if [ "$(uname -s)" = "Linux" ] && [ -z "${DISPLAY:-}" ]; then
XVFB_OK=0
command -v Xvfb &>/dev/null && XVFB_OK=1
PVD_OK=0
"$PY" -c "import pyvirtualdisplay" &>/dev/null && PVD_OK=1
if [ "$XVFB_OK" = "1" ] && [ "$PVD_OK" = "1" ]; then
echo "[OK] Xvfb + pyvirtualdisplay 就绪,云端可直接点「登录」用手机扫码"
else
echo "[警告] 扫码登录所需的虚拟显示未就绪:"
[ "$XVFB_OK" = "0" ] && echo " - 缺少 Xvfb,请 root 执行: dnf install -y xorg-x11-server-Xvfb"
[ "$PVD_OK" = "0" ] && echo " - 缺少 pyvirtualdisplay,请执行: $PY -m pip install pyvirtualdisplay==3.0"
echo " 或改用: xvfb-run -a bash $ROOT/start_web.sh 启动服务"
fi
fi
echo
echo "请将以下写入 .env 或宝塔环境变量:"
echo " PLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH"
echo
echo "安装完成,重启项目: bash $BACKEND/baota_start.sh"
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
# 宝塔 Python 项目 - 启动命令
# 面板「启动命令」必须填这一行(不要用 .venv/bin/uvicorn):
# bash /www/wwwroot/douyin/backend/baota_start.sh
set -euo pipefail
BACKEND="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$BACKEND/.." && pwd)"
# shellcheck source=lib/resolve_python.sh
source "$BACKEND/lib/resolve_python.sh"
cd "$BACKEND"
PY="$(resolve_python "$BACKEND" "$ROOT")"
ensure_uvicorn "$PY" "$BACKEND"
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$ROOT/playwright-browsers}"
export KEFU_SERVE_WEB="${KEFU_SERVE_WEB:-true}"
export KEFU_STATIC_DIR="${KEFU_STATIC_DIR:-$ROOT/frontend/dist}"
# 启动前检查 Chromium(缺失时给出明确提示)
if ! find "$PLAYWRIGHT_BROWSERS_PATH" -name chrome -type f 2>/dev/null | grep -q .; then
echo "[警告] 未找到 Chromium,请执行: bash $BACKEND/baota_playwright.sh" >&2
fi
if [ -f "$ROOT/.env" ]; then
set -a
# shellcheck disable=SC1090
source "$ROOT/.env"
set +a
fi
HOST="${KEFU_HOST:-0.0.0.0}"
PORT="${KEFU_PORT:-8000}"
echo "[baota] Python: $PY"
echo "[baota] Listen: http://${HOST}:${PORT}"
exec "$PY" -m uvicorn main:app --host "$HOST" --port "$PORT"
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env bash
# 宝塔/服务器快速诊断:服务是否在跑、端口是否监听
set -uo pipefail
BACKEND="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$BACKEND/.." && pwd)"
PORT="${KEFU_PORT:-8000}"
echo "========================================"
echo " 抖音客服 - 运行诊断"
echo "========================================"
echo
echo "[1] 端口 ${PORT} 监听状态:"
if command -v ss &>/dev/null; then
ss -tlnp | grep ":${PORT} " || echo " -> 未监听(服务未启动)"
elif command -v netstat &>/dev/null; then
netstat -tlnp 2>/dev/null | grep ":${PORT} " || echo " -> 未监听(服务未启动)"
else
echo " (无法检测,请安装 ss 或 netstat)"
fi
echo
echo "[2] Python 环境:"
BROKEN_BT="/home/www/server/pyporject_evn/versions/3.11.15/bin/python"
if [ -x "$BROKEN_BT" ] && ! "$BROKEN_BT" -c "import ssl" 2>/dev/null; then
echo " [X] 宝塔 Python 无 SSL(不可用): $BROKEN_BT"
echo " 修复: AUTO_BUILD_PYTHON=1 bash $ROOT/install.sh"
fi
if [ -f "$BACKEND/lib/resolve_python.sh" ]; then
# shellcheck source=lib/resolve_python.sh
source "$BACKEND/lib/resolve_python.sh"
if PY="$(resolve_python "$BACKEND" "$ROOT" 2>/dev/null)"; then
echo " Python: $PY"
"$PY" -V 2>&1 | sed 's/^/ /'
if "$PY" -c "import ssl" 2>/dev/null; then
"$PY" -c "import ssl; print(' SSL:', ssl.OPENSSL_VERSION)"
else
echo " SSL: 无(不可用)"
fi
if "$PY" -c "import uvicorn" 2>/dev/null; then
echo " uvicorn: 已安装"
else
echo " uvicorn: 未安装 -> 运行 bash $BACKEND/baota_fix.sh"
fi
else
echo " 未找到可用 Python 3.10+"
fi
else
command -v python && python -V || echo " 未找到 python"
fi
echo
echo "[3] 前端静态:"
if [ -f "$ROOT/frontend/dist/index.html" ]; then
echo " OK: $ROOT/frontend/dist/index.html"
else
echo " 缺失 -> 运行 bash $BACKEND/baota_init.sh"
fi
echo
echo "[4] 宝塔项目进程:"
ps aux 2>/dev/null | grep -E "uvicorn|douyin" | grep -v grep || echo " 未发现 uvicorn 进程"
echo
echo "[5] 防火墙(CentOS firewalld):"
if command -v firewall-cmd &>/dev/null && systemctl is-active firewalld &>/dev/null; then
firewall-cmd --list-ports 2>/dev/null | grep -q "${PORT}" \
&& echo " 端口 ${PORT} 已放行" \
|| echo " 端口 ${PORT} 未放行 -> firewall-cmd --add-port=${PORT}/tcp --permanent && firewall-cmd --reload"
else
echo " firewalld 未运行或未安装(可忽略,检查云服务器安全组)"
fi
echo
echo "========================================"
echo " 若端口未监听,按顺序执行:"
echo " 若宝塔 Python 无 SSL,必须先编译可用 Python:"
echo " AUTO_BUILD_PYTHON=1 bash $ROOT/install.sh"
echo " KEFU_PYTHON=/usr/local/python3.11/bin/python3.11 bash $BACKEND/baota_fix.sh"
echo " bash $BACKEND/baota_init.sh"
echo " 宝塔启动命令: bash $BACKEND/baota_start.sh"
echo
echo " 手动测试启动:"
echo " bash $BACKEND/baota_start.sh"
echo "========================================"
+104
View File
@@ -0,0 +1,104 @@
#!/usr/bin/env bash
# 使用宝塔「Python环境管理」安装的 Python 完成项目配置
# 用法: bash /www/wwwroot/douyin/backend/baota_use_panel_python.sh
set -euo pipefail
BACKEND="$(cd "$(dirname "$0")" && pwd)"
ROOT="$(cd "$BACKEND/.." && pwd)"
echo "========================================"
echo " 宝塔 Python 环境配置"
echo "========================================"
echo
# 扫描宝塔 Python(环境管理 / Python项目)
PY=""
for base in \
/home/www/server/pyporject_evn/versions \
/www/server/pyporject_evn/versions \
/www/server/panel/pyenv/versions; do
[ -d "$base" ] || continue
for ver_dir in $(ls -d "$base"/* 2>/dev/null | sort -V -r); do
for name in python3 python; do
candidate="$ver_dir/bin/$name"
[ -x "$candidate" ] || continue
ver="$("$candidate" -V 2>&1 | awk '{print $2}')"
echo "[发现] $candidate ($ver)"
if ! "$candidate" -c "import sys; sys.exit(0 if sys.version_info>=(3,10) else 1)" 2>/dev/null; then
echo " 跳过: 版本低于 3.10"
continue
fi
if ! "$candidate" -c "import ssl" 2>/dev/null; then
echo " 跳过: 无 SSL(需在宝塔重装 Python 前先 yum install openssl-devel)"
continue
fi
PY="$candidate"
echo " -> 选用"
break 3
done
done
done
if [ -z "$PY" ]; then
echo
echo "[错误] 未找到可用的宝塔 Python 3.10+(需带 SSL)。"
echo
echo "请在宝塔面板操作:"
echo " 1) 软件商店 -> Python项目管理器 -> 环境管理"
echo " 2) SSH 先执行: yum install -y openssl openssl-devel"
echo " 3) 环境管理 -> 安装 Python 3.11"
echo " 4) 重新运行本脚本"
echo
echo "若已安装但仍无 SSL,卸载该版本后重装。"
exit 1
fi
echo
echo "[OK] 使用宝塔 Python: $PY"
"$PY" -c "import ssl; print('[OK] OpenSSL:', ssl.OPENSSL_VERSION)"
echo
export KEFU_PYTHON="$PY"
export PLAYWRIGHT_BROWSERS_PATH="${PLAYWRIGHT_BROWSERS_PATH:-$ROOT/playwright-browsers}"
echo "[1/4] 安装 Python 依赖..."
"$PY" -m pip install -U pip
"$PY" -m pip install -r "$BACKEND/requirements.txt"
echo
echo "[2/4] 创建项目 venv(可选,便于宝塔启动)..."
"$PY" -m venv "$BACKEND/.venv" --clear
"$BACKEND/.venv/bin/python" -m pip install -U pip -q
"$BACKEND/.venv/bin/pip" install -r "$BACKEND/requirements.txt"
chmod -R 755 "$BACKEND/.venv"
chown -R www:www "$BACKEND/.venv" 2>/dev/null || true
echo " venv: $BACKEND/.venv"
echo
echo "[3/4] Playwright Chromium..."
KEFU_PYTHON="$BACKEND/.venv/bin/python" bash "$BACKEND/baota_playwright.sh"
echo
echo "[4/4] 前端与 IM 依赖..."
if command -v npm &>/dev/null; then
cd "$BACKEND/rpa_engine/douyin_im/static" && npm install --no-fund --no-audit
cd "$ROOT/frontend" && npm install --no-fund --no-audit && npm run build
else
echo "[警告] 未安装 Node.js,请宝塔安装 Node 18+ 后手动 npm install"
fi
chown -R www:www "$ROOT" 2>/dev/null || true
echo
echo "========================================"
echo " 配置完成"
echo
echo " 宝塔 Python项目 设置:"
echo " 项目路径: $BACKEND"
echo " Python环境: 选环境管理里的 3.11"
echo " 启动命令: bash $BACKEND/baota_start.sh"
echo " 依赖包: $BACKEND/requirements.txt"
echo
echo " .env 添加:"
echo " PLAYWRIGHT_BROWSERS_PATH=$PLAYWRIGHT_BROWSERS_PATH"
echo "========================================"
Binary file not shown.

After

Width:  |  Height:  |  Size: 318 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 296 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 KiB

+116
View File
@@ -0,0 +1,116 @@
"""桌面客户端在线升级:发布配置(存 AppConfig 行)+ 安装包文件管理。
发布配置以 JSON 形式存放在 app_config 表的 id=DESKTOP_CONFIG_ROW_ID 行,
安装包文件落在 backend/uploads/desktop/ 下。仅管理员可读写。
"""
from __future__ import annotations
import json
import os
from dataclasses import asdict, dataclass, fields
from datetime import datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from models.models import AppConfig
# app_config 表里给桌面发布单独用一行(系统设置用的是 id=1)
DESKTOP_CONFIG_ROW_ID = 2
INSTALLER_DIR = os.path.join(os.path.dirname(__file__), "uploads", "desktop")
INSTALLER_FILENAME = "DouyinHostedDesktop-Setup.exe"
@dataclass
class DesktopReleaseData:
version: str = "" # 已发布的最新版本号,如 1.0.1
force: bool = False # true=强制升级(不可跳过)
notes: str = "" # 更新说明
installer_name: str = "" # 上传时的原始文件名
installer_size: int = 0 # 安装包字节数
installer_url: str = "" # 外部安装包直链(填写后优先于本地上传文件)
updated_at: str = "" # 最近一次发布/上传时间
def package_ready(data: "DesktopReleaseData") -> bool:
"""是否具备可供客户端下载的安装包:外部直链 或 本地已上传文件。"""
return bool((data.installer_url or "").strip()) or installer_exists()
def installer_path() -> str:
return os.path.join(INSTALLER_DIR, INSTALLER_FILENAME)
def installer_exists() -> bool:
p = installer_path()
return os.path.isfile(p) and os.path.getsize(p) > 0
def _coerce_bool(value: Any, default: bool) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.lower() in ("1", "true", "yes", "on")
if value is None:
return default
return bool(value)
def _parse(raw: dict[str, Any]) -> DesktopReleaseData:
base = DesktopReleaseData()
allowed = {f.name for f in fields(DesktopReleaseData)}
merged = {k: raw[k] for k in allowed if k in raw}
if "force" in merged:
merged["force"] = _coerce_bool(merged["force"], base.force)
if "installer_size" in merged:
try:
merged["installer_size"] = int(merged["installer_size"])
except (TypeError, ValueError):
merged["installer_size"] = base.installer_size
return DesktopReleaseData(**{**asdict(base), **merged})
async def load_release(db: AsyncSession) -> DesktopReleaseData:
row = (
await db.execute(select(AppConfig).where(AppConfig.id == DESKTOP_CONFIG_ROW_ID))
).scalar_one_or_none()
if not row or not row.data:
return DesktopReleaseData()
try:
payload = json.loads(row.data)
except json.JSONDecodeError:
payload = {}
return _parse(payload if isinstance(payload, dict) else {})
async def save_release(db: AsyncSession, updates: dict[str, Any]) -> DesktopReleaseData:
current = await load_release(db)
merged = asdict(current)
for key, value in updates.items():
if key not in merged or value is None:
continue
merged[key] = value
data = _parse(merged)
data.updated_at = datetime.utcnow().isoformat()
row = (
await db.execute(select(AppConfig).where(AppConfig.id == DESKTOP_CONFIG_ROW_ID))
).scalar_one_or_none()
if not row:
row = AppConfig(id=DESKTOP_CONFIG_ROW_ID, data="{}", updated_at=datetime.utcnow())
db.add(row)
row.data = json.dumps(asdict(data), ensure_ascii=False)
row.updated_at = datetime.utcnow()
await db.commit()
return data
def remove_installer() -> None:
p = installer_path()
try:
if os.path.exists(p):
os.remove(p)
except OSError:
pass
+201
View File
@@ -0,0 +1,201 @@
"""桌面客户端在线升级接口。
公开接口(桌面客户端启动时调用,无需登录):
GET /api/desktop/latest 返回最新版本清单 {version,url,force,notes},无发布时返回 {}
GET /api/desktop/download 下载安装包
管理员接口(仅 admin):
GET /api/desktop/release 读取当前发布配置
PUT /api/desktop/release 修改版本号/强制升级/更新说明
POST /api/desktop/release/installer 上传安装包(.exe)
DELETE /api/desktop/release/installer 删除安装包(即停用在线升级)
"""
from __future__ import annotations
import os
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
from fastapi.responses import FileResponse
from pydantic import BaseModel, Field
from sqlalchemy.ext.asyncio import AsyncSession
from auth.dependencies import require_admin
from auth.system_settings import get_cached_settings
from desktop_release import (
INSTALLER_DIR,
installer_exists,
installer_path,
load_release,
package_ready,
remove_installer,
save_release,
)
from models.database import get_db
from models.models import User
router = APIRouter(prefix="/api/desktop", tags=["desktop-update"])
_MAX_INSTALLER_BYTES = 500 * 1024 * 1024 # 500MB 上限,足够覆盖安装包
class DesktopReleaseResponse(BaseModel):
version: str = ""
force: bool = False
notes: str = ""
installer_name: str = ""
installer_size: int = 0
installer_url: str = ""
updated_at: str = ""
has_installer: bool = False # 本地是否已上传安装包文件
package_ready: bool = False # 是否具备可下载安装包(外链或本地文件)
download_url: str = "" # 本地上传文件的下载地址
class DesktopReleaseUpdate(BaseModel):
version: str | None = Field(default=None, max_length=40)
force: bool | None = None
notes: str | None = Field(default=None, max_length=4000)
installer_url: str | None = Field(default=None, max_length=1000)
def _public_base(request: Request) -> str:
"""构造给客户端用的绝对地址:优先后台配置的 app_url,本地址兜底用请求来源。"""
try:
app_url = get_cached_settings().app_url_normalized()
except Exception:
app_url = ""
if not app_url or "localhost" in app_url or "127.0.0.1" in app_url:
app_url = str(request.base_url).rstrip("/")
return app_url
def _to_response(data, request: Request) -> DesktopReleaseResponse:
return DesktopReleaseResponse(
version=data.version,
force=data.force,
notes=data.notes,
installer_name=data.installer_name,
installer_size=data.installer_size,
installer_url=(data.installer_url or "").strip(),
updated_at=data.updated_at,
has_installer=installer_exists(),
package_ready=package_ready(data),
download_url=f"{_public_base(request)}/api/desktop/download",
)
# ---------- 公开接口 ----------
@router.get("/latest")
async def desktop_latest(request: Request, db: AsyncSession = Depends(get_db)):
"""桌面客户端启动检查更新。未发布或安装包缺失时返回空对象。"""
data = await load_release(db)
if not data.version or not package_ready(data):
return {}
external = (data.installer_url or "").strip()
download_url = external or f"{_public_base(request)}/api/desktop/download"
return {
"version": data.version,
"url": download_url,
"force": bool(data.force),
"notes": data.notes or "",
}
@router.get("/download")
async def desktop_download(db: AsyncSession = Depends(get_db)):
"""下载安装包。"""
if not installer_exists():
raise HTTPException(status_code=404, detail="安装包不存在")
return FileResponse(
installer_path(),
media_type="application/octet-stream",
filename="DouyinHostedDesktop-Setup.exe",
)
# ---------- 管理员接口 ----------
@router.get("/release", response_model=DesktopReleaseResponse)
async def get_release(
request: Request,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
data = await load_release(db)
return _to_response(data, request)
@router.put("/release", response_model=DesktopReleaseResponse)
async def update_release(
body: DesktopReleaseUpdate,
request: Request,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
updates = body.model_dump(exclude_unset=True)
if "version" in updates and updates["version"] is not None:
updates["version"] = updates["version"].strip()
if "notes" in updates and updates["notes"] is not None:
updates["notes"] = updates["notes"].strip()
if "installer_url" in updates and updates["installer_url"] is not None:
url = updates["installer_url"].strip()
if url and not url.lower().startswith(("http://", "https://")):
raise HTTPException(status_code=400, detail="安装包网址需以 http:// 或 https:// 开头")
updates["installer_url"] = url
data = await save_release(db, updates)
return _to_response(data, request)
@router.post("/release/installer", response_model=DesktopReleaseResponse)
async def upload_installer(
request: Request,
file: UploadFile = File(...),
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
filename = (file.filename or "").strip()
if not filename.lower().endswith(".exe"):
raise HTTPException(status_code=400, detail="请上传 .exe 安装包")
os.makedirs(INSTALLER_DIR, exist_ok=True)
tmp_path = installer_path() + ".uploading"
size = 0
try:
with open(tmp_path, "wb") as f:
while True:
chunk = await file.read(1024 * 1024)
if not chunk:
break
size += len(chunk)
if size > _MAX_INSTALLER_BYTES:
raise HTTPException(status_code=400, detail="安装包超过 500MB 上限")
f.write(chunk)
except HTTPException:
if os.path.exists(tmp_path):
os.remove(tmp_path)
raise
except Exception as exc: # noqa: BLE001
if os.path.exists(tmp_path):
os.remove(tmp_path)
raise HTTPException(status_code=500, detail=f"保存安装包失败: {exc}") from exc
if size == 0:
if os.path.exists(tmp_path):
os.remove(tmp_path)
raise HTTPException(status_code=400, detail="安装包为空")
os.replace(tmp_path, installer_path())
data = await save_release(db, {"installer_name": filename, "installer_size": size})
return _to_response(data, request)
@router.delete("/release/installer", response_model=DesktopReleaseResponse)
async def delete_installer(
request: Request,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
remove_installer()
data = await save_release(db, {"installer_name": "", "installer_size": 0})
return _to_response(data, request)
+29
View File
@@ -0,0 +1,29 @@
"""帮助中心静态页面。"""
from pathlib import Path
from fastapi import HTTPException
from fastapi.responses import FileResponse
from settings import BACKEND_DIR, PROJECT_ROOT
HELP_STATIC_DIR = BACKEND_DIR / "static" / "help"
CREDENTIAL_TOOL_CANDIDATES = (
PROJECT_ROOT / "凭证采集工具.html",
HELP_STATIC_DIR / "credential-tool.html",
)
def _resolve_credential_tool_html() -> Path:
for path in CREDENTIAL_TOOL_CANDIDATES:
if path.is_file():
return path
raise HTTPException(status_code=404, detail="凭证采集工具页面不存在")
def serve_credential_tool() -> FileResponse:
html_path = _resolve_credential_tool_html()
return FileResponse(
html_path,
media_type="text/html; charset=utf-8",
content_disposition_type="inline",
)
BIN
View File
Binary file not shown.
Binary file not shown.

After

Width:  |  Height:  |  Size: 211 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 211 KiB

+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
# 共用:解析可用 Python(必须 3.10+ 且带 SSL,跳过宝塔坏环境)
_py_ok() {
local py="$1"
[ -n "$py" ] && [ -x "$py" ] || return 1
"$py" -c "import sys, ssl; sys.exit(0 if sys.version_info>=(3,10) else 1)" 2>/dev/null
}
resolve_python() {
local backend="${1:?}"
local root="${2:-$(cd "$backend/.." && pwd)}"
local py="" candidate ver_dir
# 不 source 宝塔环境,避免 PATH 被坏 Python 污染
local -a candidates=()
candidates+=("$backend/.venv/bin/python")
candidates+=("/usr/local/python3.11/bin/python3.11")
candidates+=("/usr/local/python3.10/bin/python3.10")
candidates+=("/usr/bin/python3.11")
candidates+=("/usr/bin/python3.10")
candidates+=("/usr/local/bin/python3.11")
candidates+=("/usr/local/bin/python3.10")
# 动态扫描宝塔 Python(不写死版本号)
for base in \
/home/www/server/pyporject_evn/versions \
/www/server/pyporject_evn/versions \
/www/server/panel/pyenv/versions; do
[ -d "$base" ] || continue
for ver_dir in $(ls -d "$base"/* 2>/dev/null | sort -V -r); do
for name in python3.11 python3.10 python3 python; do
candidates+=("$ver_dir/bin/$name")
done
done
done
candidates+=("$(command -v python3.11 2>/dev/null || true)")
candidates+=("$(command -v python3.10 2>/dev/null || true)")
candidates+=("$(command -v python3 2>/dev/null || true)")
candidates+=("$(command -v python 2>/dev/null || true)")
declare -A seen=()
for candidate in "${candidates[@]}"; do
[ -n "$candidate" ] || continue
[ -n "${seen[$candidate]:-}" ] && continue
seen[$candidate]=1
if _py_ok "$candidate"; then
py="$candidate"
break
fi
done
if [ -z "$py" ]; then
echo "[错误] 未找到带 SSL 的 Python 3.10+。" >&2
echo " 宝塔当前 Python 缺 _ssl,请执行:" >&2
echo " AUTO_BUILD_PYTHON=1 bash $root/install.sh" >&2
echo " 然后: KEFU_PYTHON=/usr/local/python3.11/bin/python3.11 bash $backend/baota_fix.sh" >&2
return 1
fi
echo "$py"
}
ensure_uvicorn() {
local py="${1:?}"
local backend="${2:?}"
if "$py" -c "import uvicorn" 2>/dev/null; then
return 0
fi
if ! "$py" -c "import ssl" 2>/dev/null; then
echo "[错误] $py 无 SSL,无法 pip install" >&2
return 1
fi
echo "[修复] 正在安装 Python 依赖..." >&2
"$py" -m pip install -U pip -q
"$py" -m pip install -r "$backend/requirements.txt"
}
+186
View File
@@ -0,0 +1,186 @@
"""链接卡片落地页:带 SEO meta 与自动跳转。"""
from __future__ import annotations
import html
import json
import os
import re
import secrets
from io import BytesIO
from typing import Optional, TYPE_CHECKING
if TYPE_CHECKING:
from fastapi import Request
_SLUG_RE = re.compile(r"^[a-zA-Z0-9_-]{6,64}$")
FAVICON_SIZE = 32
COVER_SIZE = 256
FAVICON_MIME = "image/png"
def generate_slug() -> str:
return secrets.token_urlsafe(9).replace("-", "_").replace(".", "_")[:12]
def is_valid_slug(slug: str) -> bool:
return bool(_SLUG_RE.match(slug or ""))
def public_base_url(request: Optional["Request"] = None) -> str:
env = os.getenv("KEFU_PUBLIC_BASE_URL", "").strip().rstrip("/")
if env:
return env
if request is not None:
return str(request.base_url).rstrip("/")
return ""
def absolute_media_url(path: str, request: Optional["Request"] = None) -> str:
path = (path or "").strip()
if not path:
return ""
if path.startswith("http://") or path.startswith("https://"):
return path
base = public_base_url(request)
if not base:
return path
if not path.startswith("/"):
path = f"/{path}"
return f"{base}{path}"
def favicon_mime(image_path: str) -> str:
ext = (image_path or "").rsplit(".", 1)[-1].lower()
return {
"svg": "image/svg+xml",
"png": "image/png",
"gif": "image/gif",
"webp": "image/webp",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"ico": "image/x-icon",
}.get(ext, FAVICON_MIME)
def favicon_path_for_cover(cover_path: str) -> str:
"""由封面路径推导同名 favicon 文件路径。"""
path = (cover_path or "").strip()
if not path:
return ""
if path.endswith(".favicon.png"):
return path
if path.endswith(".png"):
return f"{path[:-4]}.favicon.png"
base, _ = os.path.splitext(path)
return f"{base}.favicon.png"
def media_path_to_disk(upload_dir: str, media_path: str) -> str:
prefix = "/api/media/link-cards/"
path = (media_path or "").strip()
if not path.startswith(prefix):
return ""
rel = path[len(prefix) :].replace("/", os.sep)
return os.path.join(upload_dir, rel)
def process_card_upload(raw: bytes) -> tuple[bytes, bytes]:
"""将任意位图转为封面 PNG (256x256) 与 favicon PNG (32x32)。"""
if not raw:
raise ValueError("图片为空")
try:
from PIL import Image
except ImportError as exc:
raise RuntimeError("服务器未安装 Pillow,无法转换 favicon") from exc
try:
with Image.open(BytesIO(raw)) as img:
if getattr(img, "is_animated", False):
img.seek(0)
if img.mode not in ("RGB", "RGBA"):
img = img.convert("RGBA")
else:
img = img.copy()
width, height = img.size
if width < 1 or height < 1:
raise ValueError("图片尺寸无效")
side = min(width, height)
left = (width - side) // 2
top = (height - side) // 2
square = img.crop((left, top, left + side, top + side))
resample = Image.Resampling.LANCZOS
cover = square.resize((COVER_SIZE, COVER_SIZE), resample)
favicon = square.resize((FAVICON_SIZE, FAVICON_SIZE), resample)
cover_buf = BytesIO()
cover.save(cover_buf, format="PNG", optimize=True)
favicon_buf = BytesIO()
favicon.save(favicon_buf, format="PNG", optimize=True)
return cover_buf.getvalue(), favicon_buf.getvalue()
except ValueError:
raise
except Exception as exc:
raise ValueError("无法识别图片格式,请上传 JPG、PNG、GIF 或 WebP") from exc
def normalize_target_url(url: str) -> str:
url = (url or "").strip()
if not url:
return ""
if not url.startswith("http://") and not url.startswith("https://"):
url = f"https://{url}"
return url
def build_keywords(title: str, content: str) -> str:
parts = [p.strip() for p in (title, content) if p and p.strip()]
return ", ".join(dict.fromkeys(parts))
def render_link_card_page(
*,
title: str,
content: str,
keywords: str,
cover_url: str,
favicon_url: str,
target_url: str,
) -> str:
safe_title = html.escape(title or "跳转中")
safe_desc = html.escape(content or title or "")
safe_keywords = html.escape(keywords or build_keywords(title, content))
safe_cover = html.escape(cover_url or "")
safe_favicon = html.escape(favicon_url or cover_url or "")
redirect_url = normalize_target_url(target_url)
safe_target = html.escape(redirect_url)
js_target = json.dumps(redirect_url, ensure_ascii=False)
favicon_tag = ""
og_image_tag = ""
if safe_favicon:
favicon_tag = f' <link rel="icon" type="{FAVICON_MIME}" href="{safe_favicon}" />\n'
if safe_cover:
og_image_tag = f' <meta property="og:image" content="{safe_cover}" />\n'
return f"""<!DOCTYPE html>
<html lang="zh-CN">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>{safe_title}</title>
<meta name="description" content="{safe_desc}" />
<meta name="keywords" content="{safe_keywords}" />
<meta property="og:title" content="{safe_title}" />
<meta property="og:description" content="{safe_desc}" />
{og_image_tag}{favicon_tag} <meta http-equiv="refresh" content="0;url={safe_target}" />
<script>location.replace({js_target});</script>
</head>
<body>
<p>正在跳转到目标页面…</p>
<p><a href="{safe_target}">若未自动跳转,请点击这里</a></p>
</body>
</html>"""
+210
View File
@@ -0,0 +1,210 @@
"""链接卡片 API 与公开落地页。"""
from __future__ import annotations
import os
import uuid
from datetime import datetime
from typing import Optional
from fastapi import APIRouter, Depends, File, HTTPException, Request, UploadFile
from fastapi.responses import HTMLResponse
from pydantic import BaseModel, Field
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from auth.dependencies import get_current_user, require_write
from link_cards import (
absolute_media_url,
build_keywords,
favicon_path_for_cover,
generate_slug,
is_valid_slug,
media_path_to_disk,
normalize_target_url,
process_card_upload,
public_base_url,
render_link_card_page,
)
from models.database import get_db
from models.models import LinkCardPage, User
router = APIRouter(tags=["link-cards"])
UPLOAD_DIR = os.path.join(os.path.dirname(__file__), "uploads", "link-cards")
os.makedirs(UPLOAD_DIR, exist_ok=True)
class LinkCardUpsert(BaseModel):
id: Optional[int] = None
title: str = Field(..., min_length=1, max_length=200)
content: str = Field(default="", max_length=2000)
target_url: str = Field(..., min_length=1, max_length=2000)
image_path: str = Field(..., min_length=1, max_length=512)
class LinkCardResponse(BaseModel):
id: int
slug: str
title: str
content: str
target_url: str
image_path: str
page_url: str
cover_url: str
class Config:
from_attributes = True
class UploadImageResponse(BaseModel):
image_path: str
cover_url: str
async def _get_owned_card(
db: AsyncSession, user: User, card_id: int, *, write: bool = False
) -> LinkCardPage:
stmt = select(LinkCardPage).where(LinkCardPage.id == card_id)
card = (await db.execute(stmt)).scalar_one_or_none()
if not card or card.owner_id != user.id:
raise HTTPException(status_code=404, detail="卡片不存在")
if write and user.role == "viewer":
raise HTTPException(status_code=403, detail="无写入权限")
return card
def _card_response(card: LinkCardPage, request: Request) -> LinkCardResponse:
base = public_base_url(request)
page_url = f"{base}/p/{card.slug}" if base else f"/p/{card.slug}"
cover_url = absolute_media_url(card.image_path, request)
return LinkCardResponse(
id=card.id,
slug=card.slug,
title=card.title,
content=card.content or "",
target_url=card.target_url,
image_path=card.image_path,
page_url=page_url,
cover_url=cover_url,
)
@router.post("/api/link-cards/upload-image", response_model=UploadImageResponse)
async def upload_link_card_image(
request: Request,
file: UploadFile = File(...),
user: User = Depends(require_write),
):
if not file.content_type or not file.content_type.startswith("image/"):
raise HTTPException(status_code=400, detail="仅支持上传图片文件")
if file.content_type == "image/svg+xml":
raise HTTPException(status_code=400, detail="请上传 JPG/PNG 等位图,系统将自动转为 favicon PNG")
raw = await file.read()
if not raw:
raise HTTPException(status_code=400, detail="图片为空")
if len(raw) > 4 * 1024 * 1024:
raise HTTPException(status_code=400, detail="图片大小不能超过 4MB")
try:
cover_bytes, favicon_bytes = process_card_upload(raw)
except RuntimeError as exc:
raise HTTPException(status_code=500, detail=str(exc)) from exc
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
owner_dir = os.path.join(UPLOAD_DIR, str(user.id))
os.makedirs(owner_dir, exist_ok=True)
file_id = uuid.uuid4().hex
cover_filename = f"{file_id}.png"
favicon_filename = f"{file_id}.favicon.png"
cover_path = os.path.join(owner_dir, cover_filename)
favicon_path = os.path.join(owner_dir, favicon_filename)
with open(cover_path, "wb") as f:
f.write(cover_bytes)
with open(favicon_path, "wb") as f:
f.write(favicon_bytes)
image_path = f"/api/media/link-cards/{user.id}/{cover_filename}"
return UploadImageResponse(
image_path=image_path,
cover_url=absolute_media_url(image_path, request),
)
@router.post("/api/link-cards", response_model=LinkCardResponse)
async def upsert_link_card(
body: LinkCardUpsert,
request: Request,
db: AsyncSession = Depends(get_db),
user: User = Depends(require_write),
):
title = body.title.strip()
content = (body.content or "").strip()
target_url = normalize_target_url(body.target_url)
image_path = (body.image_path or "").strip()
if not title:
raise HTTPException(status_code=400, detail="请填写卡片标题")
if not target_url:
raise HTTPException(status_code=400, detail="请填写跳转链接")
if not image_path.startswith("/api/media/link-cards/"):
raise HTTPException(status_code=400, detail="请先上传卡片封面图")
if body.id:
card = await _get_owned_card(db, user, body.id, write=True)
card.title = title
card.content = content
card.target_url = target_url
card.image_path = image_path
card.updated_at = datetime.utcnow()
else:
slug = generate_slug()
for _ in range(5):
exists = (
await db.execute(select(LinkCardPage.id).where(LinkCardPage.slug == slug))
).scalar_one_or_none()
if not exists:
break
slug = generate_slug()
else:
raise HTTPException(status_code=500, detail="生成页面标识失败,请重试")
card = LinkCardPage(
owner_id=user.id,
slug=slug,
title=title,
content=content,
target_url=target_url,
image_path=image_path,
)
db.add(card)
await db.commit()
await db.refresh(card)
return _card_response(card, request)
@router.get("/p/{slug}", response_class=HTMLResponse, include_in_schema=False)
async def serve_link_card_page(slug: str, request: Request, db: AsyncSession = Depends(get_db)):
if not is_valid_slug(slug):
raise HTTPException(status_code=404, detail="页面不存在")
stmt = select(LinkCardPage).where(LinkCardPage.slug == slug)
card = (await db.execute(stmt)).scalar_one_or_none()
if not card:
raise HTTPException(status_code=404, detail="页面不存在")
cover_url = absolute_media_url(card.image_path, request)
favicon_path = favicon_path_for_cover(card.image_path)
favicon_disk = media_path_to_disk(UPLOAD_DIR, favicon_path)
favicon_url = absolute_media_url(favicon_path, request) if favicon_disk and os.path.isfile(favicon_disk) else cover_url
html = render_link_card_page(
title=card.title,
content=card.content or "",
keywords=build_keywords(card.title, card.content or ""),
cover_url=cover_url,
favicon_url=favicon_url,
target_url=card.target_url,
)
return HTMLResponse(content=html, media_type="text/html; charset=utf-8")
+2425
View File
File diff suppressed because it is too large Load Diff
+28
View File
@@ -0,0 +1,28 @@
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import declarative_base, sessionmaker
from models.db_config import (
build_database_url,
create_database_engine,
read_database_config,
)
_config = read_database_config()
DATABASE_URL = build_database_url(_config)
engine = create_database_engine(_config)
AsyncSessionLocal = sessionmaker(
engine, class_=AsyncSession, expire_on_commit=False
)
Base = declarative_base()
async def get_db():
async with AsyncSessionLocal() as session:
try:
yield session
await session.commit()
except Exception:
await session.rollback()
raise
finally:
await session.close()
+252
View File
@@ -0,0 +1,252 @@
"""数据库连接配置:支持 SQLite / MySQL / PostgreSQL。"""
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from urllib.parse import quote_plus
from dotenv import load_dotenv
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
BACKEND_DIR = Path(__file__).resolve().parent.parent
PROJECT_ROOT = BACKEND_DIR.parent
ENV_FILE = PROJECT_ROOT / ".env"
load_dotenv(ENV_FILE)
SUPPORTED_DB_TYPES = ("sqlite", "mysql", "postgresql")
PASSWORD_PLACEHOLDER = "******"
DEFAULT_SQLITE_PATH = BACKEND_DIR / "kefu.db"
def _env(name: str, default: str = "") -> str:
return (os.getenv(name) or default).strip()
def _env_int(name: str, default: int) -> int:
raw = _env(name)
if not raw:
return default
try:
return int(raw)
except ValueError:
return default
def normalize_db_type(value: str | None) -> str:
raw = (value or "sqlite").strip().lower()
if raw in ("postgres", "pgsql"):
return "postgresql"
if raw in SUPPORTED_DB_TYPES:
return raw
return "sqlite"
@dataclass
class DatabaseConfig:
db_type: str = "sqlite"
db_host: str = "127.0.0.1"
db_port: int = 0
db_user: str = ""
db_password: str = ""
db_name: str = "kefu"
db_path: str = ""
database_url: str = ""
def normalized_type(self) -> str:
return normalize_db_type(self.db_type)
def resolved_port(self) -> int:
if self.db_port:
return self.db_port
if self.normalized_type() == "mysql":
return 3306
if self.normalized_type() == "postgresql":
return 5432
return 0
def sqlite_path(self) -> Path:
if self.db_path:
path = Path(self.db_path)
if not path.is_absolute():
path = BACKEND_DIR / path
return path
if self.database_url and self.database_url.startswith("sqlite"):
# sqlite+aiosqlite:///path
raw = self.database_url.split("///", 1)[-1]
return Path(raw)
return DEFAULT_SQLITE_PATH
def read_database_config() -> DatabaseConfig:
explicit_url = _env("KEFU_DATABASE_URL")
db_type = normalize_db_type(_env("KEFU_DB_TYPE", "sqlite"))
db_path = _env("KEFU_DB_PATH")
if explicit_url and not _env("KEFU_DB_TYPE"):
lowered = explicit_url.lower()
if lowered.startswith("mysql"):
db_type = "mysql"
elif lowered.startswith("postgresql") or lowered.startswith("postgres"):
db_type = "postgresql"
elif lowered.startswith("sqlite"):
db_type = "sqlite"
return DatabaseConfig(
db_type=db_type,
db_host=_env("KEFU_DB_HOST", "127.0.0.1"),
db_port=_env_int("KEFU_DB_PORT", 0),
db_user=_env("KEFU_DB_USER"),
db_password=_env("KEFU_DB_PASSWORD"),
db_name=_env("KEFU_DB_NAME", "kefu"),
db_path=db_path,
database_url=explicit_url,
)
def build_database_url(config: DatabaseConfig | None = None) -> str:
cfg = config or read_database_config()
if cfg.database_url:
return cfg.database_url
db_type = cfg.normalized_type()
if db_type == "sqlite":
path = cfg.sqlite_path()
path.parent.mkdir(parents=True, exist_ok=True)
return f"sqlite+aiosqlite:///{path.as_posix()}"
user = quote_plus(cfg.db_user or "")
password = quote_plus(cfg.db_password or "")
host = cfg.db_host or "127.0.0.1"
port = cfg.resolved_port()
db_name = cfg.db_name or "kefu"
if db_type == "mysql":
auth = f"{user}:{password}@" if user else ""
return (
f"mysql+asyncmy://{auth}{host}:{port}/{db_name}"
"?charset=utf8mb4"
)
auth = f"{user}:{password}@" if user else ""
return f"postgresql+asyncpg://{auth}{host}:{port}/{db_name}"
def engine_kwargs_for_url(url: str) -> dict[str, Any]:
kwargs: dict[str, Any] = {"echo": False}
if not url.startswith("sqlite"):
kwargs["pool_pre_ping"] = True
kwargs["pool_recycle"] = 3600
# 默认连接池仅 pool_size=5 + max_overflow=10。多账号托管 + 前端并发请求时
# 容易耗尽连接导致请求阻塞/失败,这里放大连接池(可用环境变量覆盖)。
try:
_pool = int(os.getenv("KEFU_DB_POOL_SIZE", "20") or 20)
except ValueError:
_pool = 20
try:
_overflow = int(os.getenv("KEFU_DB_MAX_OVERFLOW", "40") or 40)
except ValueError:
_overflow = 40
kwargs["pool_size"] = max(5, _pool)
kwargs["max_overflow"] = max(0, _overflow)
kwargs["pool_timeout"] = 30
return kwargs
def create_database_engine(config: DatabaseConfig | None = None) -> AsyncEngine:
url = build_database_url(config)
return create_async_engine(url, **engine_kwargs_for_url(url))
def mask_database_url(url: str) -> str:
if "@" not in url or "://" not in url:
return url
scheme, rest = url.split("://", 1)
if "@" not in rest:
return url
creds, host_part = rest.rsplit("@", 1)
if ":" in creds:
user = creds.split(":", 1)[0]
return f"{scheme}://{user}:{PASSWORD_PLACEHOLDER}@{host_part}"
return f"{scheme}://{PASSWORD_PLACEHOLDER}@{host_part}"
def database_config_to_response(cfg: DatabaseConfig | None = None) -> dict[str, Any]:
cfg = cfg or read_database_config()
url = build_database_url(cfg)
return {
"db_type": cfg.normalized_type(),
"db_host": cfg.db_host,
"db_port": cfg.resolved_port(),
"db_user": cfg.db_user,
"db_name": cfg.db_name,
"db_path": str(cfg.sqlite_path()) if cfg.normalized_type() == "sqlite" else "",
"db_password": PASSWORD_PLACEHOLDER if cfg.db_password else "",
"db_password_configured": bool(cfg.db_password),
"database_url_display": mask_database_url(url),
"supported_types": list(SUPPORTED_DB_TYPES),
}
def persist_database_config(payload: dict[str, Any]) -> None:
"""将数据库配置写入项目根 .env 文件。"""
from dotenv import set_key
if not ENV_FILE.exists():
ENV_FILE.write_text("", encoding="utf-8")
db_type = normalize_db_type(payload.get("db_type"))
env_map = {
"KEFU_DB_TYPE": db_type,
"KEFU_DB_HOST": (payload.get("db_host") or "127.0.0.1").strip(),
"KEFU_DB_PORT": str(payload.get("db_port") or ""),
"KEFU_DB_USER": (payload.get("db_user") or "").strip(),
"KEFU_DB_NAME": (payload.get("db_name") or "kefu").strip(),
"KEFU_DB_PATH": (payload.get("db_path") or "").strip(),
}
password = payload.get("db_password")
if password and str(password).strip() not in ("", PASSWORD_PLACEHOLDER):
env_map["KEFU_DB_PASSWORD"] = str(password).strip()
# 清除完整 URL,避免与分项配置冲突
set_key(str(ENV_FILE), "KEFU_DATABASE_URL", "")
for key, value in env_map.items():
set_key(str(ENV_FILE), key, value or "")
# 让当前进程也能读到新值(重启后才会重建 engine)
for key, value in env_map.items():
os.environ[key] = value or ""
os.environ.pop("KEFU_DATABASE_URL", None)
async def test_database_connection(payload: dict[str, Any]) -> None:
cfg = DatabaseConfig(
db_type=normalize_db_type(payload.get("db_type")),
db_host=(payload.get("db_host") or "127.0.0.1").strip(),
db_port=int(payload.get("db_port") or 0),
db_user=(payload.get("db_user") or "").strip(),
db_password=(payload.get("db_password") or "").strip(),
db_name=(payload.get("db_name") or "kefu").strip(),
db_path=(payload.get("db_path") or "").strip(),
)
current = read_database_config()
if cfg.db_password in ("", PASSWORD_PLACEHOLDER):
cfg.db_password = current.db_password
url = build_database_url(cfg)
engine = create_async_engine(url, **engine_kwargs_for_url(url))
try:
async with engine.connect() as conn:
await conn.execute(text("SELECT 1"))
except ModuleNotFoundError as exc:
driver = "asyncmy" if cfg.normalized_type() == "mysql" else "asyncpg"
raise RuntimeError(
f"缺少数据库驱动,请执行: pip install {driver}"
) from exc
finally:
await engine.dispose()
+223
View File
@@ -0,0 +1,223 @@
"""跨数据库兼容的轻量 schema 迁移。"""
from __future__ import annotations
from sqlalchemy import inspect, text
def _table_columns(conn, table: str) -> set[str]:
try:
insp = inspect(conn)
return {col["name"] for col in insp.get_columns(table)}
except Exception:
return set()
def _dialect(conn) -> str:
return conn.dialect.name
def _bool_default(conn, value: bool = True) -> str:
if _dialect(conn) == "postgresql":
return "TRUE" if value else "FALSE"
return "1" if value else "0"
def add_column_if_missing(conn, table: str, column: str, ddl_by_dialect: dict[str, str]) -> None:
cols = _table_columns(conn, table)
if not cols or column in cols:
return
dialect = _dialect(conn)
ddl = ddl_by_dialect.get(dialect) or ddl_by_dialect.get("default")
if ddl:
conn.execute(text(ddl))
def migrate_accounts_table(conn) -> None:
add_column_if_missing(
conn,
"accounts",
"cookie_data",
{"default": "ALTER TABLE accounts ADD COLUMN cookie_data TEXT"},
)
add_column_if_missing(
conn,
"accounts",
"cookie_updated_at",
{
"default": "ALTER TABLE accounts ADD COLUMN cookie_updated_at DATETIME",
"postgresql": "ALTER TABLE accounts ADD COLUMN cookie_updated_at TIMESTAMP",
},
)
add_column_if_missing(
conn,
"accounts",
"im_session_data",
{"default": "ALTER TABLE accounts ADD COLUMN im_session_data TEXT"},
)
add_column_if_missing(
conn,
"accounts",
"reply_delay_seconds",
{"default": "ALTER TABLE accounts ADD COLUMN reply_delay_seconds INTEGER DEFAULT 0"},
)
add_column_if_missing(
conn,
"accounts",
"reply_cooldown_seconds",
{"default": "ALTER TABLE accounts ADD COLUMN reply_cooldown_seconds INTEGER"},
)
add_column_if_missing(
conn,
"accounts",
"follow_welcome_enabled",
{
"default": "ALTER TABLE accounts ADD COLUMN follow_welcome_enabled BOOLEAN DEFAULT 0",
"postgresql": "ALTER TABLE accounts ADD COLUMN follow_welcome_enabled BOOLEAN DEFAULT FALSE",
},
)
add_column_if_missing(
conn,
"accounts",
"follow_welcome_content",
{"default": "ALTER TABLE accounts ADD COLUMN follow_welcome_content TEXT"},
)
add_column_if_missing(
conn,
"accounts",
"owner_id",
{"default": "ALTER TABLE accounts ADD COLUMN owner_id INTEGER"},
)
add_column_if_missing(
conn,
"accounts",
"user_agent",
{"default": "ALTER TABLE accounts ADD COLUMN user_agent TEXT"},
)
add_column_if_missing(
conn,
"accounts",
"avatar_url",
{"default": "ALTER TABLE accounts ADD COLUMN avatar_url TEXT"},
)
add_column_if_missing(
conn,
"accounts",
"douyin_uid",
{"default": "ALTER TABLE accounts ADD COLUMN douyin_uid VARCHAR(64)"},
)
def migrate_account_videos_table(conn) -> None:
add_column_if_missing(
conn,
"account_videos",
"media_type",
{"default": "ALTER TABLE account_videos ADD COLUMN media_type VARCHAR(20)"},
)
def migrate_message_logs_table(conn) -> None:
add_column_if_missing(
conn,
"message_logs",
"sender_avatar",
{"default": "ALTER TABLE message_logs ADD COLUMN sender_avatar TEXT"},
)
def migrate_rules_table(conn) -> None:
add_column_if_missing(
conn,
"rules",
"owner_id",
{"default": "ALTER TABLE rules ADD COLUMN owner_id INTEGER"},
)
add_column_if_missing(
conn,
"rules",
"sort_order",
{"default": "ALTER TABLE rules ADD COLUMN sort_order INTEGER DEFAULT 0"},
)
cols = _table_columns(conn, "rules")
if cols and "sort_order" in cols:
conn.execute(
text("UPDATE rules SET sort_order = id WHERE sort_order IS NULL OR sort_order = 0")
)
def migrate_users_table(conn) -> None:
add_column_if_missing(
conn,
"users",
"email",
{"default": "ALTER TABLE users ADD COLUMN email VARCHAR(255)"},
)
add_column_if_missing(
conn,
"users",
"email_verified",
{
"default": "ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT 0",
"postgresql": "ALTER TABLE users ADD COLUMN email_verified BOOLEAN DEFAULT FALSE",
},
)
add_column_if_missing(
conn,
"users",
"email_verified_at",
{
"default": "ALTER TABLE users ADD COLUMN email_verified_at DATETIME",
"postgresql": "ALTER TABLE users ADD COLUMN email_verified_at TIMESTAMP",
},
)
add_column_if_missing(
conn,
"users",
"max_accounts",
{"default": "ALTER TABLE users ADD COLUMN max_accounts INTEGER DEFAULT 3"},
)
cols = _table_columns(conn, "users")
if not cols:
return
verified = _bool_default(conn, True)
conn.execute(text(f"UPDATE users SET email_verified = {verified} WHERE email_verified IS NULL"))
conn.execute(text("UPDATE users SET max_accounts = -1 WHERE role = 'admin'"))
conn.execute(text("UPDATE users SET max_accounts = 3 WHERE max_accounts IS NULL"))
def migrate_payment_orders_table(conn) -> None:
cols = _table_columns(conn, "payment_orders")
if not cols:
return
add_column_if_missing(
conn,
"payment_orders",
"slots_applied",
{
"default": "ALTER TABLE payment_orders ADD COLUMN slots_applied BOOLEAN DEFAULT 0",
"postgresql": "ALTER TABLE payment_orders ADD COLUMN slots_applied BOOLEAN DEFAULT FALSE",
},
)
applied = _bool_default(conn, True)
conn.execute(
text(
"UPDATE payment_orders SET slots_applied = "
f"{applied} WHERE status = 'paid' AND (slots_applied IS NULL OR slots_applied = 0)"
)
)
def migrate_accounts_quota_disabled(conn) -> None:
cols = _table_columns(conn, "accounts")
if not cols:
return
add_column_if_missing(
conn,
"accounts",
"quota_disabled",
{
"default": "ALTER TABLE accounts ADD COLUMN quota_disabled BOOLEAN DEFAULT 0",
"postgresql": "ALTER TABLE accounts ADD COLUMN quota_disabled BOOLEAN DEFAULT FALSE",
},
)
+323
View File
@@ -0,0 +1,323 @@
"""从 SQLite 源库迁移数据到目标数据库。"""
from __future__ import annotations
from datetime import date, datetime
from pathlib import Path
from typing import Any
from sqlalchemy import MetaData, insert, inspect, text
from sqlalchemy.ext.asyncio import AsyncEngine
from models.database import Base
import models.models # noqa: F401 — 注册 ORM 表结构
from models.db_config import (
PASSWORD_PLACEHOLDER,
DatabaseConfig,
build_database_url,
create_database_engine,
mask_database_url,
read_database_config,
)
from models.db_migrate import (
migrate_accounts_quota_disabled,
migrate_accounts_table,
migrate_message_logs_table,
migrate_payment_orders_table,
migrate_rules_table,
migrate_users_table,
)
# 按外键依赖顺序导入
MIGRATION_TABLES: tuple[str, ...] = (
"users",
"email_verification_tokens",
"password_reset_tokens",
"app_config",
"accounts",
"rules",
"message_logs",
"received_message_logs",
"system_logs",
"payment_orders",
)
BATCH_SIZE = 400
def _parse_datetime(value: Any) -> Any:
if value is None or isinstance(value, (datetime, date)):
return value
if isinstance(value, str):
text_value = value.strip()
if not text_value:
return None
for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d"):
try:
return datetime.strptime(text_value, fmt)
except ValueError:
continue
try:
return datetime.fromisoformat(text_value.replace("Z", "+00:00"))
except ValueError:
return value
return value
def _normalize_rows(rows: list[dict[str, Any]], tbl) -> list[dict[str, Any]]:
if not rows:
return rows
col_types = {column.name: column.type for column in tbl.columns}
normalized: list[dict[str, Any]] = []
for row in rows:
item = dict(row)
for key, col_type in col_types.items():
if key not in item or item[key] is None:
continue
type_name = col_type.__class__.__name__.lower()
if "datetime" in type_name or "timestamp" in type_name:
item[key] = _parse_datetime(item[key])
elif "boolean" in type_name and not isinstance(item[key], bool):
if isinstance(item[key], (int, float)):
item[key] = bool(item[key])
elif isinstance(item[key], str):
item[key] = item[key].strip().lower() in ("1", "true", "t", "yes")
normalized.append(item)
return normalized
def resolve_sqlite_source_path(source_path: str | None = None) -> Path:
if source_path and str(source_path).strip():
path = Path(source_path.strip())
if not path.is_absolute():
from models.db_config import BACKEND_DIR
path = BACKEND_DIR / path
return path
from models.db_config import DEFAULT_SQLITE_PATH
return DEFAULT_SQLITE_PATH
def build_sqlite_config(path: Path) -> DatabaseConfig:
return DatabaseConfig(db_type="sqlite", db_path=str(path))
def _same_sqlite_target(source: Path, target: DatabaseConfig) -> bool:
if target.normalized_type() != "sqlite":
return False
try:
return source.resolve() == target.sqlite_path().resolve()
except OSError:
return str(source) == str(target.sqlite_path())
async def inspect_sqlite_source(source_path: str | None = None) -> dict[str, Any]:
path = resolve_sqlite_source_path(source_path)
if not path.exists():
return {
"source_path": str(path),
"exists": False,
"tables": {},
"total_rows": 0,
}
cfg = build_sqlite_config(path)
engine = create_database_engine(cfg)
tables: dict[str, int] = {}
total_rows = 0
try:
async with engine.connect() as conn:
for table in MIGRATION_TABLES:
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
tables[table] = 0
continue
count = await conn.scalar(text(f"SELECT COUNT(*) FROM {table}"))
row_count = int(count or 0)
tables[table] = row_count
total_rows += row_count
finally:
await engine.dispose()
return {
"source_path": str(path),
"exists": True,
"tables": tables,
"total_rows": total_rows,
}
async def _ensure_target_schema(engine: AsyncEngine) -> None:
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
await conn.run_sync(migrate_accounts_table)
await conn.run_sync(migrate_rules_table)
await conn.run_sync(migrate_message_logs_table)
await conn.run_sync(migrate_users_table)
await conn.run_sync(migrate_payment_orders_table)
await conn.run_sync(migrate_accounts_quota_disabled)
async def _count_target_rows(engine: AsyncEngine) -> dict[str, int]:
counts: dict[str, int] = {}
async with engine.connect() as conn:
for table in MIGRATION_TABLES:
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
counts[table] = 0
continue
count = await conn.scalar(text(f"SELECT COUNT(*) FROM {table}"))
counts[table] = int(count or 0)
return counts
async def _fetch_table_rows(engine: AsyncEngine, table: str) -> list[dict[str, Any]]:
async with engine.connect() as conn:
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
return []
result = await conn.execute(text(f"SELECT * FROM {table} ORDER BY id"))
return [dict(row) for row in result.mappings()]
async def _clear_target_tables(engine: AsyncEngine) -> None:
async with engine.begin() as conn:
dialect = conn.dialect.name
if dialect == "mysql":
await conn.execute(text("SET FOREIGN_KEY_CHECKS=0"))
for table in reversed(MIGRATION_TABLES):
if await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
await conn.execute(text(f"DELETE FROM {table}"))
await conn.execute(text("SET FOREIGN_KEY_CHECKS=1"))
elif dialect == "postgresql":
existing = []
for table in MIGRATION_TABLES:
if await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
existing.append(table)
if existing:
await conn.execute(
text(f"TRUNCATE {', '.join(existing)} RESTART IDENTITY CASCADE")
)
else:
await conn.execute(text("PRAGMA foreign_keys=OFF"))
for table in reversed(MIGRATION_TABLES):
if await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
await conn.execute(text(f"DELETE FROM {table}"))
await conn.execute(text("PRAGMA foreign_keys=ON"))
async def _insert_rows(engine: AsyncEngine, table: str, rows: list[dict[str, Any]]) -> int:
if not rows:
return 0
tbl = Base.metadata.tables.get(table)
if tbl is None:
metadata = MetaData()
def _reflect(sync_conn) -> None:
metadata.reflect(sync_conn, only=[table])
async with engine.connect() as conn:
await conn.run_sync(_reflect)
tbl = metadata.tables[table]
inserted = 0
prepared_rows = _normalize_rows(rows, tbl)
async with engine.begin() as conn:
for offset in range(0, len(prepared_rows), BATCH_SIZE):
batch = prepared_rows[offset : offset + BATCH_SIZE]
await conn.execute(insert(tbl), batch)
inserted += len(batch)
return inserted
async def _reset_auto_increment(engine: AsyncEngine) -> None:
async with engine.begin() as conn:
dialect = conn.dialect.name
for table in MIGRATION_TABLES:
if not await conn.run_sync(lambda sync_conn, t=table: inspect(sync_conn).has_table(t)):
continue
max_id = await conn.scalar(text(f"SELECT COALESCE(MAX(id), 0) FROM {table}"))
if not max_id:
continue
next_id = int(max_id) + 1
if dialect == "mysql":
await conn.execute(text(f"ALTER TABLE {table} AUTO_INCREMENT = {next_id}"))
elif dialect == "postgresql":
await conn.execute(
text(
"SELECT setval("
f"pg_get_serial_sequence('{table}', 'id'), "
f"{next_id}, false)"
)
)
def _build_target_config(payload: dict[str, Any]) -> DatabaseConfig:
current = read_database_config()
password = (payload.get("db_password") or "").strip()
if password in ("", PASSWORD_PLACEHOLDER):
password = current.db_password
return DatabaseConfig(
db_type=payload.get("db_type") or current.db_type,
db_host=(payload.get("db_host") or current.db_host or "127.0.0.1").strip(),
db_port=int(payload.get("db_port") or current.resolved_port() or 0),
db_user=(payload.get("db_user") or current.db_user).strip(),
db_password=password,
db_name=(payload.get("db_name") or current.db_name or "kefu").strip(),
db_path=(payload.get("db_path") or current.db_path).strip(),
)
async def migrate_sqlite_to_target(
payload: dict[str, Any],
*,
clear_target: bool = False,
) -> dict[str, Any]:
source_path = resolve_sqlite_source_path(payload.get("source_db_path"))
if not source_path.exists():
raise FileNotFoundError(f"SQLite 源文件不存在: {source_path}")
target_cfg = _build_target_config(payload)
if _same_sqlite_target(source_path, target_cfg):
raise ValueError("目标库不能与 SQLite 源文件相同,请指定不同的目标数据库")
source_cfg = build_sqlite_config(source_path)
source_engine = create_database_engine(source_cfg)
target_engine = create_database_engine(target_cfg)
try:
preview = await inspect_sqlite_source(str(source_path))
if preview["total_rows"] == 0:
raise ValueError("SQLite 源库中没有可迁移的数据")
await _ensure_target_schema(target_engine)
target_counts = await _count_target_rows(target_engine)
target_total = sum(target_counts.values())
if target_total > 0 and not clear_target:
raise ValueError(
f"目标库已有 {target_total} 条数据,请勾选「清空目标库后再导入」或先手动清空"
)
if clear_target and target_total > 0:
await _clear_target_tables(target_engine)
copied: dict[str, int] = {}
total_rows = 0
for table in MIGRATION_TABLES:
rows = await _fetch_table_rows(source_engine, table)
copied[table] = await _insert_rows(target_engine, table, rows)
total_rows += copied[table]
await _reset_auto_increment(target_engine)
finally:
await source_engine.dispose()
await target_engine.dispose()
target_type = target_cfg.normalized_type()
return {
"message": f"已从 SQLite 迁移 {total_rows} 条记录到 {target_type} 数据库",
"source_path": str(source_path),
"target_type": target_type,
"target_url": mask_database_url(build_database_url(target_cfg)),
"tables": copied,
"total_rows": total_rows,
}
+278
View File
@@ -0,0 +1,278 @@
from datetime import datetime
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, Text, UniqueConstraint
from sqlalchemy.orm import relationship
from .database import Base
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True, index=True)
username = Column(String(50), unique=True, index=True, nullable=False)
email = Column(String(255), unique=True, index=True, nullable=True)
password_hash = Column(String(255), nullable=False)
display_name = Column(String(100), nullable=True)
role = Column(String(20), default="operator", index=True) # admin, operator, viewer
is_active = Column(Boolean, default=True)
email_verified = Column(Boolean, default=False)
email_verified_at = Column(DateTime, nullable=True)
max_accounts = Column(Integer, default=3)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
accounts = relationship("Account", back_populates="owner")
verification_tokens = relationship(
"EmailVerificationToken",
back_populates="user",
cascade="all, delete-orphan",
)
password_reset_tokens = relationship(
"PasswordResetToken",
back_populates="user",
cascade="all, delete-orphan",
)
class EmailVerificationToken(Base):
__tablename__ = "email_verification_tokens"
id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
token = Column(String(128), unique=True, index=True, nullable=False)
expires_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, default=datetime.utcnow)
user = relationship("User", back_populates="verification_tokens")
class PasswordResetToken(Base):
__tablename__ = "password_reset_tokens"
id = Column(Integer, primary_key=True, index=True)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
token = Column(String(128), unique=True, index=True, nullable=False)
expires_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, default=datetime.utcnow)
user = relationship("User", back_populates="password_reset_tokens")
class AppConfig(Base):
"""系统功能配置(单例行 id=1,JSON 存储)。"""
__tablename__ = "app_config"
id = Column(Integer, primary_key=True)
data = Column(Text, nullable=False, default="{}")
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class Account(Base):
__tablename__ = "accounts"
id = Column(Integer, primary_key=True, index=True)
owner_id = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
username = Column(String(100), unique=True, index=True, nullable=True) # 登录后获取的用户名/ID
avatar_url = Column(String(512), nullable=True) # 登录后获取的头像 URL
douyin_uid = Column(String(64), nullable=True, index=True) # 抖音 UID
phone = Column(String(20), nullable=True) # 绑定的手机号(可选)
status = Column(String(50), default="offline") # offline, logging_in, online, error
cookie_path = Column(String(255), nullable=True) # 存储 cookie/session 的路径
cookie_data = Column(Text, nullable=True) # Playwright storage_state JSON
cookie_updated_at = Column(DateTime, nullable=True) # Cookie 最近更新时间
im_session_data = Column(Text, nullable=True) # IM 直连会话 (WS URL / device_id 等)
reply_delay_seconds = Column(Integer, default=0) # 账号回复排队间隔;0/NULL=继承系统默认
reply_cooldown_seconds = Column(Integer, nullable=True) # 自动回复冷却秒数;NULL=继承全局设置
follow_welcome_enabled = Column(Boolean, default=False) # 新粉丝关注后自动发送欢迎语
follow_welcome_content = Column(Text, nullable=True) # 关注欢迎语内容(空=不发)
user_agent = Column(Text, nullable=True) # 伪装设备头(User-Agent),空=默认
qr_code_base64 = Column(Text, nullable=True) # 当前登录二维码的 base64 字符串
error_message = Column(Text, nullable=True) # 错误信息
quota_disabled = Column(Boolean, default=False, index=True) # 额度不足被停用
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# 关联
owner = relationship("User", back_populates="accounts")
rules = relationship("AutoReplyRule", back_populates="account", cascade="all, delete-orphan")
logs = relationship("MessageLog", back_populates="account", cascade="all, delete-orphan")
profile_detail = relationship(
"AccountProfileDetail",
back_populates="account",
uselist=False,
cascade="all, delete-orphan",
)
videos = relationship(
"AccountVideo",
back_populates="account",
cascade="all, delete-orphan",
order_by="AccountVideo.sort_order",
)
class AccountProfileDetail(Base):
"""托管账号抖音详细资料(本地缓存)。"""
__tablename__ = "account_profile_details"
id = Column(Integer, primary_key=True, index=True)
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), unique=True, nullable=False, index=True)
uid = Column(String(64), nullable=True)
nickname = Column(String(200), nullable=True)
avatar_url = Column(String(512), nullable=True)
unique_id = Column(String(100), nullable=True)
signature = Column(Text, nullable=True)
sec_user_id = Column(String(255), nullable=True)
video_count = Column(Integer, nullable=True)
follower_count = Column(Integer, nullable=True)
following_count = Column(Integer, nullable=True)
total_favorited = Column(Integer, nullable=True)
favoriting_count = Column(Integer, nullable=True)
sync_message = Column(Text, nullable=True)
synced_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
account = relationship("Account", back_populates="profile_detail")
class AccountVideo(Base):
"""托管账号已发布作品(本地缓存)。"""
__tablename__ = "account_videos"
id = Column(Integer, primary_key=True, index=True)
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True)
aweme_id = Column(String(64), nullable=False, index=True)
title = Column(Text, nullable=True)
cover_url = Column(String(1024), nullable=True)
video_url = Column(String(2048), nullable=True)
share_url = Column(String(1024), nullable=True)
create_time = Column(DateTime, nullable=True)
digg_count = Column(Integer, nullable=True)
comment_count = Column(Integer, nullable=True)
play_count = Column(Integer, nullable=True)
media_type = Column(String(20), nullable=True) # video, image, other
sort_order = Column(Integer, default=0, index=True)
synced_at = Column(DateTime, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
account = relationship("Account", back_populates="videos")
class LinkCardPage(Base):
"""规则回复用的链接卡片落地页(带 SEO meta,打开后跳转至目标 URL)。"""
__tablename__ = "link_card_pages"
id = Column(Integer, primary_key=True, index=True)
owner_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
slug = Column(String(64), unique=True, index=True, nullable=False)
title = Column(String(200), nullable=False)
content = Column(Text, nullable=True)
target_url = Column(String(2000), nullable=False)
image_path = Column(String(512), nullable=False)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class AutoReplyRule(Base):
__tablename__ = "rules"
id = Column(Integer, primary_key=True, index=True)
owner_id = Column(Integer, ForeignKey("users.id", ondelete="SET NULL"), nullable=True, index=True)
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=True) # 为空代表全局规则
keyword = Column(String(255), index=True) # 触发关键词,或者空代表兜底
reply_content = Column(Text) # 回复内容
match_type = Column(String(50), default="contains") # exact (精确), contains (包含), regex (正则), default (兜底)
sort_order = Column(Integer, default=0, index=True) # 规则优先级,越小越优先
is_active = Column(Boolean, default=True)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
account = relationship("Account", back_populates="rules")
class MessageLog(Base):
__tablename__ = "message_logs"
id = Column(Integer, primary_key=True, index=True)
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"))
sender_name = Column(String(100)) # 发送者名字
sender_id = Column(String(100), nullable=True) # 发送者唯一ID
sender_avatar = Column(String(512), nullable=True) # 发送者头像 URL
message_content = Column(Text) # 接收到的消息
reply_content = Column(Text, nullable=True) # 回复的消息
status = Column(String(50), default="received") # received, replied, ignored, failed
error_message = Column(Text, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
account = relationship("Account", back_populates="logs")
class ReceivedMessageLog(Base):
"""接收消息原始日志:仅记录收到的消息,内容原样保存。"""
__tablename__ = "received_message_logs"
id = Column(Integer, primary_key=True, index=True)
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True)
conversation_id = Column(String(128), nullable=True, index=True)
sender_id = Column(String(100), nullable=True, index=True)
sender_name = Column(String(100), nullable=True)
sender_avatar = Column(String(512), nullable=True)
message_type = Column(Integer, nullable=True)
server_message_id = Column(String(64), nullable=True, index=True)
raw_content = Column(Text, nullable=False)
created_at = Column(DateTime, default=datetime.utcnow, index=True)
class FollowWelcomeLog(Base):
"""关注欢迎语去重表:每个账号对每个新粉丝只发送一次欢迎语(重启后仍生效)。"""
__tablename__ = "follow_welcome_logs"
id = Column(Integer, primary_key=True, index=True)
account_id = Column(Integer, ForeignKey("accounts.id", ondelete="CASCADE"), nullable=False, index=True)
follower_uid = Column(String(64), nullable=False, index=True) # 新粉丝的抖音 UID
status = Column(String(20), default="sent") # sent, failed
detail = Column(Text, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
__table_args__ = (
UniqueConstraint("account_id", "follower_uid", name="uq_follow_welcome_account_follower"),
)
class SystemLog(Base):
"""系统诊断日志:记录私信收发/连接/鉴权等链路事件,便于排查失败原因。"""
__tablename__ = "system_logs"
id = Column(Integer, primary_key=True, index=True)
account_id = Column(Integer, nullable=True, index=True) # 关联账号(可空,全局事件)
level = Column(String(20), default="info", index=True) # info, success, warning, error
category = Column(String(40), default="system", index=True) # ws, send, recv, auth, poll, system
event = Column(String(255)) # 事件标题
detail = Column(Text, nullable=True) # 详细原因
created_at = Column(DateTime, default=datetime.utcnow, index=True)
class PaymentOrder(Base):
"""账号额度购买订单。"""
__tablename__ = "payment_orders"
id = Column(Integer, primary_key=True, index=True)
order_no = Column(String(64), unique=True, index=True, nullable=False)
user_id = Column(Integer, ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True)
channel = Column(String(20), nullable=False) # wechat, alipay
slots = Column(Integer, nullable=False)
amount_fen = Column(Integer, nullable=False)
status = Column(String(20), default="pending", index=True) # pending, paid, expired, cancelled, refunded
slots_applied = Column(Boolean, default=False)
qr_code = Column(Text, nullable=True)
pay_url = Column(Text, nullable=True)
trade_no = Column(String(128), nullable=True)
notify_payload = Column(Text, nullable=True)
paid_at = Column(DateTime, nullable=True)
expires_at = Column(DateTime, nullable=False)
created_at = Column(DateTime, default=datetime.utcnow)
+1
View File
@@ -0,0 +1 @@
+123
View File
@@ -0,0 +1,123 @@
"""支付宝当面付扫码(precreate)。"""
from __future__ import annotations
import json
import logging
from base64 import b64decode, b64encode
from datetime import datetime
from typing import Any
from urllib.parse import quote_plus, urlencode
import httpx
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import padding
from auth.system_settings import SystemSettingsData
logger = logging.getLogger("payments.alipay")
ALIPAY_GATEWAY = "https://openapi.alipay.com/gateway.do"
ALIPAY_SANDBOX_GATEWAY = "https://openapi-sandbox.dl.alipaydev.com/gateway.do"
def _normalize_pem(text: str, label: str) -> str:
body = text.strip()
if "BEGIN" in body:
return body
return f"-----BEGIN {label}-----\n{body}\n-----END {label}-----"
def _load_private_key(pem: str):
return serialization.load_pem_private_key(
_normalize_pem(pem, "RSA PRIVATE KEY").encode("utf-8"),
password=None,
)
def _load_public_key(pem: str):
return serialization.load_pem_public_key(
_normalize_pem(pem, "PUBLIC KEY").encode("utf-8"),
)
def _sign_content(content: str, private_key) -> str:
signature = private_key.sign(
content.encode("utf-8"),
padding.PKCS1v15(),
hashes.SHA256(),
)
return b64encode(signature).decode("utf-8")
def _gateway(settings: SystemSettingsData) -> str:
return ALIPAY_SANDBOX_GATEWAY if settings.alipay_sandbox else ALIPAY_GATEWAY
def create_precreate_order(
settings: SystemSettingsData,
order_no: str,
subject: str,
amount_yuan: str,
notify_url: str,
) -> str:
"""创建当面付预下单,返回 qr_code 字符串。"""
private_key = _load_private_key(settings.alipay_private_key)
biz_content = {
"out_trade_no": order_no,
"total_amount": amount_yuan,
"subject": subject[:256],
}
params = {
"app_id": settings.alipay_app_id.strip(),
"method": "alipay.trade.precreate",
"format": "JSON",
"charset": "utf-8",
"sign_type": "RSA2",
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"version": "1.0",
"notify_url": notify_url,
"biz_content": json.dumps(biz_content, ensure_ascii=False, separators=(",", ":")),
}
unsigned = "&".join(
f"{k}={quote_plus(str(v), safe='')}"
for k, v in sorted(params.items())
if v is not None and str(v) != ""
)
params["sign"] = _sign_content(unsigned, private_key)
with httpx.Client(timeout=30.0) as client:
resp = client.post(_gateway(settings), data=params)
resp.raise_for_status()
data = resp.json()
key = "alipay_trade_precreate_response"
result = data.get(key) or {}
if result.get("code") != "10000":
raise RuntimeError(result.get("sub_msg") or result.get("msg") or "支付宝下单失败")
qr_code = result.get("qr_code")
if not qr_code:
raise RuntimeError("支付宝未返回 qr_code")
return qr_code
def verify_notify(params: dict[str, Any], settings: SystemSettingsData) -> bool:
sign = params.get("sign")
if not sign:
return False
verify_params = {k: v for k, v in params.items() if k not in ("sign", "sign_type") and v is not None}
unsigned = "&".join(
f"{k}={v}"
for k, v in sorted(verify_params.items())
if str(v) != ""
)
try:
public_key = _load_public_key(settings.alipay_public_key)
public_key.verify(
b64decode(sign),
unsigned.encode("utf-8"),
padding.PKCS1v15(),
hashes.SHA256(),
)
return True
except Exception as exc:
logger.warning("Alipay notify verify failed: %s", exc)
return False
+125
View File
@@ -0,0 +1,125 @@
from fastapi import APIRouter, Depends, HTTPException, Query, Request
from fastapi.responses import PlainTextResponse
from sqlalchemy.ext.asyncio import AsyncSession
from auth.dependencies import get_current_user, require_admin
from auth.system_settings import load_settings
from models.database import get_db
from models.models import User
from . import service
from .schemas import (
AdminUpdateOrderStatusRequest,
CreatePaymentOrderRequest,
MessageResponse,
PaymentConfigResponse,
PaymentOrderListItem,
PaymentOrderListResponse,
PaymentOrderResponse,
)
router = APIRouter(prefix="/api/payments", tags=["payments"])
@router.get("/config", response_model=PaymentConfigResponse)
async def get_payment_config(
db: AsyncSession = Depends(get_db),
_: User = Depends(get_current_user),
):
settings = await load_settings(db)
return PaymentConfigResponse(**service.payment_config_payload(settings))
@router.post("/orders", response_model=PaymentOrderResponse)
async def create_payment_order(
body: CreatePaymentOrderRequest,
db: AsyncSession = Depends(get_db),
user: User = Depends(get_current_user),
):
order, demo_mode = await service.create_order(db, user, body.slots, body.channel)
return PaymentOrderResponse(**service.order_to_dict(order, demo_mode=demo_mode))
@router.get("/orders", response_model=PaymentOrderListResponse)
async def list_payment_orders(
status: str | None = Query(default=None, description="pending/paid/expired/cancelled"),
channel: str | None = Query(default=None, description="wechat/alipay"),
page: int = Query(default=1, ge=1),
page_size: int = Query(default=20, ge=1, le=100),
db: AsyncSession = Depends(get_db),
user: User = Depends(get_current_user),
):
if status and status not in service.ORDER_STATUSES:
raise HTTPException(status_code=400, detail="无效的订单状态")
data = await service.list_orders(
db,
user,
status=status,
channel=channel,
page=page,
page_size=page_size,
)
return PaymentOrderListResponse(**data)
@router.get("/orders/{order_no}", response_model=PaymentOrderResponse)
async def get_payment_order(
order_no: str,
db: AsyncSession = Depends(get_db),
user: User = Depends(get_current_user),
):
settings = await load_settings(db)
order = await service.get_user_order(db, user, order_no)
demo_mode = settings.payment_demo_mode and not settings.payment_channel_available(order.channel)
return PaymentOrderResponse(**service.order_to_dict(order, demo_mode=demo_mode))
@router.post("/orders/{order_no}/simulate", response_model=PaymentOrderResponse)
async def simulate_payment_order(
order_no: str,
db: AsyncSession = Depends(get_db),
user: User = Depends(get_current_user),
):
settings = await load_settings(db)
order = await service.simulate_pay(db, user, order_no)
return PaymentOrderResponse(**service.order_to_dict(order, demo_mode=settings.payment_demo_mode))
@router.put("/orders/{order_no}/status", response_model=PaymentOrderListItem)
async def admin_update_payment_order_status(
order_no: str,
body: AdminUpdateOrderStatusRequest,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
order = await service.admin_update_order_status(db, order_no, body.status)
return PaymentOrderListItem(**order)
@router.delete("/orders/{order_no}", response_model=MessageResponse)
async def admin_delete_payment_order(
order_no: str,
db: AsyncSession = Depends(get_db),
_: User = Depends(require_admin),
):
await service.admin_delete_order(db, order_no)
return MessageResponse(message="订单已删除")
@router.post("/notify/wechat")
async def wechat_payment_notify(request: Request, db: AsyncSession = Depends(get_db)):
body = await request.body()
headers = {k.lower(): v for k, v in request.headers.items()}
ok = await service.handle_wechat_notify(db, body, headers)
if not ok:
return PlainTextResponse("FAIL", status_code=400)
return PlainTextResponse('{"code": "SUCCESS", "message": "成功"}', media_type="application/json")
@router.post("/notify/alipay")
async def alipay_payment_notify(request: Request, db: AsyncSession = Depends(get_db)):
form = await request.form()
params = {k: v for k, v in form.items()}
ok = await service.handle_alipay_notify(db, params)
if not ok:
return PlainTextResponse("fail")
return PlainTextResponse("success")
+69
View File
@@ -0,0 +1,69 @@
from datetime import datetime
from typing import Literal, Optional
from pydantic import BaseModel, Field
class PaymentConfigResponse(BaseModel):
payment_enabled: bool
demo_mode: bool
unit_price: float
min_slots: int
max_slots: int
channels: list[str]
wechat_available: bool
alipay_available: bool
class CreatePaymentOrderRequest(BaseModel):
slots: int = Field(ge=1, le=100)
channel: Literal["wechat", "alipay"]
class PaymentOrderResponse(BaseModel):
order_no: str
channel: str
slots: int
amount_fen: int
amount_yuan: float
status: str
qr_code: Optional[str] = None
pay_url: Optional[str] = None
paid_at: Optional[datetime] = None
expires_at: Optional[datetime] = None
created_at: Optional[datetime] = None
demo_mode: bool = False
class PaymentOrderListItem(BaseModel):
id: int
order_no: str
user_id: int
username: Optional[str] = None
display_name: Optional[str] = None
channel: str
slots: int
amount_fen: int
amount_yuan: float
status: str
trade_no: Optional[str] = None
is_demo: bool = False
paid_at: Optional[datetime] = None
created_at: datetime
class PaymentOrderListResponse(BaseModel):
items: list[PaymentOrderListItem]
total: int
page: int
page_size: int
paid_count: int = 0
paid_amount_yuan: float = 0
class AdminUpdateOrderStatusRequest(BaseModel):
status: Literal["pending", "paid", "expired", "cancelled", "refunded"]
class MessageResponse(BaseModel):
message: str
+432
View File
@@ -0,0 +1,432 @@
"""支付订单业务逻辑。"""
from __future__ import annotations
import logging
import secrets
from datetime import datetime, timedelta
from typing import Any
from fastapi import HTTPException
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession
from auth.account_quota import default_stop_worker, sync_user_account_quota
from auth.roles import is_admin
from auth.system_settings import SystemSettingsData, load_settings
from models.models import PaymentOrder, User
from . import alipay, wechat
logger = logging.getLogger("payments.service")
ORDER_TTL_MINUTES = 30
ORDER_STATUSES = ("pending", "paid", "expired", "cancelled", "refunded")
def generate_order_no() -> str:
return f"AC{datetime.utcnow().strftime('%Y%m%d%H%M%S')}{secrets.token_hex(4).upper()}"
def yuan_to_fen(yuan: float) -> int:
return int(round(yuan * 100))
def fen_to_yuan(fen: int) -> float:
return round(fen / 100, 2)
def order_to_dict(order: PaymentOrder, demo_mode: bool = False) -> dict[str, Any]:
return {
"order_no": order.order_no,
"channel": order.channel,
"slots": order.slots,
"amount_fen": order.amount_fen,
"amount_yuan": fen_to_yuan(order.amount_fen),
"status": order.status,
"qr_code": order.qr_code,
"pay_url": order.pay_url,
"paid_at": order.paid_at.isoformat() if order.paid_at else None,
"expires_at": order.expires_at.isoformat() if order.expires_at else None,
"created_at": order.created_at.isoformat() if order.created_at else None,
"demo_mode": demo_mode,
}
def payment_config_payload(settings: SystemSettingsData) -> dict[str, Any]:
channels = settings.available_payment_channels()
demo = settings.payment_demo_mode and settings.payment_enabled
return {
"payment_enabled": settings.payment_enabled,
"demo_mode": demo,
"unit_price": settings.account_slot_unit_price,
"min_slots": settings.account_slot_purchase_min,
"max_slots": settings.account_slot_purchase_max,
"channels": channels,
"wechat_available": "wechat" in channels,
"alipay_available": "alipay" in channels,
}
def _validate_purchase_request(settings: SystemSettingsData, slots: int, channel: str) -> None:
if not settings.payment_enabled:
raise HTTPException(status_code=400, detail="在线购买功能未开启")
if slots < settings.account_slot_purchase_min or slots > settings.account_slot_purchase_max:
raise HTTPException(
status_code=400,
detail=f"购买数量需在 {settings.account_slot_purchase_min}~{settings.account_slot_purchase_max} 之间",
)
if channel not in ("wechat", "alipay"):
raise HTTPException(status_code=400, detail="不支持的支付方式")
if channel == "wechat" and not settings.wechat_pay_enabled:
raise HTTPException(status_code=400, detail="微信支付未开启")
if channel == "alipay" and not settings.alipay_pay_enabled:
raise HTTPException(status_code=400, detail="支付宝支付未开启")
if not settings.payment_channel_selectable(channel):
name = "微信" if channel == "wechat" else "支付宝"
raise HTTPException(status_code=400, detail=f"{name}支付暂不可用,请联系管理员")
if not settings.payment_channel_available(channel):
if not settings.payment_demo_mode:
name = "微信" if channel == "wechat" else "支付宝"
raise HTTPException(status_code=400, detail=f"{name}支付尚未配置完成,请联系管理员")
async def create_order(
db: AsyncSession,
user: User,
slots: int,
channel: str,
) -> tuple[PaymentOrder, bool]:
if is_admin(user.role):
raise HTTPException(status_code=400, detail="管理员账号无需购买额度")
settings = await load_settings(db)
_validate_purchase_request(settings, slots, channel)
amount_fen = yuan_to_fen(settings.account_slot_unit_price * slots)
if amount_fen < 1:
raise HTTPException(status_code=400, detail="订单金额无效")
order = PaymentOrder(
order_no=generate_order_no(),
user_id=user.id,
channel=channel,
slots=slots,
amount_fen=amount_fen,
status="pending",
expires_at=datetime.utcnow() + timedelta(minutes=ORDER_TTL_MINUTES),
)
db.add(order)
await db.flush()
demo_mode = False
notify_base = settings.app_url_normalized()
description = f"抖音账号额度 x{slots}"
try:
if channel == "wechat" and settings.wechat_pay_enabled and settings.wechat_pay_configured():
notify_url = f"{notify_base}/api/payments/notify/wechat"
order.qr_code = wechat.create_native_order(
settings,
order.order_no,
description,
amount_fen,
notify_url,
)
elif channel == "alipay" and settings.alipay_pay_enabled and settings.alipay_configured():
notify_url = f"{notify_base}/api/payments/notify/alipay"
order.qr_code = alipay.create_precreate_order(
settings,
order.order_no,
description,
f"{fen_to_yuan(amount_fen):.2f}",
notify_url,
)
elif settings.payment_demo_mode and settings.payment_channel_selectable(channel):
demo_mode = True
order.qr_code = f"DEMO-{order.order_no}"
else:
raise HTTPException(status_code=400, detail="支付渠道未配置")
except HTTPException:
await db.rollback()
raise
except Exception as exc:
await db.rollback()
logger.exception("Create payment order failed")
raise HTTPException(status_code=400, detail=str(exc)) from exc
await db.commit()
await db.refresh(order)
return order, demo_mode
async def get_user_order(db: AsyncSession, user: User, order_no: str) -> PaymentOrder:
result = await db.execute(
select(PaymentOrder).where(
PaymentOrder.order_no == order_no,
PaymentOrder.user_id == user.id,
)
)
order = result.scalar_one_or_none()
if not order:
raise HTTPException(status_code=404, detail="订单不存在")
if order.status == "pending" and order.expires_at < datetime.utcnow():
order.status = "expired"
await db.commit()
await db.refresh(order)
return order
async def _get_order_user(db: AsyncSession, order: PaymentOrder) -> User:
result = await db.execute(select(User).where(User.id == order.user_id))
user = result.scalar_one_or_none()
if not user:
raise HTTPException(status_code=404, detail="用户不存在")
return user
async def _apply_order_slots(db: AsyncSession, order: PaymentOrder, user: User) -> None:
if order.slots_applied:
return
if is_admin(user.role) or (user.max_accounts is not None and user.max_accounts < 0):
order.slots_applied = True
return
base = int(user.max_accounts) if user.max_accounts is not None else 0
user.max_accounts = base + int(order.slots)
order.slots_applied = True
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
async def _revoke_order_slots(db: AsyncSession, order: PaymentOrder, user: User) -> None:
if not order.slots_applied:
return
if is_admin(user.role) or (user.max_accounts is not None and user.max_accounts < 0):
order.slots_applied = False
return
base = int(user.max_accounts) if user.max_accounts is not None else 0
new_limit = max(0, base - int(order.slots))
user.max_accounts = new_limit
order.slots_applied = False
await sync_user_account_quota(db, user, stop_worker=default_stop_worker)
async def _get_order_by_no(db: AsyncSession, order_no: str) -> PaymentOrder:
result = await db.execute(select(PaymentOrder).where(PaymentOrder.order_no == order_no))
order = result.scalar_one_or_none()
if not order:
raise HTTPException(status_code=404, detail="订单不存在")
return order
async def fulfill_order(db: AsyncSession, order: PaymentOrder, trade_no: str | None = None) -> None:
if order.status == "paid" and order.slots_applied:
return
if order.status != "pending":
raise HTTPException(status_code=400, detail="订单状态不可支付")
user = await _get_order_user(db, order)
await _apply_order_slots(db, order, user)
order.status = "paid"
order.paid_at = datetime.utcnow()
if trade_no:
order.trade_no = trade_no
await db.commit()
async def simulate_pay(db: AsyncSession, user: User, order_no: str) -> PaymentOrder:
settings = await load_settings(db)
if not settings.payment_demo_mode:
raise HTTPException(status_code=403, detail="演示支付未开启")
order = await get_user_order(db, user, order_no)
if order.status != "pending":
raise HTTPException(status_code=400, detail="订单不可支付")
await fulfill_order(db, order, trade_no=f"DEMO-{order.order_no}")
await db.refresh(order)
return order
async def handle_wechat_notify(db: AsyncSession, body: bytes, headers: dict[str, str]) -> bool:
settings = await load_settings(db)
if not settings.wechat_pay_configured():
return False
if not wechat.verify_notify_signature(
settings,
body,
headers.get("wechatpay-timestamp", ""),
headers.get("wechatpay-nonce", ""),
headers.get("wechatpay-signature", ""),
headers.get("wechatpay-serial", ""),
):
return False
import json
payload = json.loads(body.decode("utf-8"))
resource = payload.get("resource") or {}
data = wechat.decrypt_notify_resource(settings.wechat_api_v3_key.strip(), resource)
if data.get("trade_state") != "SUCCESS":
return True
order_no = data.get("out_trade_no")
trade_no = data.get("transaction_id")
result = await db.execute(select(PaymentOrder).where(PaymentOrder.order_no == order_no))
order = result.scalar_one_or_none()
if not order:
return False
order.notify_payload = body.decode("utf-8", errors="replace")
await fulfill_order(db, order, trade_no=trade_no)
return True
async def handle_alipay_notify(db: AsyncSession, params: dict[str, Any]) -> bool:
settings = await load_settings(db)
if not settings.alipay_configured():
return False
if not alipay.verify_notify(params, settings):
return False
if params.get("trade_status") not in ("TRADE_SUCCESS", "TRADE_FINISHED"):
return True
order_no = params.get("out_trade_no")
trade_no = params.get("trade_no")
result = await db.execute(select(PaymentOrder).where(PaymentOrder.order_no == order_no))
order = result.scalar_one_or_none()
if not order:
return False
order.notify_payload = str(params)
await fulfill_order(db, order, trade_no=trade_no)
return True
async def refresh_expired_orders(db: AsyncSession) -> None:
now = datetime.utcnow()
await db.execute(
update(PaymentOrder)
.where(PaymentOrder.status == "pending", PaymentOrder.expires_at < now)
.values(status="expired")
)
await db.commit()
def _order_is_demo(order: PaymentOrder) -> bool:
trade_no = (order.trade_no or "").strip()
return trade_no.startswith("DEMO-")
def order_list_item(order: PaymentOrder, user: User | None = None) -> dict[str, Any]:
payload = {
"id": order.id,
"order_no": order.order_no,
"user_id": order.user_id,
"username": user.username if user else None,
"display_name": user.display_name if user else None,
"channel": order.channel,
"slots": order.slots,
"amount_fen": order.amount_fen,
"amount_yuan": fen_to_yuan(order.amount_fen),
"status": order.status,
"trade_no": order.trade_no,
"is_demo": _order_is_demo(order),
"paid_at": order.paid_at,
"created_at": order.created_at,
}
return payload
async def list_orders(
db: AsyncSession,
current_user: User,
*,
status: str | None = None,
channel: str | None = None,
page: int = 1,
page_size: int = 20,
) -> dict[str, Any]:
await refresh_expired_orders(db)
page = max(1, page)
page_size = max(1, min(100, page_size))
filters = []
if not is_admin(current_user.role):
filters.append(PaymentOrder.user_id == current_user.id)
if status:
filters.append(PaymentOrder.status == status)
if channel in ("wechat", "alipay"):
filters.append(PaymentOrder.channel == channel)
count_stmt = select(func.count()).select_from(PaymentOrder)
for clause in filters:
count_stmt = count_stmt.where(clause)
total = int((await db.execute(count_stmt)).scalar() or 0)
paid_filters = list(filters) + [PaymentOrder.status == "paid"]
paid_count_stmt = select(func.count()).select_from(PaymentOrder)
for clause in paid_filters:
paid_count_stmt = paid_count_stmt.where(clause)
paid_count = int((await db.execute(paid_count_stmt)).scalar() or 0)
paid_amount_stmt = select(func.coalesce(func.sum(PaymentOrder.amount_fen), 0)).select_from(PaymentOrder)
for clause in paid_filters:
paid_amount_stmt = paid_amount_stmt.where(clause)
paid_amount_fen = int((await db.execute(paid_amount_stmt)).scalar() or 0)
stmt = select(PaymentOrder, User).join(User, User.id == PaymentOrder.user_id)
for clause in filters:
stmt = stmt.where(clause)
stmt = (
stmt.order_by(PaymentOrder.created_at.desc())
.offset((page - 1) * page_size)
.limit(page_size)
)
rows = (await db.execute(stmt)).all()
items = [order_list_item(order, user) for order, user in rows]
return {
"items": items,
"total": total,
"page": page,
"page_size": page_size,
"paid_count": paid_count,
"paid_amount_yuan": fen_to_yuan(paid_amount_fen),
}
async def admin_update_order_status(
db: AsyncSession,
order_no: str,
new_status: str,
) -> dict[str, Any]:
if new_status not in ORDER_STATUSES:
raise HTTPException(status_code=400, detail="无效的订单状态")
order = await _get_order_by_no(db, order_no)
user = await _get_order_user(db, order)
old_status = order.status
if new_status == old_status:
return order_list_item(order, user)
if new_status == "paid":
await _apply_order_slots(db, order, user)
order.status = "paid"
if not order.paid_at:
order.paid_at = datetime.utcnow()
else:
if order.slots_applied:
await _revoke_order_slots(db, order, user)
order.status = new_status
if new_status not in ("paid", "refunded"):
order.paid_at = None
await db.commit()
await db.refresh(order)
return order_list_item(order, user)
async def admin_delete_order(db: AsyncSession, order_no: str) -> None:
order = await _get_order_by_no(db, order_no)
if order.slots_applied:
user = await _get_order_user(db, order)
await _revoke_order_slots(db, order, user)
await db.delete(order)
await db.commit()
+139
View File
@@ -0,0 +1,139 @@
"""微信支付 V3 Native 扫码。"""
from __future__ import annotations
import json
import logging
import time
import uuid
from base64 import b64decode, b64encode
from typing import Any
import httpx
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import padding
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
from auth.system_settings import SystemSettingsData
logger = logging.getLogger("payments.wechat")
WECHAT_API = "https://api.mch.weixin.qq.com"
def _load_private_key(pem: str):
text = pem.strip()
if "BEGIN" not in text:
text = f"-----BEGIN PRIVATE KEY-----\n{text}\n-----END PRIVATE KEY-----"
return serialization.load_pem_private_key(text.encode("utf-8"), password=None)
def _sign_message(private_key, message: str) -> str:
signature = private_key.sign(
message.encode("utf-8"),
padding.PKCS1v15(),
hashes.SHA256(),
)
return b64encode(signature).decode("utf-8")
def _build_auth_header(
method: str,
url_path: str,
body: str,
mch_id: str,
serial_no: str,
private_key,
) -> str:
timestamp = str(int(time.time()))
nonce = uuid.uuid4().hex
message = f"{method}\n{url_path}\n{timestamp}\n{nonce}\n{body}\n"
sign = _sign_message(private_key, message)
return (
f'WECHATPAY2-SHA256-RSA2048 mchid="{mch_id}",'
f'nonce_str="{nonce}",signature="{sign}",'
f'timestamp="{timestamp}",serial_no="{serial_no}"'
)
def create_native_order(
settings: SystemSettingsData,
order_no: str,
description: str,
amount_fen: int,
notify_url: str,
) -> str:
"""创建 Native 订单,返回 code_url。"""
private_key = _load_private_key(settings.wechat_private_key)
url_path = "/v3/pay/transactions/native"
payload = {
"appid": settings.wechat_app_id.strip(),
"mchid": settings.wechat_mch_id.strip(),
"description": description[:127],
"out_trade_no": order_no,
"notify_url": notify_url,
"amount": {"total": amount_fen, "currency": "CNY"},
}
body = json.dumps(payload, ensure_ascii=False)
headers = {
"Authorization": _build_auth_header(
"POST",
url_path,
body,
settings.wechat_mch_id.strip(),
settings.wechat_cert_serial.strip(),
private_key,
),
"Content-Type": "application/json",
"Accept": "application/json",
}
with httpx.Client(timeout=30.0) as client:
resp = client.post(f"{WECHAT_API}{url_path}", content=body.encode("utf-8"), headers=headers)
if resp.status_code >= 400:
detail = resp.text
try:
detail = resp.json().get("message") or detail
except Exception:
pass
raise RuntimeError(f"微信支付下单失败: {detail}")
data = resp.json()
code_url = data.get("code_url")
if not code_url:
raise RuntimeError("微信支付未返回 code_url")
return code_url
def decrypt_notify_resource(api_v3_key: str, resource: dict[str, Any]) -> dict[str, Any]:
nonce = resource.get("nonce", "")
ciphertext = resource.get("ciphertext", "")
associated_data = resource.get("associated_data", "")
aesgcm = AESGCM(api_v3_key.encode("utf-8"))
plain = aesgcm.decrypt(
nonce.encode("utf-8"),
b64decode(ciphertext),
associated_data.encode("utf-8") if associated_data else None,
)
return json.loads(plain.decode("utf-8"))
def verify_notify_signature(
settings: SystemSettingsData,
body: bytes,
timestamp: str,
nonce: str,
signature: str,
serial: str,
) -> bool:
"""简化验签:使用平台证书较复杂,此处用商户私钥对应逻辑 + 回调解密校验。"""
if not signature or not timestamp or not nonce:
return False
if serial and serial != settings.wechat_cert_serial.strip():
logger.warning("WeChat notify serial mismatch: %s", serial)
try:
payload = json.loads(body.decode("utf-8"))
resource = payload.get("resource") or {}
decrypt_notify_resource(settings.wechat_api_v3_key.strip(), resource)
return True
except Exception as exc:
logger.warning("WeChat notify verify failed: %s", exc)
return False
+134
View File
@@ -0,0 +1,134 @@
# =============================================================================
# 抖音多账号自动回复系统 - 完整 Python 依赖
# 安装: pip install -r requirements.txt
# 环境: Python 3.10 ~ 3.11
# =============================================================================
#
# 【推荐】一键安装与 Web 部署:
#
# Linux/宝塔(自动探测 Python,兼容 pyporject_evn / 系统 Python):
# chmod +x install.sh start_web.sh
# ./install.sh --diagnose # 查看本机哪些 Python 可用
# ./install.sh # 自动选择可用 Python 安装
# AUTO_BUILD_PYTHON=1 ./install.sh # 无可用 Python 时自动编译 3.11
# ./start_web.sh
#
# Windows:
# install.bat # 安装 + 构建前端
# start_web.bat # 单端口 Web 访问
#
# pip 安装完成后,还需在服务器执行(install.sh 已包含):
#
# 1) Playwright 浏览器(扫码登录需要)
# python -m playwright install chromium
# python -m playwright install-deps # 仅 Linux
#
# 2) IM 签名 Node 依赖(发送私信 a_bogus 必需,需系统已装 Node.js 18+)
# cd rpa_engine/douyin_im/static && npm install
#
# 3) 前端构建(生产 Web 访问)
# cd ../../frontend && npm install && npm run build
#
# 4) 环境变量(生产务必设置,见项目根 .env.example)
# KEFU_SECRET_KEY=随机长字符串
# KEFU_ADMIN_PASSWORD=强密码
# PLAYWRIGHT_BROWSERS_PATH=/path/to/playwright-browsers
#
# 宝塔 Python 项目管理器启动命令:
# /www/wwwroot/douyin/backend/.venv/bin/python -m uvicorn main:app --host 0.0.0.0 --port 8000
# 工作目录: /www/wwwroot/douyin/backend
# 运行用户: www(与 chown -R www:www 一致)
# =============================================================================
# --- Web 框架 / ASGI ---
fastapi==0.111.0
starlette==0.37.2
uvicorn==0.30.1
httptools==0.8.0
watchfiles==1.2.0
websockets==12.0
python-multipart==0.0.9
# --- 数据校验 / 配置 ---
pydantic==2.7.4
pydantic_core==2.18.4
annotated-types==0.7.0
typing_extensions==4.15.0
email-validator==2.3.0
dnspython==2.8.0
python-dotenv==1.2.2
# --- 数据库 ---
SQLAlchemy==2.0.30
aiosqlite==0.20.0
# MySQL / PostgreSQL 可选驱动(使用对应数据库类型时需安装)
asyncmy==0.2.9
asyncpg==0.29.0
greenlet==3.0.3
# --- HTTP 客户端 ---
httpx==0.27.0
httpcore==1.0.9
h11==0.16.0
requests==2.34.2
urllib3==2.7.0
certifi==2026.5.20
charset-normalizer==3.4.7
idna==3.18
sniffio==1.3.1
anyio==4.13.0
# --- 鉴权 / 密码 ---
python-jose==3.3.0
passlib==1.7.4
bcrypt==4.0.1
cryptography==49.0.0
cffi==2.0.0
pycparser==3.0
ecdsa==0.19.2
pyasn1==0.6.3
rsa==4.9.1
# --- 图片处理(卡片 favicon 转换 / 私信图片尺寸)---
Pillow==10.4.0
# --- 浏览器自动化(登录 / 采集凭证)---
playwright==1.44.0
pyee==11.1.0
# 无图形界面的 Linux 服务器跑有头浏览器需要虚拟显示(还需系统安装 Xvfb):
# Debian/Ubuntu: apt install -y xvfb
# CentOS/Rocky : yum install -y xorg-x11-server-Xvfb
pyvirtualdisplay==3.0
# --- 抖音 IM(Protobuf / 签名 / WebSocket)---
protobuf==5.27.1
protobuf3-to-dict==0.1.5
six==1.17.0
PyExecJS==1.5.1
websocket-client==1.9.0
# --- FastAPI CLI / 工具(随 fastapi 安装,显式锁定避免缺包)---
fastapi-cli==0.0.24
typer==0.26.7
click==8.4.1
shellingham==1.5.4
rich==15.0.0
rich-toolkit==0.20.1
Pygments==2.20.0
markdown-it-py==4.2.0
mdurl==0.1.2
Jinja2==3.1.6
MarkupSafe==3.0.3
annotated-doc==0.0.4
orjson==3.11.9
ujson==5.12.1
PyYAML==6.0.3
colorama==0.4.6
# =============================================================================
# [npm] IM 签名 jsrsasign(在 rpa_engine/douyin_im/static/package.json)
# jsrsasign@^11.1.0
#
# [npm] 前端(在 frontend/package.json)
# vue, ant-design-vue, axios, pinia, vue-router, vite
# =============================================================================
File diff suppressed because it is too large Load Diff
+318
View File
@@ -0,0 +1,318 @@
"""Bounded, deduplicated background queue for bulk account starts."""
from __future__ import annotations
import asyncio
import logging
import os
import time
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Awaitable, Callable
logger = logging.getLogger("rpa.batch_start")
StartHandler = Callable[[int], Awaitable[dict[str, Any]]]
JobToken = tuple[str, int]
def _configured_concurrency() -> int:
try:
return max(1, min(8, int(os.getenv("KEFU_BATCH_START_CONCURRENCY", "2"))))
except (TypeError, ValueError):
return 2
def _utc_now() -> str:
return datetime.now(timezone.utc).isoformat()
@dataclass
class _BatchRecord:
batch_id: str
owner_id: int | None = None
metadata: dict[str, Any] = field(default_factory=dict)
items: dict[int, dict[str, Any]] = field(default_factory=dict)
created_at: str = field(default_factory=_utc_now)
updated_at: str = field(default_factory=_utc_now)
class BatchStartQueue:
"""Run account preparation with a small process-wide concurrency cap.
The HTTP endpoint only enqueues account ids. Long credential checks then
run in these workers, so a batch of many accounts cannot block the request
that submitted it. ``_pending_accounts`` makes overlapping clicks and
overlapping batches idempotent for each account. Pending and active
ownership is tied to a concrete job token so cleanup from a cancelled old
job cannot release a newer submission for the same account.
"""
def __init__(
self,
handler: StartHandler,
concurrency: int | None = None,
max_batches: int = 100,
) -> None:
self._handler = handler
self.concurrency = max(1, int(concurrency or _configured_concurrency()))
self.max_batches = max(10, int(max_batches or 100))
self._queue: asyncio.Queue[JobToken] = asyncio.Queue()
self._pending_jobs: dict[int, JobToken] = {}
self._active_tasks: dict[int, tuple[JobToken, asyncio.Task]] = {}
self._batches: dict[str, _BatchRecord] = {}
self._workers: list[asyncio.Task] = []
self._lock = asyncio.Lock()
self._stopping = False
async def _ensure_workers(self) -> None:
async with self._lock:
self._workers = [task for task in self._workers if not task.done()]
if self._workers or self._stopping:
return
for index in range(self.concurrency):
self._workers.append(
asyncio.create_task(
self._worker(index + 1),
name=f"account-batch-start-{index + 1}",
)
)
async def submit(
self,
account_ids: list[int],
owner_id: int | None = None,
metadata: dict[str, Any] | None = None,
) -> dict[str, Any]:
await self._ensure_workers()
unique_ids = list(dict.fromkeys(int(value) for value in account_ids if int(value) > 0))
batch_id = uuid.uuid4().hex
record = _BatchRecord(
batch_id=batch_id,
owner_id=owner_id,
metadata=dict(metadata or {}),
)
async with self._lock:
if self._stopping:
raise RuntimeError("账号启动队列正在停止")
self._prune_locked()
self._batches[batch_id] = record
for account_id in unique_ids:
if account_id in self._pending_jobs:
record.items[account_id] = {
"account_id": account_id,
"status": "already_queued",
"message": "账号已在启动队列中",
}
continue
job_token = (batch_id, account_id)
self._pending_jobs[account_id] = job_token
record.items[account_id] = {
"account_id": account_id,
"status": "queued",
"message": "等待启动",
}
self._queue.put_nowait(job_token)
record.updated_at = _utc_now()
return self._snapshot_locked(record)
async def get_batch(
self,
batch_id: str,
owner_id: int | None = None,
include_items: bool = False,
) -> dict[str, Any] | None:
async with self._lock:
record = self._batches.get(str(batch_id or ""))
if record and owner_id is not None and record.owner_id != int(owner_id):
return None
return self._snapshot_locked(record, include_items=include_items) if record else None
async def _worker(self, worker_number: int) -> None:
while True:
batch_id, account_id = await self._queue.get()
job_token = (batch_id, account_id)
started_at = time.monotonic()
handler_task: asyncio.Task | None = None
try:
async with self._lock:
record = self._batches.get(batch_id)
if not record:
if self._pending_jobs.get(account_id) == job_token:
self._pending_jobs.pop(account_id, None)
continue
item = record.items[account_id]
if item.get("status") == "cancelled":
if self._pending_jobs.get(account_id) == job_token:
self._pending_jobs.pop(account_id, None)
continue
item.update(status="processing", message="正在校验并启动")
record.updated_at = _utc_now()
handler_task = asyncio.create_task(
self._handler(account_id),
name=f"account-start-{account_id}",
)
self._active_tasks[account_id] = (job_token, handler_task)
result = await handler_task
async with self._lock:
record = self._batches.get(batch_id)
if record:
item = record.items[account_id]
if item.get("status") != "cancelled":
item.update(
status="submitted",
message=str(result.get("message") or "已提交启动"),
login_mode=result.get("login_mode"),
skip_browser=bool(result.get("skip_browser", False)),
elapsed_seconds=round(time.monotonic() - started_at, 3),
)
record.updated_at = _utc_now()
except asyncio.CancelledError:
async with self._lock:
record = self._batches.get(batch_id)
if record:
record.items[account_id].update(
status="cancelled",
message="服务停止,启动任务已取消",
)
record.updated_at = _utc_now()
# Cancelling one account must not kill a long-lived queue
# worker. Re-raise only when stop() cancelled the worker.
if asyncio.current_task().cancelling():
raise
except Exception as exc:
logger.exception(
"Batch start failed account=%s worker=%s: %s",
account_id,
worker_number,
exc,
)
async with self._lock:
record = self._batches.get(batch_id)
if record:
detail = getattr(exc, "detail", None) or str(exc) or "启动失败"
item = record.items[account_id]
if item.get("status") != "cancelled":
item.update(
status="failed",
message=str(detail),
elapsed_seconds=round(time.monotonic() - started_at, 3),
)
record.updated_at = _utc_now()
finally:
async with self._lock:
active_entry = self._active_tasks.get(account_id)
if active_entry == (job_token, handler_task):
self._active_tasks.pop(account_id, None)
if self._pending_jobs.get(account_id) == job_token:
self._pending_jobs.pop(account_id, None)
self._queue.task_done()
async def cancel_account(self, account_id: int) -> int:
"""Cancel queued/processing work so stop/delete cannot restart it."""
account_key = int(account_id)
cancelled = 0
active_task: asyncio.Task | None = None
async with self._lock:
for record in self._batches.values():
item = record.items.get(account_key)
if not item or item.get("status") not in ("queued", "processing"):
continue
item.update(status="cancelled", message="启动任务已取消")
record.updated_at = _utc_now()
cancelled += 1
self._pending_jobs.pop(account_key, None)
active_entry = self._active_tasks.get(account_key)
active_task = active_entry[1] if active_entry else None
if active_task and not active_task.done():
active_task.cancel()
if active_task and not active_task.done():
await asyncio.gather(active_task, return_exceptions=True)
return cancelled
def _snapshot_locked(
self,
record: _BatchRecord,
*,
include_items: bool = True,
) -> dict[str, Any]:
counts = {
"queued": 0,
"processing": 0,
"submitted": 0,
"failed": 0,
"skipped": 0,
"cancelled": 0,
}
browser_required = 0
for item in record.items.values():
status = item.get("status")
if status == "submitted" and item.get("skip_browser") is False:
browser_required += 1
if status == "already_queued":
counts["skipped"] += 1
elif status in counts:
counts[status] += 1
active = counts["queued"] + counts["processing"]
snapshot = {
"batch_id": record.batch_id,
"total_count": len(record.items),
"accepted_count": len(record.items) - counts["skipped"],
"queued_count": counts["queued"],
"processing_count": counts["processing"],
"submitted_count": counts["submitted"],
"failed_count": counts["failed"],
"skipped_count": counts["skipped"],
"cancelled_count": counts["cancelled"],
"browser_required_count": browser_required,
"complete": active == 0,
"created_at": record.created_at,
"updated_at": record.updated_at,
}
snapshot.update(record.metadata)
if include_items:
snapshot["items"] = [dict(item) for item in record.items.values()]
return snapshot
def _prune_locked(self) -> None:
if len(self._batches) < self.max_batches:
return
removable = [
batch_id
for batch_id, record in self._batches.items()
if self._snapshot_locked(record, include_items=False)["complete"]
]
for batch_id in removable[: max(1, len(self._batches) - self.max_batches + 1)]:
self._batches.pop(batch_id, None)
async def stop(self) -> None:
async with self._lock:
self._stopping = True
workers = list(self._workers)
self._workers.clear()
for task in workers:
task.cancel()
if workers:
await asyncio.gather(*workers, return_exceptions=True)
async with self._lock:
self._active_tasks.clear()
while True:
try:
batch_id, account_id = self._queue.get_nowait()
except asyncio.QueueEmpty:
break
record = self._batches.get(batch_id)
if record:
record.items[account_id].update(
status="cancelled",
message="服务停止,启动任务已取消",
)
record.updated_at = _utc_now()
job_token = (batch_id, account_id)
if self._pending_jobs.get(account_id) == job_token:
self._pending_jobs.pop(account_id, None)
self._queue.task_done()
+226
View File
@@ -0,0 +1,226 @@
"""账号凭证检测:静态 Cookie 分析 + IM 运行时校验"""
import asyncio
import json
import logging
from typing import Optional
from rpa_engine.douyin_im.auth import DouyinAuth
from rpa_engine.douyin_im.frontier import ensure_frontier_ws
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
from rpa_engine.douyin_im.session import DouyinImSession
from utils.cookie_store import analyze_cookie
logger = logging.getLogger("credential")
def _should_reset_credentials(assessment: dict) -> bool:
"""凭证全面失效时需清空 Cookie/IM 数据并重新登录。"""
if not assessment.get("has_cookie"):
return False
if not assessment.get("cookie_valid"):
return True
message = assessment.get("message") or ""
# 仅缺 ticket/签名/浏览器采集 — 保留 Cookie,走浏览器补全即可
if any(
token in message
for token in ("ticket", "签名密钥", "浏览器模式", "web_protect")
):
return False
if (
not assessment.get("im_ready")
and not assessment.get("can_skip_browser")
and assessment.get("has_sessionid")
):
return True
return False
def build_im_session_from_storage(
storage: dict,
saved_im_data: Optional[str] = None,
) -> DouyinImSession:
session = DouyinImSession.from_storage_state(storage or {})
if saved_im_data:
try:
saved = DouyinImSession.from_dict(json.loads(saved_im_data))
# 新粘贴的 storage_state(含真实 frontier_ws_url)优先;只有它没带时才用缓存的。
if saved.ws_urls and not session.ws_urls:
session.ws_urls = saved.ws_urls
if saved.sdk_cert and not session.sdk_cert:
session.sdk_cert = saved.sdk_cert
if saved.frontier_ts_sign and not session.frontier_ts_sign:
session.frontier_ts_sign = saved.frontier_ts_sign
if saved.keys_str and not session.keys_str:
session.keys_str = saved.keys_str
if saved.web_protect_str and not session.web_protect_str:
session.web_protect_str = saved.web_protect_str
if saved.my_uid and not session.my_uid:
session.my_uid = saved.my_uid
if saved.device_id and not session.device_id:
session.device_id = saved.device_id
if saved.web_id and not session.web_id:
session.web_id = saved.web_id
if saved.conv_meta:
session.conv_meta = {**saved.conv_meta, **session.conv_meta}
except Exception:
pass
session.sanitize_ws_urls()
# Keep this builder pure and non-blocking. Frontier discovery can perform
# a synchronous network request (up to 15 seconds); callers that need it
# already do so from validate_im_session() through asyncio.to_thread() and
# the shared background-traffic limiter. Running it here made a large
# batch freeze the FastAPI event loop before any limiter was acquired.
return session
def has_im_session_token(session: DouyinImSession) -> bool:
return bool(session.cookies.get("sessionid") or session.cookies.get("sessionid_ss"))
def extract_sessionid_info(session: DouyinImSession) -> dict:
sessionid = session.cookies.get("sessionid") or ""
sessionid_ss = session.cookies.get("sessionid_ss") or ""
return {
"has_sessionid": bool(sessionid or sessionid_ss),
"sessionid": sessionid,
"sessionid_ss": sessionid_ss,
}
async def build_cookie_credential_detail(
cookie_data: Optional[str],
im_session_data: Optional[str] = None,
runtime_check: bool = True,
) -> dict:
"""供编辑账号页展示 IM 凭证与 sessionid 信息"""
sessionid_info = {
"has_sessionid": False,
"sessionid": "",
"sessionid_ss": "",
"im_ready": False,
"im_status": "未保存 Cookie",
"can_skip_browser": False,
"should_reset": False,
}
if not cookie_data:
return sessionid_info
try:
storage = json.loads(cookie_data)
session = build_im_session_from_storage(storage, im_session_data)
sessionid_info.update(extract_sessionid_info(session))
except Exception:
sessionid_info["im_status"] = "Cookie 格式错误"
return sessionid_info
if not runtime_check:
if sessionid_info["has_sessionid"]:
sessionid_info["im_status"] = "已检测到 sessionid(未做运行时验证)"
else:
sessionid_info["im_status"] = "缺少 sessionid,无法 IM 直连"
return sessionid_info
assessment = await assess_account_credential(cookie_data, im_session_data)
sessionid_info["im_ready"] = assessment["im_ready"]
sessionid_info["im_status"] = assessment["message"]
sessionid_info["can_skip_browser"] = assessment["can_skip_browser"]
sessionid_info["should_reset"] = assessment["should_reset"]
return sessionid_info
async def validate_im_session(
session: DouyinImSession,
_bypass_global_limit: bool = False,
) -> tuple[bool, str]:
if not _bypass_global_limit:
from rpa_engine.douyin_im.traffic_control import get_traffic_controller
controller = get_traffic_controller()
async with controller.background_slot(0, "credential validation"):
return await validate_im_session(session, _bypass_global_limit=True)
if not session.can_direct_im():
if not has_im_session_token(session):
return False, "缺少 sessionid,无法直连 IM"
return False, "Cookie 不满足 IM 直连条件"
await asyncio.to_thread(ensure_frontier_ws, session)
try:
auth = DouyinAuth.from_im_session(session)
# 优先用已持久化的 my_uid,避免每次都发起网络 query_my_uid(uid_tt 是加密串,
# int() 解析必然失败而回退到网络请求;该请求偶发失败会误判为“未就绪”)。
uid = session.my_uid or auth.get_uid()
if not uid:
return False, "服务端未认可当前 Cookie(无法获取用户 UID)"
if not auth.is_sign_ready():
return False, "缺少 IM 签名密钥(web_protect/keys),请用浏览器登录补全"
session.my_uid = int(uid)
async with DouyinImHttpClient(session) as http:
await http.get_unread_count()
# 若已缓存到会话票据,优先校验其是否仍新鲜(最理想)。
if session.conv_meta:
ok, reason = await http.verify_messaging_capability(auth, session.my_uid)
if ok:
return True, reason
# 没有缓存会话票据是首次登录的正常情况:会话 ticket 会在发送时即时
# 创建/获取(resolve_conversation_meta),因此只要 Cookie + sessionid +
# 签名密钥(web_protect/keys) + UID 齐全,就视为可 IM 直连托管,不必再开浏览器。
return True, "IM 凭证就绪(Cookie 与签名密钥齐全,可直连托管)"
except Exception as e:
logger.warning(f"IM session validation failed: {e}")
return False, f"IM 运行时验证失败: {e}"
async def assess_account_credential(
cookie_data: Optional[str],
im_session_data: Optional[str] = None,
) -> dict:
cookie_info = analyze_cookie(cookie_data)
result = {
"has_cookie": cookie_info.get("has_cookie", False),
"cookie_valid": cookie_info.get("cookie_valid", False),
"cookie_status": cookie_info.get("reason", ""),
"has_sessionid": False,
"im_ready": False,
"can_skip_browser": False,
"login_mode": "browser",
"message": "未保存 Cookie,需浏览器扫码登录",
"should_reset": False,
}
if not cookie_data:
return result
try:
storage = json.loads(cookie_data)
except Exception:
result["message"] = "Cookie 格式错误"
result["should_reset"] = _should_reset_credentials(result)
return result
session = build_im_session_from_storage(storage, im_session_data)
result["has_sessionid"] = has_im_session_token(session)
if not cookie_info.get("cookie_valid"):
result["message"] = cookie_info.get("reason") or "Cookie 无效,需重新登录"
result["should_reset"] = _should_reset_credentials(result)
return result
if not result["has_sessionid"]:
result["login_mode"] = "browser"
result["message"] = "Cookie 已保存但缺少 sessionid,需浏览器刷新登录态"
result["should_reset"] = _should_reset_credentials(result)
return result
im_ok, im_reason = await validate_im_session(session)
result["im_ready"] = im_ok
if im_ok:
result["can_skip_browser"] = True
result["login_mode"] = "im_direct"
result["message"] = im_reason
else:
result["login_mode"] = "browser"
result["message"] = im_reason
result["should_reset"] = _should_reset_credentials(result)
return result
+89
View File
@@ -0,0 +1,89 @@
"""浏览器 / IM 伪装设备头(User-Agent)预设。
a_bogus 签名、Playwright 浏览器上下文、IM HTTP 请求头、Protobuf body 必须使用同一 UA,
否则抖音会返回 7911 安全校验失败。
"""
from rpa_engine.douyin_im.dy_util import DEFAULT_USER_AGENT
# id 用于前端下拉;user_agent 为完整字符串
DEVICE_PROFILES: list[dict[str, str]] = [
{
"id": "chrome_win120",
"label": "Chrome 120 · Windows(默认,推荐)",
"platform": "Windows",
"user_agent": DEFAULT_USER_AGENT,
},
{
"id": "chrome_win131",
"label": "Chrome 131 · Windows",
"platform": "Windows",
"user_agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
),
},
{
"id": "edge_win125",
"label": "Edge 125 · Windows",
"platform": "Windows",
"user_agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36 Edg/125.0.0.0"
),
},
{
"id": "firefox_win117",
"label": "Firefox 117 · Windows",
"platform": "Windows",
"user_agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) "
"Gecko/20100101 Firefox/117.0"
),
},
{
"id": "chrome_mac120",
"label": "Chrome 120 · macOS",
"platform": "macOS",
"user_agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
),
},
{
"id": "safari_mac17",
"label": "Safari 17 · macOS",
"platform": "macOS",
"user_agent": (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 "
"(KHTML, like Gecko) Version/17.0 Safari/605.1.15"
),
},
]
_PROFILE_BY_UA = {p["user_agent"]: p for p in DEVICE_PROFILES}
def list_device_profiles() -> list[dict[str, str]]:
return [
{"id": p["id"], "label": p["label"], "platform": p["platform"], "user_agent": p["user_agent"]}
for p in DEVICE_PROFILES
]
def resolve_user_agent(stored: str | None) -> str:
"""账号保存的 UA;空则使用默认 Chrome 120。"""
text = (stored or "").strip()
return text or DEFAULT_USER_AGENT
def profile_label_for_ua(ua: str | None) -> str:
text = (ua or "").strip()
if not text:
return "Chrome 120 · Windows(默认)"
hit = _PROFILE_BY_UA.get(text)
if hit:
return hit["label"]
if len(text) > 48:
return text[:48] + "…"
return text
+3
View File
@@ -0,0 +1,3 @@
from .service import DouyinImService
__all__ = ["DouyinImService"]
+163
View File
@@ -0,0 +1,163 @@
import base64
import json
import logging
import requests
from .dy_util import (
trans_cookies,
generate_msToken,
generate_a_bogus,
splice_url,
generate_webid,
normalize_client_cert,
resolve_proto_device_id,
DEFAULT_USER_AGENT,
)
logger = logging.getLogger("douyin_im.auth")
def _parse_storage_json(raw) -> dict | None:
"""Parse localStorage JSON (supports nested data wrapper)."""
if not raw:
return None
if isinstance(raw, dict):
data = raw
else:
text = str(raw).strip()
data = None
for _ in range(4):
try:
parsed = json.loads(text)
except Exception:
break
if isinstance(parsed, str):
text = parsed
continue
if isinstance(parsed, dict):
data = parsed
break
break
if not data:
return None
inner = data.get("data")
if isinstance(inner, str):
try:
inner = json.loads(inner)
except Exception:
inner = None
if isinstance(inner, dict):
return inner
return data
class DouyinAuth:
def __init__(self):
self.cookie = None
self.cookie_str = None
self.private_key = None
self.ticket = None
self.ts_sign = None
self.client_cert = None
self.ree_public_key = None
self.uid = None
self.msToken = None
self.web_id = None
def perepare_auth(self, cookieStr: str, web_protect_: str = "", keys_: str = ""):
self.cookie = trans_cookies(cookieStr)
self.cookie_str = cookieStr
self.msToken = self.cookie["msToken"] if "msToken" in self.cookie else generate_msToken()
self.cookie["msToken"] = self.msToken
self.cookie_str = "; ".join([f"{k}={v}" for k, v in self.cookie.items()])
web_data = _parse_storage_json(web_protect_)
if web_data:
try:
self.ticket = web_data.get("ticket") or ""
self.ts_sign = web_data.get("ts_sign") or ""
self.client_cert = web_data.get("client_cert") or ""
except Exception as e:
logger.debug(f"web_protect parse failed: {e}")
keys_data = _parse_storage_json(keys_)
if keys_data:
try:
self.private_key = keys_data.get("ec_privateKey") or keys_data.get("privateKey") or ""
if self.private_key:
self.ree_public_key = base64.b64encode(self.private_key.encode()).decode()
except Exception as e:
logger.debug(f"keys parse failed: {e}")
def is_sign_ready(self) -> bool:
return bool(
self.private_key
and self.ticket
and self.ts_sign
and self.client_cert
)
@classmethod
def from_im_session(cls, session) -> "DouyinAuth":
"""从 DouyinImSession 构建 HTTP 发送/拉会话用的签名上下文。
注意:frontier WS 的 sdk_cert/ts_sign 只用于长连接,不能覆盖 web_protect,
否则 bd-ticket-guard 与 protobuf 签名会与 ticket 失配 -> 7911。
"""
auth = cls()
auth.perepare_auth(
session.cookie_header(),
session.web_protect_str,
session.keys_str,
)
auth.web_id = session.web_id or session.device_id or None
auth.user_agent = session.user_agent or DEFAULT_USER_AGENT
auth.device_id = resolve_proto_device_id(
session.device_id, session.web_id, session.my_uid
)
# web_protect 缺 client_cert 时,才用 frontier 抓包证书兜底(不覆盖 ts_sign)
if not auth.client_cert and getattr(session, "sdk_cert", ""):
auth.client_cert = normalize_client_cert(session.sdk_cert)
elif auth.client_cert:
auth.client_cert = normalize_client_cert(auth.client_cert)
return auth
def get_uid(self):
if self.uid is None:
# 优先从 cookie 尝试提取,否则请求接口
for k in ("uid_tt", "uid_tt_ss"):
if self.cookie and self.cookie.get(k):
try:
self.uid = int(self.cookie.get(k))
return self.uid
except ValueError:
pass
try:
self.uid = self.query_my_uid()
except Exception:
pass
return self.uid
def query_my_uid(self) -> int:
url = 'https://www.douyin.com/aweme/v1/web/query/user/'
headers = {
"User-Agent": DEFAULT_USER_AGENT,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
}
params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"publish_video_strategy_type": "2",
"verifyFp": self.cookie.get('s_v_web_id', ''),
"fp": self.cookie.get('s_v_web_id', ''),
"webid": generate_webid(self, "https://www.douyin.com/"),
"msToken": self.msToken
}
query = splice_url(params)
abogus = generate_a_bogus(query, user_agent=DEFAULT_USER_AGENT)
params['a_bogus'] = abogus
resp = requests.get(url, params=params, headers=headers, cookies=self.cookie, verify=False, timeout=10)
resp_json = resp.json()
return int(resp_json['user_uid'])
+47
View File
@@ -0,0 +1,47 @@
"""Conversation id helpers for Douyin IM."""
def parse_conversation_parts(conversation_id: str) -> tuple[int, int] | None:
parts = (conversation_id or "").split(":")
if len(parts) >= 4 and parts[0] == "0" and parts[1] == "1":
try:
return int(parts[2]), int(parts[3])
except (TypeError, ValueError):
return None
return None
def resolve_peer_uid(conversation_id: str, my_uid: int) -> int | None:
"""Resolve peer user id from conversation id or bare numeric id."""
raw = (conversation_id or "").strip()
if not raw:
return None
parts = parse_conversation_parts(raw)
if parts:
uid1, uid2 = parts
if my_uid and uid1 == my_uid:
return uid2
if my_uid and uid2 == my_uid:
return uid1
# 0:1:{my}:{peer} — 若 my_uid 与首段不一致,仍取末段为对方
return uid2
if raw.isdigit():
peer = int(raw)
if my_uid and peer == my_uid:
return None
return peer
return None
def build_conversation_id(my_uid: int, peer_uid: int) -> str:
return f"0:1:{int(my_uid)}:{int(peer_uid)}"
def normalize_conversation_id(conversation_id: str, my_uid: int) -> str:
"""Ensure conversation id uses current account uid as first participant."""
peer_uid = resolve_peer_uid(conversation_id, my_uid)
if peer_uid and my_uid:
return build_conversation_id(my_uid, peer_uid)
return (conversation_id or "").strip()
+205
View File
@@ -0,0 +1,205 @@
import hashlib
import re
import time
import json
import random
import base64
import urllib.parse
from os import path
import subprocess
original_popen = subprocess.Popen
def patched_popen(*args, **kwargs):
if kwargs.get('universal_newlines') or kwargs.get('text'):
if 'encoding' not in kwargs:
kwargs['encoding'] = 'utf-8'
return original_popen(*args, **kwargs)
subprocess.Popen = patched_popen
import execjs
import requests
basedir = path.dirname(__file__)
static_dir = path.join(basedir, 'static')
node_modules = path.join(static_dir, 'node_modules')
# 全局唯一 User-Agent:a_bogus 签名、HTTP 请求头、protobuf body、webid 采集等
# 必须全部使用同一个 UA,否则抖音服务端重算 a_bogus 时与请求头 UA 不一致 -> 7911。
# 该值需与浏览器登录上下文(playwright new_context user_agent)保持一致。
DEFAULT_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
# 动态编译 JS
dy_path = path.join(static_dir, 'dy_ab.js')
dy_js = execjs.compile(open(dy_path, 'r', encoding='utf-8').read(), cwd=node_modules)
sign_path = path.join(static_dir, 'dy_live_sign.js')
sign_js = execjs.compile(open(sign_path, 'r', encoding='utf-8').read(), cwd=node_modules)
login_path = path.join(static_dir, 'login.js')
login_js = execjs.compile(open(login_path, 'r', encoding='utf-8').read(), cwd=node_modules)
def generateSecretPhoneNum(phone):
return login_js.call('generateSecretPhoneNum', phone)
def generateSecretCode(phone, code):
return login_js.call('generateSecretCode', phone, code)
def trans_cookies(cookies_str):
cookies = {}
for i in cookies_str.split("; "):
try:
parts = i.split('=')
key = parts[0].strip()
val = '='.join(parts[1:]).strip()
# 防御性清洗:过滤掉因为误粘贴 fetch 等包含非法字符或换行的 Cookie 键
if not key or any(c in key for c in "()[]{}'\"\n \t\\"):
continue
cookies[key] = val
except:
continue
return cookies
def generate_req_sign(e, priK):
"""私信传 obj,其他的拼接"""
return dy_js.call('get_req_sign', e, priK)
def generate_a_bogus(query, data="", user_agent=None):
"""query, data 都是拼接字符串。
user_agent 必须与实际发出请求所用的 User-Agent 完全一致(见 DEFAULT_USER_AGENT),
否则抖音服务端用请求头 UA 重算 a_bogus 会对不上,导致 7911 安全校验失败。
"""
return dy_js.call('get_ab', query, data, user_agent or DEFAULT_USER_AGENT)
def generate_signature(room_id, user_unique_id):
raw_string = f"live_id=1,aid=6383,version_code=180800,webcast_sdk_version=1.0.15,room_id={room_id},sub_room_id=,sub_channel_id=,did_rule=3,user_unique_id={user_unique_id},device_platform=web,device_type=,ac=,identity=audience"
x_ms_stub = hashlib.md5(raw_string.encode("utf-8")).hexdigest()
result = sign_js.call("get_signature", x_ms_stub)
return result.get("X-Bogus")
def generate_ree_key(prik):
"""传递私钥"""
return dy_js.call('get_ree_key', prik)
def generate_bd_ticket_client_data(api, ticket, ts_sign, priK):
"""传递 query, ticket, ts_sign, priK"""
timestamp = int(time.time())
res_sign = f"ticket={ticket}&path={api}&timestamp={timestamp}"
p = {
'ts_sign': ts_sign,
'req_content': 'ticket,path,timestamp',
'req_sign': generate_req_sign(res_sign, priK),
'timestamp': timestamp,
}
p = json.dumps(p, ensure_ascii=False, separators=(',', ':'))
return base64.urlsafe_b64encode(p.encode('utf-8')).decode('utf-8')
def generate_msToken(randomlength=107):
random_str = ''
base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789='
length = len(base_str) - 1
for _ in range(randomlength):
random_str += base_str[random.randint(0, length)]
return random_str
def generate_fake_webid(random_length=19):
random_str = ''
base_str = '0123456789'
length = len(base_str) - 1
for _ in range(random_length):
random_str += base_str[random.randint(0, length)]
return random_str
def generate_webid(auth=None, url=""):
# 优先用已采集到的 web_id(避免每次发送都发起一次阻塞的 HTTP 请求,导致事件循环卡顿)
cached = getattr(auth, "web_id", None) if auth is not None else None
if cached:
return str(cached)
if url == "":
url = "https://www.douyin.com/discover?modal_id=7376449060384935209"
try:
from .auth import DouyinAuth
headers = {
"User-Agent": DEFAULT_USER_AGENT,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"upgrade-insecure-requests": "1"
}
if auth and auth.cookie_str:
headers['cookie'] = auth.cookie_str
try:
from rpa_engine.runtime_config import requests_proxies
proxies = requests_proxies()
except Exception:
proxies = None
response = requests.get(
url, headers=headers, verify=False, timeout=10, proxies=proxies
)
res_text = response.text
user_unique_id = re.findall(r'\\"user_unique_id\\":\\"(.*?)\\"', res_text)[0]
# 把发现的 web_id 回写到 auth 上,避免同一次同步里反复发起阻塞的 HTTP 请求
if auth is not None and user_unique_id:
try:
auth.web_id = user_unique_id
except Exception:
pass
return user_unique_id
except Exception:
# 失败时同样缓存一个伪 web_id,避免后续调用重复走 10s 超时的网络请求
fake = generate_fake_webid()
if auth is not None:
try:
if not getattr(auth, "web_id", None):
auth.web_id = fake
except Exception:
pass
return fake
def generate_millisecond():
return int(round(time.time() * 1000))
def normalize_client_cert(cert: str) -> str:
"""统一 client_cert / sdk_cert 格式为「base64 证书体」。
web_protect.client_cert 与 frontier WS 的 sdk_cert 通常已是 base64(PEM);
若误把 PEM 原文或 frontier 证书二次 base64,会导致 7911。
"""
cert = (cert or "").strip()
if not cert:
return ""
if cert.startswith("-----BEGIN"):
return base64.b64encode(cert.encode("utf-8")).decode("utf-8")
return cert
def resolve_proto_device_id(device_id: str = "", web_id: str = "", my_uid: int = 0) -> str:
"""protobuf / frontier 更倾向使用数字 device_id(通常等于 my_uid)。"""
for candidate in (device_id, web_id, str(my_uid or "")):
c = str(candidate or "").strip()
if c.isdigit():
return c
return str(device_id or web_id or "0")
def splice_url(params):
splice_url_str = ''
for key, value in params.items():
if value is None:
value = ''
splice_url_str += key + '=' + urllib.parse.quote(str(value)) + '&'
return splice_url_str[:-1]
+136
View File
@@ -0,0 +1,136 @@
"""抖音标准表情(评论区 emoji)名称 -> 图片 URL 映射。
抖音文字表情如 [酷拽]/[微笑] 通过 WS 以 message_type=7 的纯文本下发,
content 形如 {"text":"[酷拽]","aweType":700},不带图片地址;
浏览器端靠本地表情表把 [name] 渲染成小图。这里拉取官方表情列表接口,
建立 name->url 映射,收到文字表情时补成可显示的贴纸。
"""
from __future__ import annotations
import logging
import re
import threading
import time
from typing import Optional
logger = logging.getLogger("douyin_im.emoji")
_EMOJI_MAP: dict[str, str] = {}
_FETCHED_AT: float = 0.0
_TTL = 6 * 3600 # 6 小时刷新一次
_lock = threading.Lock()
_fetch_lock = threading.Lock()
_LAST_FETCH_ATTEMPT: float = 0.0
_FAILURE_RETRY_SECONDS = 60.0
_BRACKET_RE = re.compile(r"^\[[^\[\]]{1,24}\]$")
def has_emoji_map() -> bool:
return bool(_EMOJI_MAP)
def is_fresh() -> bool:
return bool(_EMOJI_MAP) and (time.time() - _FETCHED_AT) < _TTL
def set_emoji_map(mapping: dict[str, str]) -> None:
global _EMOJI_MAP, _FETCHED_AT
if mapping:
with _lock:
_EMOJI_MAP = dict(mapping)
_FETCHED_AT = time.time()
def lookup_emoji_url(name: str) -> str:
"""name 可带或不带中括号,返回标准表情图片 URL(无则空串)。"""
if not name:
return ""
key = name.strip()
if not key:
return ""
if not key.startswith("["):
key = f"[{key}]"
return _EMOJI_MAP.get(key, "")
def looks_like_emoji_token(text: str) -> bool:
return bool(_BRACKET_RE.match((text or "").strip()))
def fetch_emoji_map(session) -> dict[str, str]:
"""用账号会话拉取官方表情列表,返回 {display_name: url}。失败返回 {}。"""
try:
import requests
from .auth import DouyinAuth
from .dy_util import generate_a_bogus, generate_msToken, splice_url
auth = DouyinAuth.from_im_session(session)
ua = session.user_agent
s_v_web_id = session.cookies.get("s_v_web_id", "") if session.cookies else ""
params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"pc_client_type": "1",
"version_code": "170400",
"version_name": "17.4.0",
"cookie_enabled": "true",
"browser_language": "zh-CN",
"browser_platform": "Win32",
"browser_name": "Mozilla",
"browser_online": "true",
"verifyFp": s_v_web_id,
"fp": s_v_web_id,
"webid": session.web_id or session.device_id or "",
"msToken": generate_msToken(),
}
params["a_bogus"] = generate_a_bogus(splice_url(params), user_agent=ua)
headers = {
"User-Agent": ua,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
"Cookie": session.cookie_header(),
}
resp = requests.get(
"https://www.douyin.com/aweme/v1/web/emoji/list",
params=params,
headers=headers,
cookies=auth.cookie if getattr(auth, "cookie", None) else None,
verify=False,
timeout=20,
)
data = resp.json()
mapping: dict[str, str] = {}
for item in data.get("emoji_list") or []:
name = item.get("display_name")
urls = (item.get("emoji_url") or {}).get("url_list") or []
if name and urls:
mapping[name] = urls[0]
logger.info("Fetched %d douyin emoji", len(mapping))
return mapping
except Exception as e:
logger.warning("fetch_emoji_map failed: %s", e)
return {}
def ensure_emoji_map(session) -> None:
"""若缓存为空/过期则拉取(同步阻塞,调用方建议放线程)。"""
global _LAST_FETCH_ATTEMPT
if is_fresh():
return
# Batch-started accounts used to all observe an empty cache and fetch the
# same emoji list concurrently. Keep the network request itself inside a
# separate single-flight lock (set_emoji_map uses _lock).
with _fetch_lock:
if is_fresh():
return
now = time.time()
if now - _LAST_FETCH_ATTEMPT < _FAILURE_RETRY_SECONDS:
return
_LAST_FETCH_ATTEMPT = now
mapping = fetch_emoji_map(session)
if mapping:
set_emoji_map(mapping)
@@ -0,0 +1,183 @@
"""抖音网页版「粉丝列表」拉取,用于检测新粉丝(关注欢迎语功能)。
复用与 peer_profile / account_profile 相同的 a_bogus + msToken + cookie 签名方式,
调用 https://www.douyin.com/aweme/v1/web/user/follower/list/ 拉取本账号最近的粉丝。
返回的每个粉丝含:uid / sec_uid / nickname / follow_status / follower_status。
其中 follow_status 表示「我」与对方的关系:0=未关注 1=我已关注 2=互相关注(互关)。
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any
from .dy_util import (
DEFAULT_USER_AGENT,
generate_a_bogus,
generate_msToken,
generate_webid,
splice_url,
)
from .auth import DouyinAuth
logger = logging.getLogger("douyin_im.follower_poll")
FOLLOWER_LIST_URL = "https://www.douyin.com/aweme/v1/web/user/follower/list/"
def _requests_proxies() -> dict | None:
try:
from rpa_engine.runtime_config import requests_proxies
return requests_proxies()
except Exception:
return None
def _to_int(value: Any) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def _extract_followers(data: dict[str, Any]) -> list[dict[str, Any]]:
raw = data.get("followers")
if not isinstance(raw, list):
return []
out: list[dict[str, Any]] = []
for item in raw:
if not isinstance(item, dict):
continue
uid = str(item.get("uid") or item.get("user_id") or "").strip()
if not uid:
continue
out.append(
{
"uid": uid,
"sec_uid": str(item.get("sec_uid") or item.get("sec_user_id") or "").strip(),
"nickname": str(item.get("nickname") or item.get("nick_name") or "").strip(),
# follow_status:我对对方的关系(2=互关);follower_status:对方对我的关系
"follow_status": _to_int(item.get("follow_status")),
"follower_status": _to_int(item.get("follower_status")),
}
)
return out
def fetch_recent_followers_sync(
session,
sec_user_id: str,
count: int = 20,
max_time: int = 0,
) -> list[dict[str, Any]]:
"""同步拉取最近粉丝(第一页)。失败返回 [],并在日志里写明原因。"""
import requests
sec_user_id = (sec_user_id or "").strip()
if not sec_user_id:
logger.warning("fetch followers skipped: 缺少本账号 sec_user_id")
return []
try:
auth = DouyinAuth()
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
except Exception as exc:
logger.warning("fetch followers: build auth failed: %s", exc)
return []
ua = session.user_agent or DEFAULT_USER_AGENT
params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"sec_user_id": sec_user_id,
"count": str(count),
"max_time": str(max_time),
"min_time": "0",
"offset": "0",
"source_type": "1",
"gps_access": "0",
"address_book_access": "0",
"is_top": "1",
"update_version_code": "170400",
"pc_client_type": "1",
"version_code": "170400",
"version_name": "17.4.0",
"cookie_enabled": "true",
"screen_width": "1536",
"screen_height": "960",
"browser_language": "zh-CN",
"browser_platform": "Win32",
"browser_name": "Chrome",
"browser_version": "120.0.0.0",
"browser_online": "true",
"os_name": "Windows",
"os_version": "10",
"platform": "PC",
"webid": generate_webid(auth, "https://www.douyin.com/"),
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"msToken": auth.msToken or generate_msToken(),
}
query = splice_url(params)
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
headers = {
"User-Agent": ua,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
}
try:
resp = requests.get(
FOLLOWER_LIST_URL,
params=params,
headers=headers,
cookies=auth.cookie,
timeout=15,
verify=False,
proxies=_requests_proxies(),
)
try:
data = resp.json()
except Exception:
snippet = (resp.text or "")[:200].replace("\n", " ")
logger.warning(
"fetch followers: 非 JSON 响应 (HTTP %s): %s", resp.status_code, snippet
)
return []
if not isinstance(data, dict):
logger.warning("fetch followers: 响应不是 JSON 对象")
return []
status_code = data.get("status_code")
if status_code not in (None, 0):
logger.warning(
"fetch followers: status_code=%s msg=%s",
status_code,
data.get("status_msg") or data.get("message") or "",
)
return []
followers = _extract_followers(data)
logger.info(
"fetch followers ok: 拿到 %s 个粉丝 (has_more=%s total=%s)",
len(followers),
data.get("has_more"),
data.get("total"),
)
return followers
except Exception as exc:
logger.warning("fetch followers failed: %s", exc)
return []
async def fetch_recent_followers(
session,
sec_user_id: str,
count: int = 20,
max_time: int = 0,
) -> list[dict[str, Any]]:
return await asyncio.to_thread(
fetch_recent_followers_sync, session, sec_user_id, count, max_time
)
+153
View File
@@ -0,0 +1,153 @@
"""Build frontier-im WebSocket URL (DouYin_Spider douyin_recv_msg logic)."""
import hashlib
import logging
import re
from typing import Optional
from urllib.parse import unquote
import requests
from .auth import DouyinAuth
from .dy_util import generate_a_bogus, generate_msToken, generate_webid, splice_url
from .session import DouyinImSession, is_frontier_ws_url
logger = logging.getLogger("douyin_im.frontier")
APP_KEY = "e1bd35ec9db7b8d846de66ed140b1ad9"
FP_ID = "9"
def build_frontier_ws_url(session: DouyinImSession, device_id: str) -> Optional[str]:
token = session.cookies.get("sessionid") or session.cookies.get("sessionid_ss") or ""
if not token or not device_id:
return None
access_key_raw = f"{FP_ID}{APP_KEY}{device_id}f8a69f1719916z"
access_key = hashlib.md5(access_key_raw.encode("utf-8")).hexdigest()
params = {
"aid": "6383",
"device_platform": "douyin_pc",
"fpid": FP_ID,
"device_id": device_id,
"token": token,
"access_key": access_key,
}
query = "&".join(f"{k}={v}" for k, v in params.items())
return f"wss://frontier-im.douyin.com/ws/v2?{query}"
def fetch_device_id(session: DouyinImSession) -> str:
"""Call Douyin user query API to obtain device/web id."""
auth = DouyinAuth()
auth.perepare_auth(
session.cookie_header(),
session.web_protect_str,
session.keys_str,
)
url = "https://www.douyin.com/aweme/v1/web/query/user"
headers = {
"User-Agent": session.user_agent,
"Referer": "https://www.douyin.com/discover",
"Accept": "application/json, text/plain, */*",
"Cookie": session.cookie_header(),
}
params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"publish_video_strategy_type": "2",
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"webid": generate_webid(auth, "https://www.douyin.com/discover"),
"msToken": generate_msToken(),
}
query = splice_url(params)
params["a_bogus"] = generate_a_bogus(query, user_agent=session.user_agent)
try:
resp = requests.get(
url,
params=params,
headers=headers,
cookies=auth.cookie,
verify=False,
timeout=15,
)
data = resp.json()
device_id = str(data.get("id") or data.get("device_id") or "")
if device_id.isdigit():
logger.info(f"Fetched device_id: {device_id[:20]}...")
return device_id
logger.warning(f"query/user returned non-numeric id: {device_id[:32]!r}")
except Exception as e:
logger.warning(f"fetch_device_id failed: {e}")
return ""
def resolve_frontier_device_id(session: DouyinImSession) -> str:
"""Frontier WS requires numeric device_id from Douyin query/user API."""
current = str(session.device_id or session.web_id or "")
if current.isdigit():
return current
fetched = fetch_device_id(session)
if fetched and str(fetched).isdigit():
session.device_id = str(fetched)
logger.info(f"Using numeric device_id for frontier WS: {fetched[:16]}...")
return str(fetched)
logger.warning(
f"Invalid frontier device_id={current[:24]!r}; "
"expected numeric id from query/user API"
)
return ""
def _ws_device_id(url: str) -> str:
m = re.search(r"[?&]device_id=([^&\s]+)", url or "")
return unquote(m.group(1)) if m else ""
def _ws_device_matches_session(session: DouyinImSession, url: str) -> bool:
ws_dev = _ws_device_id(url)
if not ws_dev or not ws_dev.isdigit():
return True
for candidate in (session.my_uid, session.web_id, session.device_id):
if candidate and str(candidate) == ws_dev:
return True
return False
def _ws_token_looks_encoded(url: str) -> bool:
m = re.search(r"[?&]token=([^&\s]+)", url or "")
if not m:
return False
token = unquote(m.group(1))
return len(token) >= 40 or not token.replace("_", "").replace("-", "").isalnum()
def ensure_frontier_ws(session: DouyinImSession) -> Optional[str]:
"""Ensure session has a usable frontier WebSocket URL."""
session.sanitize_ws_urls()
for url in session.ws_urls:
if is_frontier_ws_url(url) and "sdk_cert=" in url and _ws_token_looks_encoded(url):
session.ws_urls = [url]
logger.info("Using captured real frontier WS URL (with sdk_cert)")
return url
for url in session.ws_urls:
if is_frontier_ws_url(url) and _ws_token_looks_encoded(url):
session.ws_urls = [url]
logger.info("Using captured frontier WS URL")
return url
device_id = resolve_frontier_device_id(session)
if not device_id:
session.ws_urls = []
return None
built = build_frontier_ws_url(session, device_id)
if built:
session.ws_urls = [built]
logger.info("Built frontier WS URL from cookie session")
return built
return None
@@ -0,0 +1,38 @@
"""本系统当前托管中的账号 UID 注册表。
用途:避免两个都在本系统托管的账号互相自动回复,形成无限回环——
这种高频来回发送是触发抖音风控(7911)/业务拒绝(8004)的常见根因。
"""
import logging
logger = logging.getLogger("douyin_im.hosted_registry")
_HOSTED_UIDS: set[int] = set()
def register(uid) -> None:
try:
u = int(uid)
except (TypeError, ValueError):
return
if u:
_HOSTED_UIDS.add(u)
logger.info(f"Registered hosted uid {u} (total={len(_HOSTED_UIDS)})")
def unregister(uid) -> None:
try:
u = int(uid)
except (TypeError, ValueError):
return
_HOSTED_UIDS.discard(u)
logger.info(f"Unregistered hosted uid {u} (total={len(_HOSTED_UIDS)})")
def is_hosted(uid) -> bool:
try:
u = int(uid)
except (TypeError, ValueError):
return False
return u in _HOSTED_UIDS
File diff suppressed because it is too large Load Diff
+32
View File
@@ -0,0 +1,32 @@
"""Parse Douyin IM Response protobuf for conversation metadata."""
from __future__ import annotations
def extract_conv_meta_from_response_bytes(raw: bytes) -> dict[str, dict]:
"""Return {conversation_id: {conversation_short_id, ticket}} from IM API protobuf body."""
if not raw:
return {}
try:
from .static import Response_pb2 as ResponseProto
response_proto = ResponseProto.Response()
response_proto.ParseFromString(raw)
body = response_proto.body
out: dict[str, dict] = {}
for field in (
"create_conversation_v2_body",
"get_conversation_info_list_v2_response_body",
):
if not body.HasField(field):
continue
conv_body = getattr(body, field)
for conv in conv_body.conversation_info_list:
if not conv.conversation_id:
continue
out[conv.conversation_id] = {
"conversation_short_id": str(conv.conversation_short_id),
"ticket": conv.ticket,
}
return out
except Exception:
return {}
@@ -0,0 +1,720 @@
"""抖音 IM 私信图片上传。
复刻抖音 PC 网页版私信发图的真实链路(与"创作者中心 ImageX 图文图床"是两套,
IM 发送只认这条链路产出的 tos-cn-o-* 资源):
1. GET www.douyin.com/aweme/v1/web/im/upload/config/v2 (a_bogus 签名)
→ 返回内含 STS2 凭证(AccessKeyID + SignedSecretAccessKey)与 SpaceName(=zhenzhen)
2. GET vod.bytedanceapi.com/?Action=ApplyUploadInner&SpaceName=zhenzhen&FileType=image
(AWS4-HMAC-SHA256,service=vod,带 x-amz-security-token=STS2…)
→ 返回 UploadHost / StoreUri(tos-cn-o-*) / Auth(SpaceKey JWT) / SessionKey
3. POST https://{UploadHost}/upload/v1/{StoreUri}
(Authorization: SpaceKey/zhenzhen/…JWT,Content-CRC32,X-Storage-U=my_uid)
4. POST vod.bytedanceapi.com/?Action=CommitUploadInner&SpaceName=zhenzhen (AWS4 签名)
→ 确认上传,最终 uri 即 StoreUri(tos-cn-o-*)
之后用该 tos-cn-o-* uri 构造 type=27 消息体即可被 IM 后端校验通过。
"""
from __future__ import annotations
import base64
import datetime
import hashlib
import hmac
import json
import logging
import random
import re
import string
import zlib
from typing import Any
from urllib.parse import urlencode
logger = logging.getLogger("douyin_im.image_upload")
_LOCAL_URL_RE = re.compile(
r"^(/api/media/messages/|https?://(?:localhost|127\.0\.0\.1)(?::\d+)?/api/media/messages/)",
re.I,
)
# 抖音 web 私信图片上传专用通道(VOD/ImageX inner 接口 + IM 上传配置)
VOD_HOST = "https://vod.bytedanceapi.com/"
VOD_REGION = "cn-north-1"
VOD_SERVICE = "vod"
IM_UPLOAD_CONFIG_URL = "https://www.douyin.com/aweme/v1/web/im/upload/config/v2"
DEFAULT_SPACE_NAME = "zhenzhen"
def is_local_media_url(url: str) -> bool:
raw = (url or "").strip()
if not raw:
return False
if raw.startswith("/api/media/messages/"):
return True
if raw.startswith("/api/media/link-cards/"):
return True
return bool(_LOCAL_URL_RE.match(raw))
def is_douyin_cdn_url(url: str) -> bool:
raw = (url or "").strip().lower()
if not raw.startswith("http"):
return False
return any(
host in raw
for host in (
"douyinpic.com",
"byteimg.com",
"ibyteimg.com",
"douyinstatic.com",
"snssdk.com",
"vodupload.com",
)
)
def parse_local_message_media_path(url: str) -> tuple[int, str] | None:
"""从 /api/media/messages/{account_id}/{filename} 解析 account_id 与文件名。"""
raw = (url or "").strip()
m = re.search(r"/api/media/messages/(\d+)/([^/?#]+)", raw)
if not m:
return None
return int(m.group(1)), m.group(2)
def parse_link_card_media_path(url: str) -> tuple[int, str] | None:
"""从 /api/media/link-cards/{owner_id}/{filename} 解析 owner_id 与文件名。"""
raw = (url or "").strip()
m = re.search(r"/api/media/link-cards/(\d+)/([^/?#]+)", raw)
if not m:
return None
return int(m.group(1)), m.group(2)
def _requests_proxies() -> dict | None:
try:
from rpa_engine.runtime_config import requests_proxies
return requests_proxies()
except Exception:
return None
def _random_s() -> str:
chars = string.digits + string.ascii_lowercase
return "".join(random.choice(chars) for _ in range(11))
def _signing_key(secret: str, date_stamp: str, region: str, service: str) -> bytes:
k_date = hmac.new(("AWS4" + secret).encode(), date_stamp.encode(), hashlib.sha256).digest()
k_region = hmac.new(k_date, region.encode(), hashlib.sha256).digest()
k_service = hmac.new(k_region, service.encode(), hashlib.sha256).digest()
return hmac.new(k_service, b"aws4_request", hashlib.sha256).digest()
def _aws4_authorization(
*,
method: str,
canonical_querystring: str,
amz_date: str,
date_stamp: str,
session_token: str,
access_key_id: str,
secret_access_key: str,
payload_hash: str | None = None,
signed_headers: list[str] | None = None,
service: str = VOD_SERVICE,
region: str = VOD_REGION,
) -> str:
if signed_headers is None:
signed_headers = ["x-amz-date", "x-amz-security-token"]
if payload_hash is None:
payload_hash = hashlib.sha256(b"").hexdigest()
header_lines = []
for name in signed_headers:
if name == "x-amz-date":
header_lines.append(f"x-amz-date:{amz_date}\n")
elif name == "x-amz-security-token":
header_lines.append(f"x-amz-security-token:{session_token}\n")
elif name == "x-amz-content-sha256":
header_lines.append(f"x-amz-content-sha256:{payload_hash}\n")
canonical_headers = "".join(header_lines)
signed = ";".join(signed_headers)
canonical_request = (
f"{method}\n/\n{canonical_querystring}\n{canonical_headers}\n{signed}\n{payload_hash}"
)
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
string_to_sign = (
"AWS4-HMAC-SHA256\n"
f"{amz_date}\n"
f"{credential_scope}\n"
f"{hashlib.sha256(canonical_request.encode()).hexdigest()}"
)
signature = hmac.new(
_signing_key(secret_access_key, date_stamp, region, service),
string_to_sign.encode(),
hashlib.sha256,
).hexdigest()
return (
f"AWS4-HMAC-SHA256 Credential={access_key_id}/{credential_scope}, "
f"SignedHeaders={signed}, Signature={signature}"
)
def _safe_json(resp) -> dict[str, Any]:
text = (getattr(resp, "text", None) or "").strip()
if not text:
return {"error": f"空响应 (HTTP {getattr(resp, 'status_code', '?')})"}
try:
data = resp.json()
return data if isinstance(data, dict) else {"error": "响应不是 JSON 对象"}
except Exception:
snippet = text[:200].replace("\n", " ")
return {"error": f"非 JSON 响应 (HTTP {getattr(resp, 'status_code', '?')}): {snippet}"}
# ---------------------------------------------------------------------------
# 第 1 步:拉取 IM 上传配置,提取 STS 凭证 + SpaceName
# ---------------------------------------------------------------------------
def _sanitize_for_log(obj: Any, depth: int = 0) -> Any:
"""结构化脱敏:保留字段名与层级,长字符串只留前 12 位 + 长度,便于排查凭证字段。"""
if depth > 6:
return "…"
if isinstance(obj, dict):
return {str(k): _sanitize_for_log(v, depth + 1) for k, v in obj.items()}
if isinstance(obj, list):
return [_sanitize_for_log(v, depth + 1) for v in obj[:3]]
if isinstance(obj, str):
return f"{obj[:12]}…(len={len(obj)})" if len(obj) > 24 else obj
return obj
def _find_auth_object(obj: Any, depth: int = 0) -> dict | None:
"""找到「直接含有 STS2 会话凭证字符串」的那个 dict(即临时凭证对象)。
该对象里通常同时含有 AccessKeyId / SecretAccessKey / SessionToken(=STS2…)。
关键:签名要用同级的 SecretAccessKey 字段,而**不是** STS2 令牌内部解出的
SignedSecretAccessKey(那是服务端校验用的,拿来当签名密钥会 SignatureDoesNotMatch)。
"""
if depth > 8 or not isinstance(obj, (dict, list)):
return None
if isinstance(obj, dict):
for value in obj.values():
if isinstance(value, str) and value.startswith("STS2"):
return obj
for value in obj.values():
found = _find_auth_object(value, depth + 1)
if found is not None:
return found
else:
for value in obj:
found = _find_auth_object(value, depth + 1)
if found is not None:
return found
return None
def _find_sts_token(obj: Any, depth: int = 0) -> str:
"""递归在响应里找到形如 'STS2...' 的会话凭证字符串。"""
if depth > 8:
return ""
if isinstance(obj, str):
return obj if obj.startswith("STS2") else ""
if isinstance(obj, dict):
for value in obj.values():
found = _find_sts_token(value, depth + 1)
if found:
return found
elif isinstance(obj, list):
for value in obj:
found = _find_sts_token(value, depth + 1)
if found:
return found
return ""
def _find_space_name(obj: Any, depth: int = 0) -> str:
if depth > 8:
return ""
if isinstance(obj, dict):
for key, value in obj.items():
if str(key).lower() in ("space_name", "spacename") and isinstance(value, str) and value:
return value
found = _find_space_name(value, depth + 1)
if found:
return found
elif isinstance(obj, list):
for value in obj:
found = _find_space_name(value, depth + 1)
if found:
return found
return ""
def _decode_sts(sts_token: str) -> tuple[str, str]:
"""STS2<base64(JSON)>,解出 AccessKeyID 与 SignedSecretAccessKey。"""
try:
b64 = sts_token[4:] if sts_token.startswith("STS2") else sts_token
b64 += "=" * (-len(b64) % 4)
data = json.loads(base64.b64decode(b64).decode("utf-8", "ignore"))
ak = data.get("AccessKeyID") or data.get("AccessKeyId") or ""
sk = data.get("SignedSecretAccessKey") or data.get("SecretAccessKey") or ""
return ak, sk
except Exception:
return "", ""
def _fetch_im_upload_sts(session) -> tuple[str, str, str, str]:
"""返回 (access_key_id, secret_access_key, sts_token, space_name)。"""
import requests
from .auth import DouyinAuth
from .dy_util import (
DEFAULT_USER_AGENT,
generate_a_bogus,
generate_msToken,
generate_webid,
splice_url,
)
auth = DouyinAuth()
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
ua = session.user_agent or DEFAULT_USER_AGENT
params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"update_version_code": "170400",
"pc_client_type": "1",
"pc_libra_divert": "Windows",
"support_h265": "1",
"support_dash": "1",
"version_code": "170400",
"version_name": "17.4.0",
"cookie_enabled": "true",
"screen_width": "1536",
"screen_height": "960",
"browser_language": "zh-CN",
"browser_platform": "Win32",
"browser_name": "Chrome",
"browser_version": "120.0.0.0",
"browser_online": "true",
"engine_name": "Blink",
"engine_version": "120.0.0.0",
"os_name": "Windows",
"os_version": "10",
"cpu_core_num": "8",
"device_memory": "8",
"platform": "PC",
"downlink": "10",
"effective_type": "4g",
"round_trip_time": "50",
"webid": generate_webid(auth, "https://www.douyin.com/"),
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"msToken": auth.msToken or generate_msToken(),
}
query = splice_url(params)
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
headers = {
"User-Agent": ua,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
}
resp = requests.get(
IM_UPLOAD_CONFIG_URL,
params=params,
headers=headers,
cookies=auth.cookie,
timeout=20,
verify=False,
proxies=_requests_proxies(),
)
data = _safe_json(resp)
if data.get("error"):
raise RuntimeError(f"获取 IM 上传配置失败:{data['error']}")
if data.get("status_code") not in (None, 0):
raise RuntimeError(
f"获取 IM 上传配置失败:status_code={data.get('status_code')} "
f"{data.get('status_msg') or ''}"
)
# 诊断:把 config/v2 响应结构(字段名保留、长字符串脱敏)打到日志,便于核对凭证字段。
try:
logger.info(
"im/upload/config/v2 结构: %s",
json.dumps(_sanitize_for_log(data), ensure_ascii=False),
)
except Exception:
pass
# 凭证块选择(决定 8003 是否发生的关键):
# - inner_image_config / public_image_config / public_file_config 共用同一套
# 可用于 VOD 上传的 STS 令牌,仅 space 不同;
# - public_image_config_v2 的 token 不同,不兼容 VOD 内部上传(会报
# "session token sequence is broken"),必须避开。
# 空间含义:maya_review = 审核空间,图片上传后处于待审核态,作为消息发送会被拒(8003);
# zhenzhen = 公开可发送空间,IM 图片消息应使用它。
# 因此优先 public_image_config(zhenzhen),其凭证同样能完成 VOD 上传。
def _block_has_sts(block: Any) -> bool:
return isinstance(block, dict) and any(
isinstance(v, str) and v.startswith("STS2") for v in block.values()
)
auth_obj = None
for _key in ("public_image_config", "inner_image_config"):
if isinstance(data, dict) and _block_has_sts(data.get(_key)):
auth_obj = data[_key]
break
if auth_obj is None:
auth_obj = _find_auth_object(data)
if not auth_obj:
raise RuntimeError(
"IM 上传配置响应里未找到 STS 凭证(STS2 token);可能 cookie/签名失效,请用浏览器模式重新登录"
)
sts = next(
(v for v in auth_obj.values() if isinstance(v, str) and v.startswith("STS2")),
"",
)
# 优先用凭证对象同级的 AccessKeyId / SecretAccessKey(用于 SigV4 签名的真实密钥)。
ak = (
auth_obj.get("AccessKeyID")
or auth_obj.get("AccessKeyId")
or auth_obj.get("access_key_id")
or ""
)
sk = (
auth_obj.get("SecretAccessKey")
or auth_obj.get("SecretAccesskey")
or auth_obj.get("secret_access_key")
or ""
)
# 兜底:若响应没给独立的 ak/sk,再尝试从 STS2 令牌解码(SignedSecretAccessKey 一般不可用,仅最后兜底)。
if not ak or not sk:
dec_ak, dec_sk = _decode_sts(sts)
ak = ak or dec_ak
sk = sk or dec_sk
if not ak or not sk:
raise RuntimeError("解析 STS 凭证失败(缺少 AccessKeyId / SecretAccessKey)")
# space 必须取自与凭证同一块,避免凭证用 inner_image_config 而 space 误取到别处。
space = (
auth_obj.get("space_name")
or auth_obj.get("SpaceName")
or auth_obj.get("spaceName")
or _find_space_name(data)
or DEFAULT_SPACE_NAME
)
return ak, sk, sts, space
# ---------------------------------------------------------------------------
# 第 2 步:ApplyUploadInner(VOD),申请上传地址
# ---------------------------------------------------------------------------
def _extract_apply_inner(data: dict[str, Any]) -> tuple[str, str, str, str]:
"""返回 (upload_host, store_uri, jwt_auth, session_key)。"""
result = data.get("Result") or {}
addr = result.get("InnerUploadAddress") or result.get("UploadAddress") or {}
nodes = addr.get("UploadNodes") or []
if nodes:
node = nodes[0]
stores = node.get("StoreInfos") or []
store = stores[0] if stores else {}
host = node.get("UploadHost") or ""
if not host:
hosts = node.get("UploadHosts") or addr.get("UploadHosts") or []
host = hosts[0] if hosts else ""
return (
str(host or ""),
str(store.get("StoreUri") or ""),
str(store.get("Auth") or ""),
str(node.get("SessionKey") or addr.get("SessionKey") or ""),
)
hosts = addr.get("UploadHosts") or []
host = hosts[0] if hosts else ""
stores = addr.get("StoreInfos") or []
store = stores[0] if stores else {}
return (
str(host or ""),
str(store.get("StoreUri") or ""),
str(store.get("Auth") or ""),
str(addr.get("SessionKey") or result.get("SessionKey") or ""),
)
def _vod_apply_upload_inner(
ak: str, sk: str, token: str, space: str, file_size: int
) -> tuple[str, str, str, str]:
import requests
from .dy_util import DEFAULT_USER_AGENT
now = datetime.datetime.utcnow()
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
date_stamp = now.strftime("%Y%m%d")
params = {
"Action": "ApplyUploadInner",
"Version": "2020-11-19",
"SpaceName": space,
"FileType": "image",
"IsInner": "1",
"NeedFallback": "true",
"FileSize": str(file_size),
"s": _random_s(),
}
qs = urlencode(sorted(params.items()))
authorization = _aws4_authorization(
method="GET",
canonical_querystring=qs,
amz_date=amz_date,
date_stamp=date_stamp,
session_token=token,
access_key_id=ak,
secret_access_key=sk,
service=VOD_SERVICE,
)
resp = requests.get(
f"{VOD_HOST}?{qs}",
headers={
"accept": "*/*",
"authorization": authorization,
"user-agent": DEFAULT_USER_AGENT,
"x-amz-date": amz_date,
"x-amz-security-token": token,
"Referer": "https://www.douyin.com/",
},
timeout=30,
verify=False,
proxies=_requests_proxies(),
)
data = _safe_json(resp)
if data.get("error"):
raise RuntimeError(f"申请上传地址失败:{data['error']}")
meta = data.get("ResponseMetadata") or {}
err = meta.get("Error")
if err:
raise RuntimeError(f"申请上传地址失败:{err.get('Message') or err.get('Code')}")
return _extract_apply_inner(data)
# ---------------------------------------------------------------------------
# 第 3 步:上传二进制(SpaceKey JWT 鉴权)
# ---------------------------------------------------------------------------
def _vod_upload_binary(
host: str, store_uri: str, jwt_auth: str, user_id: str, raw: bytes, session=None
) -> None:
import requests
from .dy_util import DEFAULT_USER_AGENT
crc32 = format(zlib.crc32(raw) & 0xFFFFFFFF, "08x")
url = f"https://{host}/upload/v1/{store_uri}"
ua = (getattr(session, "user_agent", None) or DEFAULT_USER_AGENT)
headers = {
"Authorization": jwt_auth,
"Content-CRC32": crc32,
"Content-Type": "application/octet-stream",
"Content-Disposition": 'attachment; filename="undefined"',
"User-Agent": ua,
}
if user_id:
headers["X-Storage-U"] = str(user_id)
resp = requests.post(
url,
headers=headers,
data=raw,
timeout=60,
verify=False,
proxies=_requests_proxies(),
)
data = _safe_json(resp)
if data.get("error"):
raise RuntimeError(f"上传图片数据失败:{data['error']}")
if data.get("code") not in (2000, 0, None):
raise RuntimeError(f"上传图片数据失败:{data.get('message') or data}")
# ---------------------------------------------------------------------------
# 第 4 步:CommitUploadInner(VOD),确认上传
# ---------------------------------------------------------------------------
def _vod_commit_upload_inner(
ak: str, sk: str, token: str, space: str, session_key: str
) -> dict[str, Any]:
import requests
from .dy_util import DEFAULT_USER_AGENT
now = datetime.datetime.utcnow()
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
date_stamp = now.strftime("%Y%m%d")
params = {
"Action": "CommitUploadInner",
"Version": "2020-11-19",
"SpaceName": space,
}
qs = urlencode(sorted(params.items()))
body = json.dumps({"SessionKey": session_key, "Functions": []}, separators=(",", ":"))
payload_hash = hashlib.sha256(body.encode()).hexdigest()
signed_headers = ["x-amz-content-sha256", "x-amz-date", "x-amz-security-token"]
authorization = _aws4_authorization(
method="POST",
canonical_querystring=qs,
amz_date=amz_date,
date_stamp=date_stamp,
session_token=token,
access_key_id=ak,
secret_access_key=sk,
payload_hash=payload_hash,
signed_headers=signed_headers,
service=VOD_SERVICE,
)
resp = requests.post(
f"{VOD_HOST}?{qs}",
data=body,
headers={
"accept": "*/*",
"authorization": authorization,
"content-type": "application/json",
"user-agent": DEFAULT_USER_AGENT,
"x-amz-content-sha256": payload_hash,
"x-amz-date": amz_date,
"x-amz-security-token": token,
"Referer": "https://www.douyin.com/",
},
timeout=30,
verify=False,
proxies=_requests_proxies(),
)
data = _safe_json(resp)
if data.get("error"):
raise RuntimeError(f"确认上传失败:{data['error']}")
meta = data.get("ResponseMetadata") or {}
err = meta.get("Error")
if err:
raise RuntimeError(f"确认上传失败:{err.get('Message') or err.get('Code')}")
return data.get("Result") or {}
def upload_im_image(
session,
raw: bytes,
*,
filename: str = "image.jpg",
content_type: str = "image/jpeg",
) -> dict[str, Any]:
"""上传图片到抖音 IM 私信图床(VOD/zhenzhen 空间)。
成功返回 {uri(tos-cn-o-*), url, url_list, md5};失败返回 {"error": "..."}。
"""
del filename, content_type # VOD 按二进制上传,文件名仅用于本地存储
if not raw:
return {"error": "图片为空"}
try:
ak, sk, token, space = _fetch_im_upload_sts(session)
host, store_uri, jwt_auth, session_key = _vod_apply_upload_inner(
ak, sk, token, space, len(raw)
)
if not host or not store_uri or not jwt_auth:
return {"error": "申请上传地址失败:缺少 UploadHost/StoreUri/Auth"}
user_id = str(getattr(session, "my_uid", "") or "")
_vod_upload_binary(host, store_uri, jwt_auth, user_id, raw, session)
_vod_commit_upload_inner(ak, sk, token, space, session_key)
uri = store_uri.lstrip("/")
out: dict[str, Any] = {"uri": uri, "md5": hashlib.md5(raw).hexdigest()}
from .message_content import uri_to_cdn_urls
urls = uri_to_cdn_urls(uri)
if urls:
out["url_list"] = urls
out["url"] = urls[0]
logger.info("Uploaded IM image via VOD uri=%s host=%s space=%s", uri, host, space)
return out
except Exception as exc:
logger.warning("upload_im_image (VOD) failed: %s", exc)
return {"error": str(exc)}
def prepare_image_reply_spec(spec: dict[str, Any], session, upload_dir: str) -> tuple[dict[str, Any], str]:
"""若图片仍是本地地址,则上传到抖音 CDN 并补全 uri。返回 (spec, error)。"""
if spec.get("type") != "image":
return spec, ""
uri = str(spec.get("uri") or "").strip()
url = str(spec.get("url") or "").strip()
# 已经持有抖音 CDN 的 uri,说明图片早已上传完成(且经历过风控/转码),直接复用即可。
# 关键修复:不要因为 url 仍是本机预览地址(/api/media/...)而再次上传——
# 重复上传会拿到一个“刚提交、尚未完成风控/转码”的新 uri,发送时常被抖音以
# raw_check_code=1 / status_code=8003 拒绝(与是否互关无关)。
# 发送链路(build_msg_payload)只用 uri / url_list,从不使用这个本机 url,故本机 url 无害。
if uri and not is_local_media_url(uri):
return spec, ""
if url and is_douyin_cdn_url(url) and not uri:
return spec, ""
raw: bytes | None = None
filename = "image.jpg"
content_type = "image/jpeg"
if is_local_media_url(url):
import os
card_parsed = parse_link_card_media_path(url)
if card_parsed:
# 卡片封面在 uploads/link-cards/{owner}/ 下,与消息图片目录(uploads/messages)不同。
owner_id, fname = card_parsed
link_cards_dir = os.path.join(os.path.dirname(upload_dir), "link-cards")
path = os.path.join(link_cards_dir, str(owner_id), fname)
else:
parsed = parse_local_message_media_path(url)
if not parsed:
return spec, "无法解析本地图片路径"
_, fname = parsed
path = os.path.join(upload_dir, str(parsed[0]), fname)
if not os.path.isfile(path):
return spec, f"本地图片不存在:{fname}"
with open(path, "rb") as f:
raw = f.read()
filename = fname
if fname.lower().endswith(".png"):
content_type = "image/png"
elif fname.lower().endswith(".webp"):
content_type = "image/webp"
elif fname.lower().endswith(".gif"):
content_type = "image/gif"
if raw is None:
if url.startswith("http") and not is_douyin_cdn_url(url):
return spec, "图片地址必须是抖音 CDN 或本地上传后的地址,外部 URL 无法用于 IM 发送"
return spec, "缺少可上传的图片数据"
uploaded = upload_im_image(session, raw, filename=filename, content_type=content_type)
if uploaded.get("error"):
return spec, uploaded["error"]
if not uploaded.get("uri"):
return spec, "抖音图片上传失败:未返回 uri"
merged = {
**spec,
**uploaded,
"type": "image",
"text": spec.get("text") or "[图片]",
}
return merged, ""
@@ -0,0 +1,746 @@
"""私信消息内容解析、存储与展示(文本 / 图片 / 表情 / 语音 / 视频)。"""
from __future__ import annotations
import json
import re
from typing import Any
MSG_TYPE_TEXT = 7
MSG_TYPE_STICKER = 5
MSG_TYPE_VOICE = 17
MSG_TYPE_IMAGE = 27
MSG_TYPE_VIDEO = 8
MSG_TYPE_LINK_CARD = 70
_TYPE_LABELS = {
"text": "文本",
"image": "图片",
"sticker": "表情",
"voice": "语音",
"video": "视频",
"link": "链接",
"link_card": "链接卡片",
}
_PLACEHOLDER_MARKERS = {
"[表情包]",
"[语音]",
"[图片]",
"[视频]",
"[未读消息]",
}
_URI_HINT_RE = re.compile(
r"(tos-cn|aweme-|voice/|ies-music|\.mp3|\.m4a|\.aac|\.mpeg|\.webp|\.jpeg|\.jpg|\.png|\.gif)",
re.I,
)
_AUDIO_URL_RE = re.compile(
r"(douyin-user-audio|/audio/|sc=audio|voice/|ies-music|\.mp3|\.m4a|\.aac|\.mpeg|\.wav|\.ogg)",
re.I,
)
_VIDEO_URL_RE = re.compile(
r"(sc=video|/video/|\.mp4|\.mov|\.webm|\.m3u8)",
re.I,
)
_MEDIA_STRING_KEYS = (
"url",
"uri",
"main_url",
"download_url",
"remote_url",
"encrypt_url",
"play_url",
"secret_url",
"audio_url",
"video_url",
"cover_url",
"local_path",
)
_MEDIA_NESTED_KEYS = (
"resource_url",
"static_url",
"animate_url",
"cover_url",
"thumb_url",
"origin_url",
"image",
"picture",
"pic",
"sticker",
"emoji",
"audio",
"voice",
"video",
"media",
"large_url",
"medium_url",
"thumb",
"avatar_thumb",
"play_url",
)
def _normalize_uri_path(raw: str) -> str:
path = (raw or "").strip().lstrip("/")
if path.startswith("obj/"):
path = path[4:]
return path
def uri_to_cdn_urls(uri: str, *, prefer_voice: bool = False) -> list[str]:
"""将抖音 IM 中的 uri / tos 路径转为可访问的 CDN URL 候选列表。"""
raw = (uri or "").strip()
if not raw:
return []
if raw.startswith("//"):
return [f"https:{raw}"]
if raw.startswith("http://") or raw.startswith("https://"):
return [raw]
path = _normalize_uri_path(raw)
if not path:
return []
candidates: list[str] = []
seen: set[str] = set()
def add(url: str) -> None:
url = (url or "").strip()
if url and url not in seen:
seen.add(url)
candidates.append(url)
lower = path.lower()
is_voice = prefer_voice or lower.startswith("voice/") or lower.endswith((".mp3", ".m4a", ".aac"))
is_image = (
not is_voice
and (
"tos-cn-i" in lower
or "aweme-" in lower
or lower.endswith((".jpeg", ".jpg", ".png", ".webp", ".gif"))
)
)
if is_voice:
for host in (
"sf6-cdn-tos.douyinstatic.com",
"sf3-cdn-tos.douyinstatic.com",
"lf3-static.bytednsdoc.com",
):
add(f"https://{host}/obj/{path}")
add(f"https://p3.douyinpic.com/obj/{path}")
if is_image or "tos-cn" in lower or "aweme" in lower:
add(f"https://p3.douyinpic.com/obj/{path}")
for size in ("720x720", "480x480", "300x300", "200x200", "100x100"):
add(f"https://p3.douyinpic.com/aweme/{size}/{path}")
add(f"https://p9-dy.byteimg.com/img/{path}")
add(f"https://p6-dy.byteimg.com/img/{path}")
add(f"https://p3.douyinpic.com/obj/{path}")
add(f"https://p3-sign.douyinpic.com/obj/{path}".replace("-sign", ""))
return candidates
def resolve_media_uri(uri: str, *, prefer_voice: bool = False) -> str:
urls = uri_to_cdn_urls(uri, prefer_voice=prefer_voice)
return urls[0] if urls else ""
def _looks_like_audio_url(value: str) -> bool:
raw = (value or "").strip()
return bool(raw and _AUDIO_URL_RE.search(raw))
def _looks_like_video_url(value: str) -> bool:
raw = (value or "").strip()
return bool(raw and _VIDEO_URL_RE.search(raw))
def _infer_media_type_from_url(url: str) -> str:
"""根据 URL 特征推断媒体类型(语音/视频/图片)。"""
if _looks_like_audio_url(url):
return "voice"
if _looks_like_video_url(url):
return "video"
return "image"
def _looks_like_media_uri(value: str) -> bool:
raw = (value or "").strip()
if not raw or raw.startswith("{"):
return False
if raw.startswith("http://") or raw.startswith("https://") or raw.startswith("//"):
return True
return bool(_URI_HINT_RE.search(raw))
def _resolve_string_media(value: str, *, prefer_voice: bool = False) -> tuple[str, str]:
raw = (value or "").strip()
if not raw:
return "", ""
if raw.startswith("//"):
return f"https:{raw}", raw
if raw.startswith("http://") or raw.startswith("https://"):
return raw, ""
if _looks_like_media_uri(raw):
return resolve_media_uri(raw, prefer_voice=prefer_voice), raw
return "", ""
def _valid_sticker_id(value: Any) -> str:
if value is None or value == "":
return ""
try:
if int(value) == 0:
return ""
except (TypeError, ValueError):
pass
sticker_id = str(value).strip()
return "" if sticker_id in ("0", "null", "None") else sticker_id
def _collect_media_candidates(value: Any, out: list[str], *, prefer_voice: bool = False, depth: int = 0) -> None:
if depth > 12:
return
if isinstance(value, str):
raw = value.strip()
if raw.startswith("http://") or raw.startswith("https://") or raw.startswith("//"):
url, _ = _resolve_string_media(raw, prefer_voice=prefer_voice)
if url:
out.append(url)
return
if isinstance(value, dict):
for list_key in ("url_list", "urls", "urlList"):
urls = value.get(list_key)
if isinstance(urls, list):
for item in urls:
_collect_media_candidates(item, out, prefer_voice=prefer_voice, depth=depth + 1)
for key in _MEDIA_STRING_KEYS:
direct = value.get(key)
if isinstance(direct, str):
url, _ = _resolve_string_media(direct, prefer_voice=prefer_voice)
if url:
out.append(url)
for nested_key in _MEDIA_NESTED_KEYS:
_collect_media_candidates(value.get(nested_key), out, prefer_voice=prefer_voice, depth=depth + 1)
for nested in value.values():
if isinstance(nested, (dict, list)):
_collect_media_candidates(nested, out, prefer_voice=prefer_voice, depth=depth + 1)
return
if isinstance(value, list):
for item in value:
_collect_media_candidates(item, out, prefer_voice=prefer_voice, depth=depth + 1)
def _pick_http_url(value: Any, *, prefer_voice: bool = False) -> str:
candidates: list[str] = []
_collect_media_candidates(value, candidates, prefer_voice=prefer_voice)
return candidates[0] if candidates else ""
def _pick_media_uri(value: Any) -> str:
if isinstance(value, str) and _looks_like_media_uri(value) and not value.strip().startswith("http"):
return _normalize_uri_path(value)
if isinstance(value, dict):
for key in ("uri", "local_path", "remote_url"):
direct = value.get(key)
if isinstance(direct, str) and _looks_like_media_uri(direct) and not direct.strip().startswith("http"):
return _normalize_uri_path(direct)
for nested_key in _MEDIA_NESTED_KEYS:
uri = _pick_media_uri(value.get(nested_key))
if uri:
return uri
for nested in value.values():
if isinstance(nested, (dict, list)):
uri = _pick_media_uri(nested)
if uri:
return uri
if isinstance(value, list):
for item in value:
uri = _pick_media_uri(item)
if uri:
return uri
return ""
def _coerce_message_type(value: Any, default: int = MSG_TYPE_TEXT) -> int:
try:
if value is None or value == "":
return default
return int(value)
except (TypeError, ValueError):
return default
def _parse_content_json(content_raw: str | dict | None) -> dict[str, Any]:
if isinstance(content_raw, dict):
data = content_raw
else:
raw = str(content_raw or "").strip()
if raw.startswith("{"):
try:
parsed = json.loads(raw)
if isinstance(parsed, dict):
data = parsed
else:
return {}
except json.JSONDecodeError:
return {}
else:
return {}
for nested_key in ("ext", "ai_ext", "extra", "payload", "data"):
nested = data.get(nested_key)
if isinstance(nested, str) and nested.strip().startswith("{"):
try:
nested_data = json.loads(nested)
if isinstance(nested_data, dict):
merged = {**nested_data, **data}
data = merged
except json.JSONDecodeError:
pass
return data
def _safe_int(value: Any) -> int | None:
try:
if value is None or value == "":
return None
return int(value)
except (TypeError, ValueError):
return None
def format_im_message(content_raw: str | dict | None, message_type: int = MSG_TYPE_TEXT) -> dict[str, Any]:
"""将 IM 原始 content 解析为统一结构 {type, text, url, ...}。"""
content_json = _parse_content_json(content_raw)
embedded_type = _coerce_message_type(
content_json.get("message_type")
or content_json.get("messageType")
or content_json.get("msg_type"),
message_type,
)
if embedded_type != MSG_TYPE_TEXT:
message_type = embedded_type
_EMPTY_MEDIA_DEFAULTS = {
MSG_TYPE_IMAGE: ("image", "[图片]"),
MSG_TYPE_STICKER: ("sticker", "[表情包]"),
MSG_TYPE_VOICE: ("voice", "[语音]"),
MSG_TYPE_VIDEO: ("video", "[视频]"),
MSG_TYPE_LINK_CARD: ("link_card", "[链接卡片]"),
}
if not content_json:
raw = str(content_raw or "").strip()
if not raw:
# 抖音相册图片(type 27)/部分语音等会以「空 content」推送,URL 不随推送下发。
# 不能直接丢弃,否则消息「收不到」;这里按类型返回占位,URL 留空待后续补取。
if message_type in _EMPTY_MEDIA_DEFAULTS:
t, txt = _EMPTY_MEDIA_DEFAULTS[message_type]
return {"type": t, "text": txt}
return {"type": "text", "text": ""}
if raw in _PLACEHOLDER_MARKERS:
mapping = {
"[图片]": "image",
"[表情包]": "sticker",
"[语音]": "voice",
"[视频]": "video",
}
return {"type": mapping.get(raw, "text"), "text": raw}
if message_type == MSG_TYPE_TEXT:
return {"type": "text", "text": raw}
content_json = {"text": raw}
prefer_voice = message_type == MSG_TYPE_VOICE or bool(content_json.get("audio") or content_json.get("voice"))
url = _pick_http_url(content_json, prefer_voice=prefer_voice)
media_uri = _pick_media_uri(content_json)
if not url and media_uri:
url = resolve_media_uri(media_uri, prefer_voice=prefer_voice)
duration = _safe_int(
content_json.get("duration")
or content_json.get("audio_duration")
or content_json.get("video_duration")
)
width = _safe_int(content_json.get("width") or content_json.get("w"))
height = _safe_int(content_json.get("height") or content_json.get("h"))
def _media_payload(msg_type: str, text: str, **extra: Any) -> dict[str, Any]:
payload: dict[str, Any] = {"type": msg_type, "text": text}
if url:
payload["url"] = url
elif media_uri:
payload["uri"] = media_uri
payload.update({k: v for k, v in extra.items() if v not in (None, "", 0)})
return payload
# 语音/视频也会带 resource_url,不能仅凭该字段判为图片;优先按类型与 URL 特征识别。
is_voice = (
message_type == MSG_TYPE_VOICE
or bool(content_json.get("audio") or content_json.get("voice"))
or (url and _looks_like_audio_url(url))
)
is_video = (
message_type == MSG_TYPE_VIDEO
or bool(content_json.get("video"))
or (url and _looks_like_video_url(url))
)
if is_voice and not is_video:
return _media_payload("voice", "[语音]", duration=duration)
if is_video:
return _media_payload("video", "[视频]", duration=duration, width=width, height=height)
has_image_hint = (
message_type == MSG_TYPE_IMAGE
or content_json.get("image")
or content_json.get("inline_pic")
or (
content_json.get("resource_url")
and not (url and (_looks_like_audio_url(url) or _looks_like_video_url(url)))
)
)
if has_image_hint:
# 抖音相册私图(biz_tag=aweme_im)的大图 URL 是加密内容,浏览器无法直接渲染;
# 但 content 内嵌 inline_pic(base64 WEBP 缩略图),直接转 data URI 即可显示。
inline = content_json.get("inline_pic")
if isinstance(inline, str) and inline.strip():
b64 = re.sub(r"\s+", "", inline)
data_uri = f"data:image/webp;base64,{b64}"
payload = {"type": "image", "text": "[图片]", "url": data_uri}
if width:
payload["width"] = width
if height:
payload["height"] = height
return payload
return _media_payload("image", "[图片]", width=width, height=height)
if message_type == MSG_TYPE_STICKER or content_json.get("static_url") or content_json.get("animate_url") or _valid_sticker_id(
content_json.get("sticker_id") or content_json.get("id")
):
sticker_id = _valid_sticker_id(content_json.get("sticker_id") or content_json.get("id"))
return _media_payload(
"sticker",
"[表情包]",
sticker_id=sticker_id,
name=str(content_json.get("display_name") or content_json.get("name") or ""),
)
link_card = _parse_link_card_payload(content_json, message_type)
if link_card:
return link_card
rich_link = _parse_rich_text_link(content_json)
if rich_link:
return rich_link
text = (
str(content_json.get("text") or content_json.get("content") or content_json.get("message") or "")
).strip()
if not text and url:
inferred = _infer_media_type_from_url(url)
if inferred == "voice":
return {"type": "voice", "text": "[语音]", "url": url, "duration": duration}
if inferred == "video":
return {"type": "video", "text": "[视频]", "url": url, "duration": duration}
if message_type == MSG_TYPE_IMAGE:
return {"type": "image", "text": "[图片]", "url": url, "width": width, "height": height}
if message_type == MSG_TYPE_STICKER:
return {"type": "sticker", "text": "[表情包]", "url": url}
if message_type == MSG_TYPE_VOICE:
return {"type": "voice", "text": "[语音]", "url": url, "duration": duration}
if message_type == MSG_TYPE_VIDEO:
return {"type": "video", "text": "[视频]", "url": url, "duration": duration}
final_text = text or str(content_raw or "").strip()
emoji = _resolve_text_emoji(final_text)
if emoji:
return emoji
return {"type": "text", "text": final_text}
def _parse_link_card_payload(content_json: dict[str, Any], message_type: int) -> dict[str, Any] | None:
link_info = content_json.get("link_info")
if not isinstance(link_info, dict):
link_info = {}
has_link = (
message_type == MSG_TYPE_LINK_CARD
or link_info
or content_json.get("link_url")
or content_json.get("cover_url")
)
if not has_link:
return None
title = str(content_json.get("title") or link_info.get("title") or "").strip()
desc = str(
content_json.get("desc")
or content_json.get("description")
or link_info.get("desc")
or link_info.get("description")
or ""
).strip()
url = str(
content_json.get("link_url")
or content_json.get("url")
or link_info.get("url")
or link_info.get("link_url")
or ""
).strip()
cover = str(content_json.get("cover_url") or link_info.get("cover_url") or "").strip()
text = title or desc or url or "[链接卡片]"
payload: dict[str, Any] = {
"type": "link_card",
"text": text,
"title": title,
"desc": desc,
}
if url:
payload["url"] = url
if cover:
payload["cover_url"] = cover
return payload
def _parse_rich_text_link(content_json: dict[str, Any]) -> dict[str, Any] | None:
rich = content_json.get("richTextInfos")
if not isinstance(rich, list):
return None
for item in rich:
if not isinstance(item, dict):
continue
link = str(item.get("link") or item.get("url") or "").strip()
if not link:
continue
text = str(item.get("text") or item.get("display_text") or link).strip()
return {"type": "link", "text": text or link, "url": link}
return None
def _resolve_text_emoji(text: str) -> dict[str, Any] | None:
"""文字表情 [酷拽] 等:查标准表情表,命中则转成可显示的贴纸。"""
stripped = (text or "").strip()
if not stripped or not (stripped.startswith("[") and stripped.endswith("]")):
return None
try:
from .emoji_pack import looks_like_emoji_token, lookup_emoji_url
if not looks_like_emoji_token(stripped):
return None
url = lookup_emoji_url(stripped)
if url:
return {
"type": "sticker",
"text": stripped,
"url": url,
"name": stripped.strip("[]"),
}
except Exception:
pass
return None
def parse_incoming_message(data: dict[str, Any]) -> str:
"""从 IM API / WebSocket 消息 dict 提取并序列化展示内容。"""
if not isinstance(data, dict):
return str(data or "").strip()
msg_type = _coerce_message_type(
data.get("message_type") or data.get("messageType") or data.get("msg_type"),
MSG_TYPE_TEXT,
)
content_raw = (
data.get("content")
or data.get("message")
or data.get("msg")
or data.get("lastMessage")
or data.get("last_msg")
or data.get("preview")
or data.get("brief")
or ""
)
if isinstance(content_raw, dict):
if not msg_type or msg_type == MSG_TYPE_TEXT:
msg_type = _coerce_message_type(
content_raw.get("message_type")
or content_raw.get("messageType")
or content_raw.get("msg_type"),
msg_type,
)
parsed = format_im_message(content_raw, msg_type)
return serialize_message_content(parsed)
if isinstance(content_raw, str):
raw = content_raw.strip()
if raw.startswith("{"):
parsed = format_im_message(raw, msg_type)
if parsed.get("type") != "text" or parsed.get("url") or msg_type != MSG_TYPE_TEXT:
return serialize_message_content(parsed)
if raw in _PLACEHOLDER_MARKERS and msg_type != MSG_TYPE_TEXT:
parsed = format_im_message(raw, msg_type)
return serialize_message_content(parsed)
if msg_type != MSG_TYPE_TEXT:
parsed = format_im_message(raw, msg_type)
return serialize_message_content(parsed)
return raw
return ""
def serialize_message_content(msg: dict[str, Any]) -> str:
"""序列化写入 message_logs / reply_content。"""
msg_type = (msg.get("type") or "text").strip()
if msg_type == "text":
text = str(msg.get("text") or "").strip()
return text
cleaned = {k: v for k, v in msg.items() if v not in (None, "", [], {})}
return json.dumps(cleaned, ensure_ascii=False, separators=(",", ":"))
def parse_stored_content(raw: str | None) -> dict[str, Any]:
"""解析数据库中的 message_content / reply_content。"""
text = (raw or "").strip()
if not text:
return {"type": "text", "text": ""}
if text.startswith("{"):
try:
data = json.loads(text)
if isinstance(data, dict) and data.get("type"):
if not data.get("url") and data.get("uri"):
prefer_voice = data.get("type") == "voice"
resolved = resolve_media_uri(str(data["uri"]), prefer_voice=prefer_voice)
if resolved:
data = {**data, "url": resolved}
url = str(data.get("url") or "").strip()
if url:
inferred = _infer_media_type_from_url(url)
current = data.get("type")
if current == "image" and inferred in ("voice", "video"):
data = {
**data,
"type": inferred,
"text": "[语音]" if inferred == "voice" else "[视频]",
}
elif current not in ("voice", "video", "image", "sticker") and inferred:
data = {**data, "type": inferred}
return data
except json.JSONDecodeError:
pass
if text in _PLACEHOLDER_MARKERS:
mapping = {
"[图片]": "image",
"[表情包]": "sticker",
"[语音]": "voice",
"[视频]": "video",
}
return {"type": mapping.get(text, "text"), "text": text}
return {"type": "text", "text": text}
def message_preview(raw: str | None) -> str:
"""会话列表/日志摘要。"""
msg = parse_stored_content(raw)
msg_type = msg.get("type") or "text"
if msg_type == "text":
return str(msg.get("text") or "")
label = _TYPE_LABELS.get(msg_type, msg.get("text") or "[消息]")
extra = str(msg.get("name") or "").strip()
if extra and msg_type == "sticker":
return f"[表情] {extra}"
return str(msg.get("text") or label)
def normalize_outgoing_content(
content: str = "",
message_type: str | None = None,
media_url: str | None = None,
sticker_url: str | None = None,
width: int | None = None,
height: int | None = None,
sticker_id: str | None = None,
) -> str:
"""构造可发送/可落库的 content 字符串。"""
raw = (content or "").strip()
parsed_json: dict[str, Any] | None = None
if raw.startswith("{"):
try:
data = json.loads(raw)
if isinstance(data, dict) and data.get("type"):
parsed_json = data
except json.JSONDecodeError:
pass
if parsed_json:
merged = dict(parsed_json)
if width and not merged.get("width"):
merged["width"] = width
if height and not merged.get("height"):
merged["height"] = height
if sticker_id and not merged.get("sticker_id"):
merged["sticker_id"] = sticker_id
url = (media_url or sticker_url or "").strip()
if url and not merged.get("url"):
merged["url"] = url
return serialize_message_content(merged)
explicit_type = (message_type or "").strip().lower()
if explicit_type in ("image", "sticker", "voice", "video", "text"):
if explicit_type == "text":
return (content or "").strip()
payload: dict[str, Any] = {"type": explicit_type}
url = (media_url or sticker_url or "").strip()
if url:
payload["url"] = url
if explicit_type == "sticker" and sticker_id:
payload["sticker_id"] = sticker_id
if width:
payload["width"] = width
if height:
payload["height"] = height
text = (content or "").strip()
if text:
payload["text"] = text
elif explicit_type == "image":
payload["text"] = "[图片]"
elif explicit_type == "sticker":
payload["text"] = "[表情包]"
return serialize_message_content(payload)
raw = (content or "").strip()
if raw.startswith("{"):
try:
data = json.loads(raw)
if isinstance(data, dict) and data.get("type"):
return serialize_message_content(data)
except json.JSONDecodeError:
pass
return raw
def is_media_message(raw: str | None) -> bool:
return parse_stored_content(raw).get("type") not in (None, "text")
def format_system_log_message(raw: str | None) -> str:
"""系统诊断日志中的消息摘要。"""
msg = parse_stored_content(raw)
msg_type = msg.get("type") or "text"
if msg_type == "text":
return str(msg.get("text") or "")
parts = [message_preview(raw)]
url = str(msg.get("url") or "").strip()
if url:
parts.append(f"URL: {url}")
duration = msg.get("duration")
if duration:
parts.append(f"时长: {duration}s")
return " | ".join(parts)
def extract_urls_from_detail(detail: str | None) -> list[str]:
if not detail:
return []
return re.findall(r"https?://[^\s\]|))\"']+", detail)
+258
View File
@@ -0,0 +1,258 @@
"""极简 protobuf wire 解码器(无第三方依赖)。
用于解析抖音 IM 发送私信的响应:官方 Response.proto 只建模了
create/get_info/new_message_notify 三种 body,没有“发送消息响应”,
导致仅凭 error_desc 为空就误判为发送成功。这里直接按 wire 格式解码,
读取真实的 status_code / server_message_id,判定是否真的投递成功。
"""
import json
from typing import Any, Optional
def _format_status_json(status_json: dict) -> str:
"""把内嵌 status JSON 格式化为更可读的失败原因。"""
code = status_json.get("status_code")
raw_check = status_json.get("raw_check_code")
decision = status_json.get("decision_type")
parts = [f"status_code={code}"]
if raw_check is not None:
parts.append(f"raw_check_code={raw_check}")
if decision:
parts.append(f"decision_type={decision}")
return ";".join(parts)
def _read_varint(buf: bytes, i: int) -> tuple[int, int]:
shift = 0
result = 0
n = len(buf)
while i < n:
b = buf[i]
i += 1
result |= (b & 0x7F) << shift
if not (b & 0x80):
return result, i
shift += 7
if shift > 70:
break
raise ValueError("truncated varint")
def decode_fields(buf: bytes) -> list[tuple[int, int, Any]]:
"""返回 [(field_num, wire_type, value), ...]。
wire_type: 0=varint(int), 1=64bit(int), 2=length-delimited(bytes), 5=32bit(int)
"""
out: list[tuple[int, int, Any]] = []
i = 0
n = len(buf)
while i < n:
key, i = _read_varint(buf, i)
field = key >> 3
wt = key & 7
if wt == 0:
val, i = _read_varint(buf, i)
out.append((field, wt, val))
elif wt == 2:
ln, i = _read_varint(buf, i)
val = buf[i:i + ln]
i += ln
out.append((field, wt, val))
elif wt == 5:
val = int.from_bytes(buf[i:i + 4], "little")
i += 4
out.append((field, wt, val))
elif wt == 1:
val = int.from_bytes(buf[i:i + 8], "little")
i += 8
out.append((field, wt, val))
else:
raise ValueError(f"unsupported wire type {wt}")
return out
def _collect_big_varints(buf: bytes, acc: list[int], depth: int = 0) -> None:
"""递归收集疑似 ID 的大整数(server_message_id / short_id 等都是大数)。"""
if depth > 6:
return
try:
fields = decode_fields(buf)
except Exception:
return
for _field, wt, val in fields:
if wt == 0 and isinstance(val, int) and val > 10 ** 12:
acc.append(val)
elif wt == 2 and isinstance(val, (bytes, bytearray)) and val:
_collect_big_varints(bytes(val), acc, depth + 1)
def _extract_status_json(raw: bytes) -> Optional[dict]:
"""抖音发送响应的 body 内嵌一段 JSON:{"status_code":x,"tips":"...","status_msg":{...}}。
这才是“消息是否真正投递”的权威结论(status_code=0 才是真成功)。
顶层 message=OK 只是接口层面的“已受理”,不代表已投递。
"""
marker = b'"status_code"'
idx = raw.find(marker)
if idx < 0:
return None
start = raw.rfind(b"{", 0, idx)
if start < 0:
return None
depth = 0
in_str = False
esc = False
end = -1
for i in range(start, len(raw)):
c = raw[i]
if in_str:
if esc:
esc = False
elif c == 0x5C: # backslash
esc = True
elif c == 0x22: # quote
in_str = False
continue
if c == 0x22:
in_str = True
elif c == 0x7B: # {
depth += 1
elif c == 0x7D: # }
depth -= 1
if depth == 0:
end = i + 1
break
if end < 0:
return None
try:
return json.loads(raw[start:end].decode("utf-8", "ignore"))
except Exception:
return None
def analyze_send_response(raw: bytes) -> dict:
"""分析“发送私信”的 protobuf 响应,判定是否真正投递成功。
顶层 Response 字段(见 Response.proto):
1=cmd, 2=sequence_id, 3=error_desc(string), 4=message(string),
5=inbox_type, 6=body(ResponseBody)。
注意:顶层没有 status_code 字段。真正“消息已写入服务端”的标志是
body(field 6) 里带有服务端分配的 server_message_id(大整数)。
sequence_id(field 2) 也是大整数,因此只在 body 内部查找 message_id,
避免把 sequence_id 误当成投递成功标志。
返回:
ok: 是否真正发送成功(body 内带服务端 message_id,或 message=OK 且有 body)
cmd: 顶层 cmd
message: 顶层 message 文本(field 4)
error_desc: 顶层 error_desc 文本(field 3)
server_message_id: body 内服务端消息 ID(投递成功的强信号)
has_body: 是否带 body
summary: 顶层字段概览 + hex 片段,便于排查
"""
info = {
"ok": False,
"cmd": None,
"status": None,
"status_code": None,
"raw_check_code": None,
"delivered_with_notice": False,
"status_reason": "",
"message": "",
"error_desc": "",
"server_message_id": None,
"has_body": False,
"summary": "",
}
if not raw:
info["summary"] = "空响应"
return info
try:
fields = decode_fields(raw)
except Exception as e:
info["summary"] = f"解码失败: {e}; hex={raw[:120].hex()}"
return info
body = None
parts = []
for field, wt, val in fields:
if field == 1 and wt == 0:
info["cmd"] = val
elif field == 3 and wt == 2:
try:
info["error_desc"] = bytes(val).decode("utf-8", "ignore")
except Exception:
pass
elif field == 4 and wt == 2:
try:
info["message"] = bytes(val).decode("utf-8", "ignore")
except Exception:
pass
elif field == 6 and wt == 2:
body = bytes(val)
info["has_body"] = len(body) > 0
if wt == 0:
parts.append(f"{field}=int:{val}")
elif wt == 2:
parts.append(f"{field}=bytes[{len(val)}]")
else:
parts.append(f"{field}={val}")
info["summary"] = " ".join(parts) + f" | hex={raw[:120].hex()}"
# 只在 body 内部查找服务端 message_id(避免误用顶层 sequence_id)
if body:
ids: list[int] = []
_collect_big_varints(body, ids)
if ids:
info["server_message_id"] = max(ids)
# 权威结论:body 内嵌 JSON 的 status_code(0 才是真成功)
status_json = _extract_status_json(raw)
if status_json is not None:
info["status_code"] = status_json.get("status_code")
info["raw_check_code"] = status_json.get("raw_check_code")
tips = (status_json.get("tips") or "").strip()
status_msg = status_json.get("status_msg")
msg_text = ""
if isinstance(status_msg, dict):
# 抖音把人类可读提示放在 status_msg.msg_content.tips
mc = status_msg.get("msg_content")
if isinstance(mc, dict):
msg_text = (mc.get("tips") or mc.get("content") or "").strip()
if not msg_text:
msg_text = (
status_msg.get("toast")
or status_msg.get("content")
or status_msg.get("msg")
or ""
)
elif isinstance(status_msg, str):
msg_text = status_msg
info["status_reason"] = tips or msg_text or _format_status_json(status_json)
msg_ok = info["message"].strip().upper() == "OK"
# 优先用 status_code 判定:明确给了 status_code 就以它为准(0=成功,非0=另判)
if info["status_code"] is not None:
if info["status_code"] == 0 and not info["error_desc"]:
info["ok"] = True
elif info["raw_check_code"] == 0 and msg_ok and not info["error_desc"]:
# raw_check_code=0 表示已通过抖音风控/安全校验;配合 message=OK,
# 说明消息已实际投递。此时非零 status_code 只是“业务侧提示”
# (如营销/陌生人限制提醒),对方仍能收到,不应判为发送失败。
info["ok"] = True
info["delivered_with_notice"] = True
else:
# raw_check_code=1(被风控拦截)或缺少 OK 标志:判为未送达
info["ok"] = False
else:
# 没有内嵌 status_code 时,退回“message=OK 且 body 内有服务端 message_id”
info["ok"] = bool(
not info["error_desc"]
and msg_ok
and info["server_message_id"] is not None
)
return info
@@ -0,0 +1,225 @@
"""私信对方用户资料抓取(昵称 / 头像 / UID)。"""
from __future__ import annotations
import asyncio
import logging
import time
from typing import Any, Optional
import requests
from rpa_engine.device_profiles import resolve_user_agent
from .auth import DouyinAuth
from .conv_util import resolve_peer_uid
from .dy_util import (
DEFAULT_USER_AGENT,
generate_a_bogus,
generate_msToken,
generate_webid,
splice_url,
)
from .protocol import _pick_avatar_url
from .session import DouyinImSession
logger = logging.getLogger("douyin_im.peer_profile")
_profile_cache: dict[str, dict[str, str]] = {}
_profile_cache_at: dict[str, float] = {}
_PROFILE_SUCCESS_TTL = 6 * 3600
_PROFILE_FAILURE_TTL = 5 * 60
def _cache_key(account_id: int, peer_uid: str) -> str:
return f"{account_id}:{peer_uid}"
def _pick_str(data: dict, *keys: str) -> str:
for key in keys:
value = data.get(key)
if value is not None and str(value).strip():
return str(value).strip()
return ""
def _extract_profile_from_payload(data: Any) -> dict[str, str]:
if not isinstance(data, dict):
return {}
nodes = [data, data.get("user"), data.get("user_info"), data.get("data")]
for node in nodes:
if not isinstance(node, dict):
continue
uid = _pick_str(node, "uid", "user_id", "user_uid", "id")
nickname = _pick_str(
node,
"nickname",
"nick_name",
"unique_id",
"display_name",
"name",
)
avatar = _pick_avatar_url(node)
if uid or nickname or avatar:
return {"uid": uid, "nickname": nickname, "avatar_url": avatar}
return {}
def _requests_proxies() -> dict | None:
try:
from rpa_engine.runtime_config import requests_proxies
return requests_proxies()
except Exception:
return None
def _build_auth(session: DouyinImSession) -> tuple[DouyinAuth, str]:
auth = DouyinAuth()
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
ua = resolve_user_agent(session.user_agent or DEFAULT_USER_AGENT)
return auth, ua
def is_generic_peer_name(name: str, peer_uid: str = "") -> bool:
value = (name or "").strip()
if not value:
return True
if peer_uid and value == peer_uid:
return True
if value.isdigit():
return True
if value.startswith("用户") and value[2:].isdigit():
return True
if value.startswith("会话") and len(value) <= 16:
return True
return False
def fetch_peer_profile_sync(
session: DouyinImSession,
peer_uid: int | str,
account_id: int = 0,
) -> dict[str, str]:
uid = str(peer_uid or "").strip()
if not uid.isdigit():
return {}
cache_key = _cache_key(account_id, uid)
cached = _profile_cache.get(cache_key)
cached_at = _profile_cache_at.get(cache_key, 0.0)
if cached:
ttl = (
_PROFILE_SUCCESS_TTL
if cached.get("nickname") or cached.get("avatar_url")
else _PROFILE_FAILURE_TTL
)
if time.time() - cached_at < ttl:
return dict(cached)
result = {"uid": uid, "nickname": "", "avatar_url": ""}
try:
auth, ua = _build_auth(session)
except Exception as exc:
logger.warning(f"build auth for peer profile failed: {exc}")
_profile_cache[cache_key] = dict(result)
_profile_cache_at[cache_key] = time.time()
return result
try:
web_id = session.web_id or generate_webid(auth, "https://www.douyin.com/")
if web_id and not session.web_id:
# Reuse the homepage-derived ID for every peer on this account.
session.web_id = str(web_id)
except Exception as exc:
logger.debug(f"generate webid for peer profile failed: {exc}")
web_id = session.web_id or ""
headers = {
"User-Agent": ua,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
}
base_params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"publish_video_strategy_type": "2",
"user_id": uid,
"sec_user_id": "",
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"webid": web_id,
"msToken": auth.msToken or generate_msToken(),
}
endpoints = [
"https://www.douyin.com/aweme/v1/web/user/profile/other/",
"https://www.douyin.com/aweme/v1/web/im/user/info/",
]
proxies = _requests_proxies()
for url in endpoints:
try:
params = dict(base_params)
query = splice_url(params)
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
resp = requests.get(
url,
params=params,
headers=headers,
cookies=auth.cookie,
verify=False,
timeout=12,
proxies=proxies,
)
data = resp.json()
extracted = _extract_profile_from_payload(data)
if extracted.get("uid") and not result["uid"]:
result["uid"] = extracted["uid"]
if extracted.get("nickname") and not result["nickname"]:
result["nickname"] = extracted["nickname"]
if extracted.get("avatar_url") and not result["avatar_url"]:
result["avatar_url"] = extracted["avatar_url"]
if result["nickname"] and result["avatar_url"]:
break
except Exception as exc:
logger.debug(f"peer profile fetch failed for {url}: {exc}")
# Cache both success and failure. Without a short negative TTL, missing or
# rate-limited profiles were fetched again for every account poll.
_profile_cache[cache_key] = dict(result)
_profile_cache_at[cache_key] = time.time()
return result
async def fetch_peer_profile(
session: DouyinImSession,
peer_uid: int | str,
account_id: int = 0,
) -> dict[str, str]:
from .traffic_control import get_traffic_controller
controller = get_traffic_controller()
async with controller.background_slot(account_id, "peer profile"):
return await asyncio.to_thread(
fetch_peer_profile_sync,
session,
peer_uid,
account_id,
)
def enrich_conversation_item(conv: dict, my_uid: int = 0) -> dict:
"""补全会话项中的 peer_uid / sender_id。"""
item = dict(conv or {})
conv_id = str(item.get("conversation_id") or "").strip()
peer_uid = str(item.get("peer_uid") or item.get("sender_id") or "").strip()
if (not peer_uid or not peer_uid.isdigit()) and conv_id and my_uid:
resolved = resolve_peer_uid(conv_id, int(my_uid))
if resolved:
peer_uid = str(resolved)
if peer_uid:
item["peer_uid"] = peer_uid
item["sender_id"] = peer_uid
elif conv_id:
item["sender_id"] = conv_id
return item
@@ -0,0 +1,120 @@
import json
import random
import uuid
from .static import Request_pb2 as RequestProto
from .dy_util import (
generate_webid,
generate_req_sign,
generate_millisecond,
normalize_client_cert,
DEFAULT_USER_AGENT,
)
def _ua_headers(auth) -> tuple[str, str]:
ua = getattr(auth, "user_agent", None) or DEFAULT_USER_AGENT
browser_version = ua.split("Mozilla/", 1)[-1] if "Mozilla/" in ua else ua
return ua, browser_version
class ProtoBuilder:
@staticmethod
def build_normal_request(auth, cmd):
ua, browser_version = _ua_headers(auth)
request = RequestProto.Request()
request.cmd = cmd
request.sequence_id = random.randint(10000, 11000)
request.sdk_version = "1.1.3"
request.token = auth.ticket if auth.ticket else ""
request.refer = 3
request.inbox_type = 0
request.build_number = "5fa6ff1:Detached: 5fa6ff1111fd53aafc4c753505d3c93daad74d27"
did = str(getattr(auth, "device_id", "") or "0")
request.device_id = did
request.device_platform = 'douyin_pc'
request.headers['session_aid'] = '6383'
request.headers['session_did'] = did
request.headers['app_name'] = 'douyin_pc'
request.headers['priority_region'] = 'cn'
request.headers['user_agent'] = ua
request.headers['cookie_enabled'] = 'true'
request.headers['browser_language'] = 'zh-CN'
request.headers['browser_platform'] = 'Win32'
request.headers['browser_name'] = 'Mozilla'
request.headers['browser_version'] = browser_version
request.headers['browser_online'] = 'true'
request.headers['screen_width'] = '1707'
request.headers['screen_height'] = '960'
request.headers['referer'] = ''
request.headers['timezone_name'] = 'Etc/GMT-8'
request.headers['deviceId'] = did
request.headers['webid'] = generate_webid(auth)
request.headers['fp'] = auth.cookie.get('s_v_web_id', '') if auth.cookie else ''
request.headers['is-retry'] = '0'
request.auth_type = 4
request.biz = 'douyin_web'
request.access = 'web_sdk'
request.ts_sign = auth.ts_sign if auth.ts_sign else ""
request.sdk_cert = normalize_client_cert(auth.client_cert or "")
return request
@staticmethod
def build_create_conversation_request(auth, toId, myId):
request = ProtoBuilder.build_normal_request(auth, 609)
request.body.create_conversation_v2_body.conversation_type = 1
request.body.create_conversation_v2_body.participants.extend([int(toId), int(myId)])
reuqest_sign = generate_req_sign({
"sign_data": f"avatar_url=&idempotent_id=&name=&participants={toId},{myId}",
"certType": "cookie",
"scene": "web_protect"
}, auth.private_key)
request.reuqest_sign = reuqest_sign
return request
@staticmethod
def build_get_conversation_list_info_request(auth, toId, myId, conversation_short_id):
request = ProtoBuilder.build_normal_request(auth, 610)
request.body.get_conversation_info_list_v2_body.data.conversation_id = f"0:1:{myId}:{toId}"
request.body.get_conversation_info_list_v2_body.data.conversation_short_id = int(conversation_short_id)
request.body.get_conversation_info_list_v2_body.data.conversation_type = 1
return request
@staticmethod
def build_send_message_request(
auth,
conversation_id,
conversation_short_id,
ticket,
msg_content,
message_type: int = 7,
):
client_message_id = str(uuid.uuid4())
request = ProtoBuilder.build_normal_request(auth, 100)
request.body.send_message_body.conversation_id = conversation_id
request.body.send_message_body.conversation_type = 1
request.body.send_message_body.conversation_short_id = int(conversation_short_id)
request.body.send_message_body.content = json.dumps(msg_content, ensure_ascii=False,
separators=(',', ':'))
request.body.send_message_body.ext.append(
RequestProto.ExtValue(key='s:client_message_id', value=client_message_id)
)
request.body.send_message_body.ext.append(
RequestProto.ExtValue(key='s:stime', value=str(generate_millisecond()))
)
request.body.send_message_body.ext.append(
RequestProto.ExtValue(key='s:mentioned_users', value='')
)
request.body.send_message_body.message_type = int(message_type)
request.body.send_message_body.ticket = ticket
request.body.send_message_body.client_message_id = client_message_id
# 签名数据计算
sign_data_str = f'content={json.dumps(msg_content, separators=(",", ":"), ensure_ascii=False)}' + f'&conversation_id={conversation_id}&conversation_short_id={conversation_short_id}'
req_sign = generate_req_sign({
"sign_data": sign_data_str,
"certType": "cookie",
"scene": "web_protect"
}, auth.private_key)
request.reuqest_sign = req_sign
return request
+518
View File
@@ -0,0 +1,518 @@
import gzip
import json
import logging
import re
from typing import Any, Optional
from .message_content import (
MSG_TYPE_IMAGE,
MSG_TYPE_LINK_CARD,
MSG_TYPE_STICKER,
MSG_TYPE_TEXT,
MSG_TYPE_VIDEO,
MSG_TYPE_VOICE,
_coerce_message_type,
format_im_message,
message_preview,
parse_incoming_message,
parse_stored_content,
serialize_message_content,
)
logger = logging.getLogger("douyin_im.protocol")
import os
from datetime import datetime
def _is_control_payload(content_json: Any, msg_type: int = 0) -> bool:
"""判断是否为「会话控制/状态更新」等非聊天内容帧。
例如 command_type=6 的连续互动统计(consecutive_chat_data)、ext_data 元数据更新、
message_type>=50000 的系统通知等——这些不是用户发的消息,不应记录/展示成聊天气泡,
更不应触发自动回复。
"""
try:
if msg_type and int(msg_type) >= 50000:
return True
except (TypeError, ValueError):
pass
if isinstance(content_json, dict) and ("command_type" in content_json or "ext_data" in content_json):
return True
return False
_WS_DEBUG_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "ws_media_debug.log")
def _should_emit_ws_message(
conversation_id: str,
msg_type: int,
) -> bool:
"""判断 WS 帧是否为用户聊天消息(控制帧已在 _is_control_payload 过滤)。"""
if not conversation_id:
return False
try:
if int(msg_type) >= 50000:
return False
except (TypeError, ValueError):
pass
return True
def _dump_ws_message(msg_type: int, conversation_id: str, content_str: str, msg: Any = None) -> None:
"""把每条 WS 消息的全部字段落到调试文件,便于排查媒体字段结构。
默认关闭,仅当设置环境变量 KEFU_WS_DEBUG=1 时写盘,避免生产环境无界增长 / 泄露聊天内容。
content 为空时(如 type=26 瘦推送)会额外打印 protobuf 其余字段,确保「接收到的全部信息」可见。
"""
if os.getenv("KEFU_WS_DEBUG", "") not in ("1", "true", "True"):
return
try:
extra = ""
if msg is not None:
fields = {}
try:
for f, v in msg.ListFields():
if f.name == "content":
continue
fields[f.name] = v
except Exception:
pass
if fields:
extra = " | fields=" + json.dumps(fields, ensure_ascii=False, default=str)
line = (
f"{datetime.now().isoformat()} type={msg_type} "
f"conv={conversation_id} content={content_str}{extra}\n"
)
with open(_WS_DEBUG_PATH, "a", encoding="utf-8") as fh:
fh.write(line)
except Exception:
pass
def _walk_strings(data: Any, depth: int = 0, max_depth: int = 10):
if depth > max_depth:
return
if isinstance(data, dict):
for v in data.values():
yield from _walk_strings(v, depth + 1, max_depth)
elif isinstance(data, list):
for item in data:
yield from _walk_strings(item, depth + 1, max_depth)
elif isinstance(data, str) and data.strip():
yield data.strip()
def extract_json_objects(raw: bytes | str) -> list[dict]:
"""从二进制帧中尽量提取 JSON 对象"""
if isinstance(raw, bytes):
for codec in ("utf-8", "latin-1"):
try:
text = raw.decode(codec, errors="ignore")
break
except Exception:
text = ""
else:
text = ""
else:
text = raw
results = []
for match in re.finditer(r"\{[^{}]{0,2000}\}", text):
chunk = match.group(0)
try:
obj = json.loads(chunk)
if isinstance(obj, dict):
results.append(obj)
except Exception:
continue
return results
def parse_ws_payload(raw: bytes | str) -> list[dict]:
"""解析 WebSocket 二进制帧,返回标准化消息 dict 列表"""
messages = []
# 尝试 Protobuf 解包
if isinstance(raw, bytes):
try:
from .static import Live_pb2, Response_pb2
frame = Live_pb2.PushFrame()
frame.ParseFromString(raw)
if frame.payloadType == 'pb':
response = Response_pb2.Response()
response.ParseFromString(frame.payload)
body = response.body
if body.HasField("new_message_notify"):
notify = body.new_message_notify
if notify.HasField("message"):
msg = notify.message
sender = str(msg.sender)
msg_type = msg.message_type
conversation_id = msg.conversation_id
content_str = msg.content
server_message_id = str(getattr(msg, "server_message_id", "") or "")
_dump_ws_message(msg_type, conversation_id, content_str, msg)
text_content = ""
media_msg: dict = {}
content_json: dict = {}
try:
content_json = json.loads(content_str) if content_str else {}
if not isinstance(content_json, dict):
content_json = {}
media_msg = format_im_message(content_json, msg_type)
text_content = media_msg.get("text") or ""
except Exception:
media_msg = format_im_message(content_str or "", msg_type)
text_content = media_msg.get("text") or content_str or ""
# 跳过会话控制/状态更新等非聊天内容帧(不记录、不展示、不触发自动回复)
if _is_control_payload(content_json, msg_type):
logger.debug(
"Skip control WS frame: type=%s conv=%s", msg_type, conversation_id
)
return messages
if _should_emit_ws_message(conversation_id, msg_type):
sender_uid = str(msg.sender)
if media_msg and (
media_msg.get("text")
or media_msg.get("type") not in (None, "text", "")
):
display_content = serialize_message_content(media_msg)
else:
display_content = text_content or content_str
payload = {
"sender_name": sender_uid,
"sender_uid": sender_uid,
"content": display_content,
"raw_content": content_str,
"conversation_id": conversation_id,
"unread_count": 1,
"server_message_id": server_message_id,
"message_type": msg_type,
}
if msg_type in (
MSG_TYPE_IMAGE,
MSG_TYPE_STICKER,
MSG_TYPE_VOICE,
MSG_TYPE_VIDEO,
MSG_TYPE_LINK_CARD,
):
if not media_msg.get("url") and not media_msg.get("uri"):
logger.warning(
"Media WS message missing url: type=%s content=%s",
msg_type,
(content_str or "")[:800],
)
elif not media_msg.get("url"):
logger.info(
"Media WS message resolved via uri: type=%s uri=%s",
msg_type,
media_msg.get("uri"),
)
messages.append(payload)
logger.info(
"Protobuf WS message parsed: sender=%s type=%s content=%s conv=%s",
sender,
msg_type,
text_content,
conversation_id,
)
return messages
except Exception as e:
logger.debug(f"Protobuf WS parse failed: {e}")
if isinstance(raw, str):
payloads = [raw.encode("utf-8", errors="ignore")]
else:
payloads = [raw]
# 尝试 gzip 解压(frontier 常见)
try:
payloads.append(gzip.decompress(raw))
except Exception:
pass
for payload in payloads:
# 1) 直接 JSON
if isinstance(payload, bytes):
text = payload.decode("utf-8", errors="ignore").strip()
else:
text = str(payload).strip()
if text.startswith("{") or text.startswith("["):
try:
data = json.loads(text)
messages.extend(normalize_im_payload(data))
continue
except Exception:
pass
# 2) 嵌入 JSON
for obj in extract_json_objects(payload):
messages.extend(normalize_im_payload(obj))
# 3) 纯文本兜底
if isinstance(payload, bytes):
text = payload.decode("utf-8", errors="ignore")
plain = _extract_plain_text(text)
if plain:
messages.append({"content": plain, "sender_name": "", "raw_content": plain, "raw": True})
return messages
def normalize_im_payload_from_bytes(raw: bytes) -> list[dict]:
"""Try to extract conversation/message payloads from binary IM API responses."""
results: list[dict] = []
for obj in extract_json_objects(raw):
results.extend(normalize_im_payload(obj))
if results:
return results
try:
from .static import Response_pb2
response = Response_pb2.Response()
response.ParseFromString(raw)
body = response.body
for field in (
"get_conversation_info_list_v2_response_body",
"create_conversation_v2_body",
):
if body.HasField(field):
conv_body = getattr(body, field)
for conv in conv_body.conversation_info_list:
conv_id = conv.conversation_id
peer_uid = ""
parts = conv_id.split(":")
if len(parts) >= 4:
peer_uid = parts[-1]
label = f"用户{peer_uid[-6:]}" if peer_uid else conv_id
results.append({
"conversation_id": conv_id,
"sender_name": label,
"content": "",
"unread_count": 0,
"peer_uid": peer_uid,
"conversation_short_id": str(conv.conversation_short_id),
"ticket": conv.ticket,
})
except Exception:
pass
return results
def normalize_im_payload(data: Any, depth: int = 0) -> list[dict]:
"""递归标准化 IM JSON 为 {sender_name, content, conversation_id, unread_count}"""
if depth > 12:
return []
results = []
if isinstance(data, list):
for item in data:
results.extend(normalize_im_payload(item, depth + 1))
return results
if not isinstance(data, dict):
return results
sender = (
_pick_str(data, "sender_name", "senderName", "nickname", "nick_name", "userName", "peerName")
or _pick_nested(data, ("core_info", "user_info", "peer_info"), "nick_name", "nickname", "name")
)
sender_avatar = _pick_avatar_url(data)
content = _pick_message_text(data)
msg_type = _coerce_message_type(
data.get("message_type") or data.get("messageType") or data.get("msg_type"),
MSG_TYPE_TEXT,
)
# 会话控制/状态更新帧(command_type / ext_data / 系统通知)直接忽略,不当作聊天消息
if _is_control_payload(data, msg_type) or _is_control_payload(data.get("content"), msg_type):
return results
if msg_type != MSG_TYPE_TEXT or (isinstance(data.get("content"), dict)):
parsed = parse_incoming_message(data)
if parsed:
content = parsed
elif content and content.startswith("{"):
try:
parsed = format_im_message(json.loads(content), msg_type)
content = serialize_message_content(parsed)
except Exception:
pass
elif content in _NON_TEXT_MESSAGE_MARKERS:
parsed = parse_stored_content(content)
content = serialize_message_content(parsed)
conv_id = _pick_str(
data,
"conversation_id",
"conversationId",
"conv_id",
"cid",
)
unread = data.get("unread_count") or data.get("unreadCount") or data.get("unread_cnt") or 0
try:
unread = int(unread or 0)
except (TypeError, ValueError):
unread = 0
if content and len(content) < 500:
from_self = data.get("is_self") or data.get("isSelf") or data.get("fromSelf") or data.get("self")
if not from_self:
raw_content = _extract_raw_content(data) or content
results.append({
"sender_name": sender or "未知用户",
"sender_avatar": sender_avatar or None,
"content": content,
"raw_content": raw_content,
"conversation_id": conv_id or "",
"unread_count": unread,
"message_type": msg_type,
})
if sender and unread > 0 and not content:
results.append({
"sender_name": sender,
"sender_avatar": sender_avatar or None,
"content": "[未读消息]",
"raw_content": "[未读消息]",
"conversation_id": conv_id or "",
"unread_count": unread,
"message_type": msg_type,
})
for key in ("conversations", "conversation_list", "data", "messages", "messagesList", "body"):
nested = data.get(key)
if nested is not None:
results.extend(normalize_im_payload(nested, depth + 1))
for value in data.values():
if isinstance(value, (dict, list)):
results.extend(normalize_im_payload(value, depth + 1))
return results
def _extract_raw_content(data: dict) -> str:
for key in ("content", "message", "msg", "lastMessage", "last_msg", "preview", "brief"):
val = data.get(key)
if isinstance(val, str) and val.strip():
return val.strip()
if isinstance(val, dict):
return json.dumps(val, ensure_ascii=False, separators=(",", ":"))
return ""
def _pick_str(data: dict, *keys: str) -> str:
for key in keys:
val = data.get(key)
if isinstance(val, str) and val.strip():
return val.strip()
return ""
def _pick_nested(data: dict, parent_keys: tuple, *child_keys: str) -> str:
for pk in parent_keys:
nested = data.get(pk)
if isinstance(nested, dict):
val = _pick_str(nested, *child_keys)
if val:
return val
return ""
def _avatar_from_value(val: Any) -> str:
if isinstance(val, str) and val.strip().startswith("http"):
return val.strip()
if isinstance(val, dict):
direct = val.get("url")
if isinstance(direct, str) and direct.startswith("http"):
return direct.strip()
for list_key in ("url_list", "urls"):
urls = val.get(list_key)
if isinstance(urls, list):
for item in urls:
if isinstance(item, str) and item.startswith("http"):
return item.strip()
return ""
def _pick_avatar_url(data: dict) -> str:
for key in ("avatar_url", "avatarUrl", "head_url", "headUrl", "avatar"):
url = _avatar_from_value(data.get(key))
if url:
return url
for thumb_key in ("avatar_thumb", "avatar_medium", "avatar_larger", "avatarThumb"):
url = _avatar_from_value(data.get(thumb_key))
if url:
return url
for parent_key in ("core_info", "user_info", "peer_info", "target_user", "conversation_core_info"):
nested = data.get(parent_key)
if isinstance(nested, dict):
url = _pick_avatar_url(nested)
if url:
return url
return ""
def _pick_message_text(data: dict) -> str:
for key in (
"text",
"content",
"message",
"msg",
"lastMessage",
"last_msg",
"preview",
"brief",
):
val = data.get(key)
if isinstance(val, str) and val.strip():
return val.strip()
if isinstance(val, dict):
inner = _pick_str(val, "text", "content", "message")
if inner:
return inner
return ""
def _extract_plain_text(text: str) -> Optional[str]:
text = (text or "").strip()
if not text or len(text) > 200:
return None
if text.startswith("{") or text.startswith("["):
return None
# 过滤明显二进制垃圾
printable = sum(1 for c in text if c.isprintable() or c in "\n\r\t")
if printable / max(len(text), 1) < 0.8:
return None
return text
_NON_TEXT_MESSAGE_MARKERS = {
"[表情包]",
"[语音]",
"[图片]",
"[视频]",
"[未读消息]",
}
def should_skip_auto_reply(content: str) -> tuple[bool, str]:
"""判断收到的内容是否不适合触发自动回复(如纯点赞/表情互动)。"""
from .message_content import is_media_message, message_preview
text = (content or "").strip()
if not text:
return True, "空消息"
if is_media_message(text):
return True, f"非文本消息({message_preview(text)})"
if text in _NON_TEXT_MESSAGE_MARKERS:
return True, f"非文本消息({text})"
if re.fullmatch(r"(\[赞\])+", text):
return True, "表情互动消息(点赞),抖音通常不允许对此类消息自动回复"
if re.fullmatch(r"\[[^\]]+\](\[[^\]]+\])*", text) and "http" not in text:
inner = re.sub(r"[\[\]]", "", text)
if len(inner) <= 20 and not any(ch.isalnum() for ch in inner):
return True, f"非文本互动消息({text})"
return False, ""
@@ -0,0 +1,330 @@
"""自动回复内容解析与 IM 消息体构造(文本 / 网址 / 卡片)。"""
from __future__ import annotations
import json
from typing import Any, Tuple
from .message_content import (
MSG_TYPE_IMAGE,
MSG_TYPE_STICKER,
MSG_TYPE_TEXT,
message_preview,
parse_stored_content,
serialize_message_content,
)
def _normalize_reply_spec(data: dict[str, Any]) -> dict[str, Any] | None:
if not isinstance(data, dict):
return None
reply_type = data.get("type")
if reply_type in ("text", "link", "card", "image", "sticker"):
return data
return None
def parse_reply_messages(raw: str) -> list[dict[str, Any]]:
"""解析规则中的 reply_content,支持单条或多条回复。"""
raw = (raw or "").strip()
if not raw:
return [{"type": "text", "text": ""}]
if raw.startswith("{") or raw.startswith("["):
try:
data = json.loads(raw)
if isinstance(data, dict) and isinstance(data.get("messages"), list):
specs = [_normalize_reply_spec(item) for item in data["messages"]]
specs = [item for item in specs if item]
if specs:
return specs
if isinstance(data, list):
specs = [_normalize_reply_spec(item) for item in data]
specs = [item for item in specs if item]
if specs:
return specs
if isinstance(data, dict):
spec = _normalize_reply_spec(data)
if spec:
return [spec]
except json.JSONDecodeError:
pass
return [{"type": "text", "text": raw}]
def parse_reply_content(raw: str) -> dict[str, Any]:
"""解析发送/规则中的 content。纯字符串视为文本,JSON 为结构化消息。"""
raw = (raw or "").strip()
if raw.startswith("{"):
try:
data = json.loads(raw)
if isinstance(data, dict):
spec = _normalize_reply_spec(data)
if spec:
return spec
except json.JSONDecodeError:
pass
parsed = parse_stored_content(raw)
if parsed.get("type") != "text":
return parsed
return parse_reply_messages(raw)[0]
def _format_reply_spec(spec: dict[str, Any]) -> str:
reply_type = spec.get("type", "text")
if reply_type in ("image", "sticker", "voice", "video"):
return message_preview(serialize_message_content(spec))
if reply_type == "text":
return (spec.get("text") or "").strip()
if reply_type == "link":
text = (spec.get("text") or spec.get("title") or "").strip()
url = (spec.get("url") or "").strip()
if text and url:
return f"{text} → {url}"
return text or url
if reply_type == "card":
title = (spec.get("title") or "").strip()
desc = (spec.get("desc") or spec.get("description") or "").strip()
page_url = (spec.get("url") or "").strip()
if title and page_url:
return f"[卡片] {title} → {page_url}"
return title or desc or page_url or "[卡片]"
return message_preview(serialize_message_content(spec))
def format_reply_display(raw: str) -> str:
"""将 reply_content 格式化为日志/列表中的可读摘要。"""
preview = message_preview(raw)
if preview:
return preview
specs = parse_reply_messages(raw)
parts = [_format_reply_spec(spec) for spec in specs]
parts = [part for part in parts if part]
if not parts:
return (raw or "").strip()
if len(parts) == 1:
return parts[0]
return " | ".join(parts)
def serialize_reply_messages(specs: list[dict[str, Any]]) -> str:
cleaned = [spec for spec in specs if _normalize_reply_spec(spec)]
if not cleaned:
cleaned = [{"type": "text", "text": ""}]
if len(cleaned) == 1:
return serialize_reply_content(cleaned[0])
return json.dumps({"messages": cleaned}, ensure_ascii=False, separators=(",", ":"))
def _extract_link_card_media_path(url: str) -> str:
value = (url or "").strip()
idx = value.find("/api/media/link-cards/")
return value[idx:] if idx >= 0 else ""
def expand_card_spec(spec: dict[str, Any]) -> list[dict[str, Any]]:
"""卡片专属发送规则:展开为「封面图片 + 标题/内容/可点击链接文字」两条消息。
抖音 Web 协议无法发原生合并卡片(type=70 带图→8004/不带图→空白),因此卡片以
「图片消息(横幅) + 文本(标题/内容/链接)」组合呈现:图片提供视觉、文本提供可点击跳转。
图片消息仅互关用户可收(陌生人会被 8003 拦截,但文本仍可送达,不影响链接触达)。
"""
title = (spec.get("title") or "").strip()
desc = (spec.get("desc") or spec.get("description") or "").strip()
target = (
spec.get("target_url")
or spec.get("url")
or spec.get("link_url")
or ""
).strip()
if "localhost" in target or "127.0.0.1" in target:
target = (spec.get("target_url") or spec.get("link_url") or "").strip() or target
cover = (spec.get("image_path") or "").strip()
if not cover:
cover = _extract_link_card_media_path(spec.get("cover_url") or "")
out: list[dict[str, Any]] = []
if cover:
out.append({"type": "image", "url": cover, "text": "[图片]"})
lines = [x for x in (title, desc, target) if x]
text = "\n".join(lines) if lines else target
if text:
out.append({"type": "text", "text": text})
if not out:
out.append({"type": "text", "text": target})
return out
def split_reply_payloads(raw: str) -> list[str]:
"""将规则 reply_content 拆成可逐条发送的 payload 列表。卡片单独展开为图片+文本。"""
payloads: list[str] = []
for spec in parse_reply_messages(raw):
if spec.get("type") == "card":
payloads.extend(serialize_reply_content(s) for s in expand_card_spec(spec))
else:
payloads.append(serialize_reply_content(spec))
return payloads
def serialize_reply_log(payloads: list[str]) -> str:
"""把实际逐条发送的 payload(JSON 字符串)合并为结构化的日志内容。
单条直接返回该 payload;多条用 {"messages":[...]} 包裹,便于前端逐条渲染
(图片/表情正常显示为媒体,而不是被压扁成 "图片" 这样的占位文本)。
"""
specs: list[dict[str, Any]] = []
for payload in payloads:
raw = (payload or "").strip()
if not raw:
continue
spec: dict[str, Any] | None = None
if raw.startswith("{"):
try:
data = json.loads(raw)
spec = _normalize_reply_spec(data)
except json.JSONDecodeError:
spec = None
if spec is None:
spec = {"type": "text", "text": raw}
specs.append(spec)
return serialize_reply_messages(specs)
def serialize_reply_content(spec: dict[str, Any]) -> str:
return json.dumps(spec, ensure_ascii=False, separators=(",", ":"))
def build_msg_payload(spec: dict[str, Any]) -> Tuple[dict[str, Any], int]:
"""根据回复规格构造 IM msg_content 与 message_type。"""
reply_type = spec.get("type", "text")
if reply_type == "text":
text = (spec.get("text") or "").strip()
return (
{
"mention_users": [],
"aweType": 700,
"richTextInfos": [],
"text": text,
},
7,
)
if reply_type == "link":
display = (spec.get("text") or spec.get("title") or "").strip()
url = (spec.get("url") or "").strip()
if not display:
display = url
msg_content: dict[str, Any] = {
"mention_users": [],
"aweType": 700,
"richTextInfos": [],
"text": display,
}
if url and display:
msg_content["richTextInfos"] = [
{
"start": 0,
"end": len(display),
"type": 2,
"link": url,
"text": display,
}
]
return msg_content, 7
if reply_type == "card":
# 卡片在 split_reply_payloads 阶段已展开为「图片 + 文本」两条,正常不会走到这里。
# 兜底:万一收到未展开的卡片 spec,退化为「标题/内容/链接」文本,确保可送达。
title = (spec.get("title") or "").strip()
desc = (spec.get("desc") or spec.get("description") or "").strip()
target = (
spec.get("target_url")
or spec.get("url")
or spec.get("link_url")
or ""
).strip()
lines = [x for x in (title, desc, target) if x]
text = "\n".join(lines) if lines else target
return (
{"mention_users": [], "aweType": 700, "richTextInfos": [], "text": text},
MSG_TYPE_TEXT,
)
if reply_type == "image":
uri = (spec.get("uri") or "").strip().lstrip("/")
url = (spec.get("url") or "").strip()
width = spec.get("width")
height = spec.get("height")
md5 = (spec.get("md5") or "").strip()
url_list = spec.get("url_list")
if not isinstance(url_list, list):
url_list = [url] if url.startswith("http") else []
clean_urls = [str(u).strip() for u in url_list if str(u).strip().startswith("http")]
resource_url: dict[str, Any] = {}
if uri:
resource_url["uri"] = uri
if clean_urls:
resource_url["url_list"] = clean_urls
if md5:
resource_url["md5"] = md5
if width:
try:
resource_url["width"] = int(width)
except (TypeError, ValueError):
pass
if height:
try:
resource_url["height"] = int(height)
except (TypeError, ValueError):
pass
msg_content: dict[str, Any] = {
"aweType": 2702,
"from_gallery": 1,
"create_type": 0,
}
if resource_url:
msg_content["resource_url"] = resource_url
if uri:
msg_content["local_path"] = uri
if md5:
msg_content["md5"] = md5
if width:
try:
msg_content["cover_width"] = int(width)
except (TypeError, ValueError):
pass
if height:
try:
msg_content["cover_height"] = int(height)
except (TypeError, ValueError):
pass
return msg_content, MSG_TYPE_IMAGE
if reply_type == "sticker":
url = (spec.get("url") or "").strip()
sticker_id = spec.get("sticker_id") or spec.get("id")
msg_content = {
"display_name": (spec.get("name") or spec.get("text") or "[表情包]").strip(),
}
if sticker_id:
msg_content["id"] = sticker_id
msg_content["sticker_id"] = sticker_id
if url:
msg_content["static_url"] = {"url_list": [url]}
msg_content["animate_url"] = {"url_list": [url]}
return msg_content, MSG_TYPE_STICKER
text = format_reply_display(serialize_reply_content(spec))
return (
{
"mention_users": [],
"aweType": 700,
"richTextInfos": [],
"text": text,
},
7,
)
+352
View File
@@ -0,0 +1,352 @@
"""Observable per-account serial queue for delayed automatic replies."""
from __future__ import annotations
import asyncio
import logging
import time
import uuid
from collections import deque
from copy import deepcopy
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any, Awaitable, Callable, Iterable, Optional
logger = logging.getLogger("douyin_im.reply_queue")
ReplyCallback = Callable[[], Awaitable[Any]]
ErrorCallback = Callable[[str, BaseException], None]
DetailsMerger = Callable[[dict[str, Any]], dict[str, Any]]
@dataclass
class _QueueItem:
job_id: str
due_at: float
slot_seconds: float
callback: ReplyCallback
description: str
queued_at: float
merge_keys: frozenset[str] = field(default_factory=frozenset)
details: dict[str, Any] = field(default_factory=dict)
expedited: bool = False
class AccountReplyQueue:
"""Run and expose delayed reply jobs for one hosted account.
A single consumer is the only code path allowed to invoke callbacks. Jobs
selected for immediate delivery are moved to an urgent FIFO, so they can
never overlap an already active send. Removing a scheduled job also moves
every job behind it forward by the removed job's reserved slot.
"""
def __init__(
self,
account_id: int,
on_error: Optional[ErrorCallback] = None,
) -> None:
self.account_id = account_id
self._on_error = on_error
self._waiting: list[_QueueItem] = []
self._urgent: deque[_QueueItem] = deque()
self._active_item: Optional[_QueueItem] = None
self._task: Optional[asyncio.Task] = None
self._running = False
self._state_lock = asyncio.Lock()
self._wake = asyncio.Event()
self._tail_due_at = 0.0
@property
def pending_count(self) -> int:
"""Approximate active + urgent + waiting count for lightweight badges."""
return (
len(self._waiting)
+ len(self._urgent)
+ (1 if self._active_item is not None else 0)
)
async def start(self) -> None:
async with self._state_lock:
if self._task and not self._task.done():
return
self._running = True
self._tail_due_at = 0.0
self._wake.clear()
self._task = asyncio.create_task(
self._run(),
name=f"account-reply-queue-{self.account_id}",
)
async def enqueue(
self,
delay_seconds: float,
callback: ReplyCallback,
description: str = "",
details: Optional[dict[str, Any]] = None,
merge_key: str = "",
merge_keys: Optional[Iterable[str]] = None,
) -> int:
"""Append one reply job and return its current 1-based queue position."""
interval = max(0.0, float(delay_seconds or 0))
loop = asyncio.get_running_loop()
async with self._state_lock:
if not self._running or not self._task or self._task.done():
raise RuntimeError("reply queue is not running")
due_at = max(loop.time(), self._tail_due_at) + interval
self._tail_due_at = due_at
self._waiting.append(
_QueueItem(
job_id=uuid.uuid4().hex,
due_at=due_at,
slot_seconds=interval,
callback=callback,
description=description,
queued_at=time.time(),
merge_keys=self._normalize_merge_keys(
merge_keys if merge_keys is not None else merge_key
),
details=deepcopy(details or {}),
)
)
position = self.pending_count
self._wake.set()
return position
@staticmethod
def _normalize_merge_keys(value: str | Iterable[str]) -> frozenset[str]:
values = [value] if isinstance(value, str) else list(value or [])
return frozenset(str(item or "").strip() for item in values if str(item or "").strip())
@staticmethod
def _merge_keys_match(
existing: frozenset[str],
incoming: frozenset[str],
) -> bool:
existing_conversations = {key for key in existing if key.startswith("conv:")}
incoming_conversations = {key for key in incoming if key.startswith("conv:")}
if existing_conversations & incoming_conversations:
return True
# Two explicit, different conversation IDs must never merge just because
# their partial source data happens to expose the same peer identifier.
if existing_conversations and incoming_conversations:
return False
existing_peers = {key for key in existing if key.startswith("peer:")}
incoming_peers = {key for key in incoming if key.startswith("peer:")}
return bool(existing_peers & incoming_peers)
async def merge_pending(
self,
merge_key: str | Iterable[str],
details_merger: DetailsMerger,
) -> dict[str, Any]:
"""Merge details into one queued conversation without changing its slot.
Only waiting and urgent jobs are mutable. Once the consumer marks a job
active, its callback is sealed and a later message must follow the normal
new-message path.
"""
normalized_keys = self._normalize_merge_keys(merge_key)
if not normalized_keys:
return {"status": "not_found"}
async with self._state_lock:
if not self._running or not self._task or self._task.done():
return {"status": "not_running"}
active_offset = 1 if self._active_item is not None else 0
matches: list[tuple[_QueueItem, str, int]] = []
for index, candidate in enumerate(self._urgent):
if self._merge_keys_match(candidate.merge_keys, normalized_keys):
matches.append((candidate, "ready", active_offset + index + 1))
waiting_offset = active_offset + len(self._urgent)
for index, candidate in enumerate(self._waiting):
if self._merge_keys_match(candidate.merge_keys, normalized_keys):
matches.append((candidate, "waiting", waiting_offset + index + 1))
if not matches:
return {"status": "not_found"}
incoming_conversations = {
key for key in normalized_keys if key.startswith("conv:")
}
if not incoming_conversations:
matched_conversations = {
key
for candidate, _, _ in matches
for key in candidate.merge_keys
if key.startswith("conv:")
}
if len(matched_conversations) > 1:
return {"status": "not_found", "reason": "ambiguous_peer"}
item, item_status, position = matches[0]
merged_details = details_merger(deepcopy(item.details))
if not isinstance(merged_details, dict):
raise TypeError("reply queue details merger must return a dict")
item.details = deepcopy(merged_details)
item.merge_keys = frozenset(item.merge_keys | normalized_keys)
return {
"status": "merged",
"job_id": item.job_id,
"position": position,
"queue_status": item_status,
"message_count": int(item.details.get("message_count") or 1),
}
async def snapshot(self) -> list[dict[str, Any]]:
"""Return a callback-free management snapshot ordered by execution."""
loop = asyncio.get_running_loop()
now_mono = loop.time()
now_epoch = time.time()
async with self._state_lock:
ordered: list[tuple[_QueueItem, str]] = []
if self._active_item is not None:
ordered.append((self._active_item, "sending"))
ordered.extend((item, "ready") for item in self._urgent)
ordered.extend((item, "waiting") for item in self._waiting)
result = []
for position, (item, status) in enumerate(ordered, start=1):
remaining = 0.0 if status != "waiting" else max(0.0, item.due_at - now_mono)
scheduled_epoch = now_epoch + max(0.0, item.due_at - now_mono)
payload = {
"job_id": item.job_id,
"account_id": self.account_id,
"position": position,
"status": status,
"expedited": bool(item.expedited),
"description": item.description,
"interval_seconds": int(round(item.slot_seconds)),
"enqueued_at": datetime.fromtimestamp(
item.queued_at, tz=timezone.utc
).isoformat(),
"scheduled_at": datetime.fromtimestamp(
scheduled_epoch, tz=timezone.utc
).isoformat(),
"remaining_seconds": int(max(0, round(remaining))),
}
# Details are controlled by DouyinImService and never contain callbacks/session data.
payload.update(deepcopy(item.details))
result.append(payload)
return result
async def send_now(self, job_id: str) -> dict[str, Any]:
"""Move one waiting job to the urgent FIFO and free its future slot."""
job_id = str(job_id or "").strip()
async with self._state_lock:
if not self._running or not self._task or self._task.done():
return {"status": "not_running", "job_id": job_id}
if self._active_item and self._active_item.job_id == job_id:
return {"status": "already_sending", "job_id": job_id}
if any(item.job_id == job_id for item in self._urgent):
return {"status": "already_requested", "job_id": job_id}
selected_index = next(
(index for index, item in enumerate(self._waiting) if item.job_id == job_id),
None,
)
if selected_index is None:
return {"status": "not_found", "job_id": job_id}
item = self._waiting.pop(selected_index)
shift_seconds = max(0.0, item.slot_seconds)
shifted_count = 0
for later in self._waiting[selected_index:]:
later.due_at -= shift_seconds
shifted_count += 1
item.due_at = asyncio.get_running_loop().time()
item.expedited = True
self._urgent.append(item)
self._recalculate_tail_due_at()
self._wake.set()
return {
"status": "accepted",
"job_id": job_id,
"shifted_count": shifted_count,
}
async def stop(self) -> None:
"""Cancel the active wait/send and discard all remaining jobs."""
async with self._state_lock:
self._running = False
self._wake.set()
task = self._task
self._task = None
if task:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
async with self._state_lock:
self._waiting.clear()
self._urgent.clear()
self._active_item = None
self._tail_due_at = 0.0
self._wake.clear()
def _recalculate_tail_due_at(self) -> None:
scheduled = [item.due_at for item in self._waiting]
self._tail_due_at = max(scheduled, default=0.0)
async def _run(self) -> None:
while True:
item: Optional[_QueueItem] = None
wait_seconds: Optional[float] = None
async with self._state_lock:
if not self._running:
return
if self._urgent:
item = self._urgent.popleft()
elif self._waiting:
candidate = self._waiting[0]
remaining = candidate.due_at - asyncio.get_running_loop().time()
if remaining <= 0:
item = self._waiting.pop(0)
else:
wait_seconds = remaining
if item is not None:
self._active_item = item
self._wake.clear()
if item is None:
try:
if wait_seconds is None:
await self._wake.wait()
else:
await asyncio.wait_for(self._wake.wait(), timeout=wait_seconds)
except asyncio.TimeoutError:
pass
continue
try:
await item.callback()
except asyncio.CancelledError:
raise
except Exception as exc:
logger.exception(
"Account %s queued reply failed (%s)",
self.account_id,
item.description,
)
if self._on_error:
try:
self._on_error(item.description, exc)
except Exception:
logger.debug("Reply queue error callback failed", exc_info=True)
finally:
async with self._state_lock:
if self._active_item is item:
self._active_item = None
if not self._waiting:
self._tail_due_at = 0.0
self._wake.set()
+983
View File
@@ -0,0 +1,983 @@
import asyncio
import logging
import time
from typing import Awaitable, Callable, Optional
from utils import system_logger
from .frontier import ensure_frontier_ws
from .http_client import DouyinImHttpClient, format_session_credential_summary
from .session import DouyinImSession
from .ws_client import DouyinImWsClient
from .reply_queue import AccountReplyQueue
from .traffic_control import get_traffic_controller
from .reply_payload import format_reply_display, serialize_reply_log
from .conv_util import resolve_peer_uid
from .peer_profile import (
enrich_conversation_item,
fetch_peer_profile,
is_generic_peer_name,
)
logger = logging.getLogger("douyin_im.service")
MatchReplyFn = Callable[[str], Awaitable[Optional[list[str]]]]
LogFn = Callable[..., Awaitable[None]]
ReceivedLogFn = Callable[..., Awaitable[None]]
class DouyinImService:
"""抖音 IM 直连服务:WebSocket 实时监听 + HTTP 轮询 + 自动回复"""
def __init__(
self,
session: DouyinImSession,
match_reply: MatchReplyFn,
log_fn: LogFn,
account_id: int,
received_log_fn: Optional[ReceivedLogFn] = None,
reply_delay_seconds: int = 0,
reply_delay_resolver: Optional[Callable[[], Awaitable[int]]] = None,
reply_cooldown_seconds: Optional[int] = None,
cooldown_resolver: Optional[Callable[[], Awaitable[int]]] = None,
refresh_credentials: Optional[Callable[[], Awaitable[bool]]] = None,
follow_tick: Optional[Callable[[], Awaitable[None]]] = None,
on_session_invalid: Optional[Callable[[str], Awaitable[None]]] = None,
):
self.session = session
self.match_reply = match_reply
self.log_fn = log_fn
self.received_log_fn = received_log_fn
self.account_id = account_id
# 由 worker 注入:周期性检测新粉丝并发送关注欢迎语(约每 60s 触发一次)
self.follow_tick = follow_tick
# 由 worker 注入:检测到 IM 登录失效(INVALID_REQUEST)时回调,用于自动下线
self.on_session_invalid = on_session_invalid
self._session_invalid_strikes = 0
self._session_invalid_fired = False
self.reply_delay_seconds = max(0, int(reply_delay_seconds or 0))
# 实时解析账号排队间隔:账号专属优先,否则使用系统默认值。
self._reply_delay_resolver = reply_delay_resolver
self._reply_queue = AccountReplyQueue(
account_id=self.account_id,
on_error=self._on_reply_queue_error,
)
# WS 帧与 HTTP 轮询会并发进入;按到达顺序串行完成预处理/入队,确保 FIFO。
self._incoming_lock = asyncio.Lock()
# 该账号专属冷却秒数;None 表示继承全局系统设置(仅作为无 resolver 时的兜底)
self._cooldown_override = (
max(0, int(reply_cooldown_seconds)) if reply_cooldown_seconds is not None else None
)
# 实时解析冷却秒数的回调(账号专属优先,否则全局);优先于 _cooldown_override
self._cooldown_resolver = cooldown_resolver
# 由 worker 注入:触发后台重新采集 web_protect/keys(刷新 ts_sign),返回是否刷新成功
self.refresh_credentials = refresh_credentials
self._running = False
self._replied_keys: set[str] = set()
self._logged_keys: set[str] = set()
self._received_logged_keys: set[str] = set()
# 每个对话/用户最近一次自动回复的时间戳(monotonic 秒),用于冷却窗口去重
self._last_reply_at: dict[str, float] = {}
self._conv_previews: dict[str, str] = {}
self._conv_names: dict[str, str] = {} # uid/conv_id -> nickname
self._conv_meta: dict[str, dict] = {} # conversation_id -> meta
self._ws_client: Optional[DouyinImWsClient] = None
self.last_error: str = ""
def _reply_key(self, conversation_key: str, content: str) -> str:
return f"{conversation_key}::{content}"
@staticmethod
def _reply_queue_merge_keys(
conversation_id: str,
peer_uid: str,
) -> tuple[str, ...]:
"""Return every stable identifier currently known for one conversation."""
conversation_id = str(conversation_id or "").strip()
peer_uid = str(peer_uid or "").strip()
aliases: list[str] = []
if conversation_id:
aliases.append(f"conv:{conversation_id}")
if peer_uid:
aliases.append(f"peer:{peer_uid}")
return tuple(aliases)
@staticmethod
def _merge_reply_queue_details(
existing: dict,
*,
incoming_content: str,
sender_name: str,
sender_id: str,
sender_avatar: Optional[str],
conversation_id: str,
) -> dict:
"""Append one received message while preserving the task's one reply."""
merged = dict(existing or {})
contents = merged.get("incoming_contents")
if isinstance(contents, list):
contents = list(contents)
else:
contents = []
if not contents and "incoming_content" in merged:
contents.append(str(merged.get("incoming_content") or ""))
latest_content = str(incoming_content or "")
contents.append(latest_content)
merged["incoming_content"] = latest_content
merged["incoming_contents"] = contents
merged["message_count"] = len(contents)
if sender_name:
merged["sender_name"] = sender_name
if sender_id:
merged["sender_id"] = sender_id
if sender_avatar:
merged["sender_avatar"] = sender_avatar
if conversation_id:
merged["conversation_id"] = conversation_id
return merged
def _cooldown_seconds_sync(self) -> int:
"""无 resolver 时的兜底:账号专属优先,否则取全局设置;0 表示关闭。"""
if self._cooldown_override is not None:
return self._cooldown_override
try:
from auth.system_settings import get_cached_settings
return max(0, int(get_cached_settings().auto_reply_cooldown_seconds or 0))
except Exception:
return 0
async def _resolve_cooldown_seconds(self) -> int:
"""实时解析冷却秒数:优先用 worker 注入的 resolver(账号优先、否则全局),否则兜底。"""
if self._cooldown_resolver is not None:
try:
return max(0, int(await self._cooldown_resolver() or 0))
except Exception as e:
logger.debug(f"cooldown resolver failed: {e}")
return self._cooldown_seconds_sync()
def _reply_delay_seconds_sync(self) -> int:
"""无 resolver 时解析排队间隔;0 表示不启用排队规则。"""
if self.reply_delay_seconds > 0:
return self.reply_delay_seconds
try:
from auth.system_settings import get_cached_settings
return max(0, int(get_cached_settings().auto_reply_delay_seconds or 0))
except Exception:
return 0
async def _resolve_reply_delay_seconds(self) -> int:
"""实时解析账号生效的回复排队间隔。"""
if self._reply_delay_resolver is not None:
try:
return max(0, int(await self._reply_delay_resolver() or 0))
except Exception as exc:
logger.debug(f"reply delay resolver failed: {exc}")
return self._reply_delay_seconds_sync()
def _on_reply_queue_error(self, description: str, exc: BaseException) -> None:
system_logger.record(
"账号回复队列执行失败",
detail=f"{description or '自动回复任务'}:{exc}",
level="error",
category="send",
account_id=self.account_id,
)
def _peer_in_cooldown(self, peer_key: str, cooldown: int) -> bool:
if cooldown <= 0 or not peer_key:
return False
last = self._last_reply_at.get(peer_key)
if last is None:
return False
return (time.monotonic() - last) < cooldown
def _resolve_sender_name(self, msg: dict) -> str:
sender_uid = str(msg.get("sender_uid") or msg.get("sender_name") or "").strip()
conv_id = str(msg.get("conversation_id") or "")
name = (msg.get("sender_name") or "").strip()
if name and not name.isdigit():
return name
if sender_uid and self._conv_names.get(sender_uid):
return self._conv_names[sender_uid]
if conv_id and self._conv_names.get(conv_id):
return self._conv_names[conv_id]
if sender_uid:
return f"用户{sender_uid[-6:]}" if len(sender_uid) > 6 else f"用户{sender_uid}"
return "未知用户"
def _is_self_message(self, msg: dict) -> bool:
sender_uid = str(msg.get("sender_uid") or "").strip()
if not sender_uid or not self.session.my_uid:
return False
try:
return int(sender_uid) == int(self.session.my_uid)
except (TypeError, ValueError):
return False
async def _resolve_peer_profile(
self,
conv_id: str,
sender_uid: str,
sender: str,
sender_avatar: str,
) -> tuple[str, str, str]:
my_uid = int(self.session.my_uid or 0)
peer_uid = str(sender_uid or "").strip()
if (not peer_uid or not peer_uid.isdigit()) and conv_id and my_uid:
resolved = resolve_peer_uid(conv_id, my_uid)
if resolved:
peer_uid = str(resolved)
meta = self._conv_meta.get(conv_id, {}) if conv_id else {}
name = (sender or meta.get("sender_name") or "").strip()
avatar = (sender_avatar or meta.get("sender_avatar") or "").strip()
if peer_uid and self._conv_names.get(peer_uid):
cached_name = self._conv_names[peer_uid]
if is_generic_peer_name(name, peer_uid):
name = cached_name
if conv_id and self._conv_names.get(conv_id) and is_generic_peer_name(name, peer_uid):
name = self._conv_names[conv_id]
if peer_uid and (is_generic_peer_name(name, peer_uid) or not avatar):
profile = await fetch_peer_profile(self.session, peer_uid, self.account_id)
if profile.get("nickname"):
name = profile["nickname"]
self._conv_names[peer_uid] = name
if profile.get("avatar_url"):
avatar = profile["avatar_url"]
if profile.get("uid"):
peer_uid = str(profile["uid"])
if not name:
name = self._resolve_sender_name(
{"sender_uid": peer_uid, "conversation_id": conv_id, "sender_name": sender}
)
return name, avatar, peer_uid
async def _fetch_message_by_id(self, conv_id: str, server_message_id: str) -> dict | None:
"""按 server_message_id 调 get_by_conversation 拉取该条消息的完整数据
(含真实 content / message_type / URL)。命中返回原始消息 dict,否则 None。"""
if not conv_id or not server_message_id:
return None
try:
from .auth import DouyinAuth
controller = get_traffic_controller()
async with controller.background_slot(self.account_id, "message detail fetch"):
meta = self._conv_meta.get(conv_id, {})
short_id = str(meta.get("conversation_short_id") or "")
auth = DouyinAuth.from_im_session(self.session)
my_uid = int(self.session.my_uid or 0)
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
if not short_id:
peer_uid = resolve_peer_uid(conv_id, my_uid)
if peer_uid:
_, short_id, _ = await http.get_conversation_info(
auth, int(peer_uid), my_uid, conv_id, 0
)
if short_id:
self._conv_meta[conv_id] = {
**self._conv_meta.get(conv_id, {}),
"conversation_short_id": short_id,
}
messages = await http.get_conversation_messages(
auth, conv_id, int(short_id or 0), limit=20
)
for m in messages:
if str(m.get("server_message_id") or "") == server_message_id:
return m
except Exception as e:
logger.debug(f"_fetch_message_by_id failed: {e}")
return None
async def _enrich_media_content(self, conv_id: str, server_message_id: str, content: str) -> str:
"""媒体消息(相册图片/语音/视频)WS 推送 content 为空时,按 server_message_id
调 get_by_conversation 拉取真实内容并补全 URL。命中失败则原样返回。"""
if not conv_id or not server_message_id or not content:
return content
try:
from .message_content import parse_stored_content, format_im_message, serialize_message_content
parsed = parse_stored_content(content)
mtype = parsed.get("type")
if mtype not in ("image", "voice", "video"):
return content
if parsed.get("url"):
return content # 已有 URL(如商店表情/带 url 的图)
m = await self._fetch_message_by_id(conv_id, server_message_id)
if m:
real = format_im_message(m.get("content") or "", int(m.get("message_type") or 0))
if real.get("url"):
enriched = serialize_message_content(real)
logger.info(
"Enriched media via get_by_conversation: smid=%s type=%s",
server_message_id, real.get("type"),
)
return enriched
except Exception as e:
logger.debug(f"_enrich_media_content failed: {e}")
return content
async def _handle_incoming(self, msg: dict):
# asyncio.Lock 按等待顺序唤醒。锁只覆盖解析、去重、规则匹配与入队;
# 未启用排队时,真正的网络发送仍在锁外执行,保持原有并发行为。
async with self._incoming_lock:
immediate_reply = await self._prepare_incoming(msg)
if immediate_reply is not None and self._running:
await immediate_reply()
async def _prepare_incoming(
self,
msg: dict,
) -> Optional[Callable[[], Awaitable[None]]]:
if self._is_self_message(msg):
return
conv_id = msg.get("conversation_id") or ""
sender_uid = str(msg.get("sender_uid") or "")
sender = self._resolve_sender_name(msg)
sender_avatar = str(msg.get("sender_avatar") or "").strip()
sender, sender_avatar, peer_uid = await self._resolve_peer_profile(
conv_id, sender_uid, sender, sender_avatar
)
content = (msg.get("content") or "").strip()
has_raw_ws = "raw_content" in msg
raw_incoming = msg.get("raw_content") if has_raw_ws else None
ws_message_type = msg.get("message_type")
# 每条 WS 消息带唯一 server_message_id:用它去重,避免“同一用户重复发送
# 相同文字(如多次‘你好’)被按内容去重而整条丢弃”,这是“有时收不到”的根因。
# HTTP 轮询的会话预览没有该 ID,则退回按 内容 去重(避免对同一未读重复回复)。
server_message_id = str(msg.get("server_message_id") or "")
# WS 仅推送瘦消息(如 type=26)content 为空:按 server_message_id 回 HTTP 拉取
# 完整消息,补全 content / message_type,确保「接收到的全部信息」都被记录。
if not content and server_message_id and not (raw_incoming or "").strip():
real = await self._fetch_message_by_id(conv_id, server_message_id)
if real:
real_content = (real.get("content") or "").strip()
if real_content:
raw_incoming = real.get("content")
has_raw_ws = True
real_type = real.get("message_type")
if real_type is not None:
ws_message_type = real_type
try:
from .message_content import format_im_message, serialize_message_content
parsed = format_im_message(real.get("content") or "", int(real_type or 0))
content = serialize_message_content(parsed) if parsed else real_content
except Exception:
content = real_content
logger.info(
"Enriched empty WS push via get_by_conversation: smid=%s type=%s",
server_message_id, real_type,
)
# 相册图片/语音等 WS 推送 content 为空,按 server_message_id 拉取真实内容补 URL
content = await self._enrich_media_content(conv_id, server_message_id, content)
unread = int(msg.get("unread_count") or 0)
if conv_id:
self._conv_meta[conv_id] = {
**self._conv_meta.get(conv_id, {}),
"conversation_id": conv_id,
"sender_name": sender,
"sender_avatar": sender_avatar or self._conv_meta.get(conv_id, {}).get("sender_avatar"),
"content": content or self._conv_meta.get(conv_id, {}).get("content", ""),
"unread_count": unread,
"peer_uid": peer_uid,
}
if sender and peer_uid:
self._conv_names[peer_uid] = sender
if not content and unread <= 0 and raw_incoming is None and not server_message_id:
return
if content == "[未读消息]" and sender in self._conv_previews:
content = self._conv_previews.get(sender, content)
if server_message_id:
log_key = f"mid:{server_message_id}"
key = f"mid:{server_message_id}"
else:
# HTTP 会话预览通常没有 message_id;必须带 conversation_id/peer_uid,
# 否则两个同名用户发送相同内容会被误判成同一条消息。
conversation_key = str(conv_id or peer_uid or sender or "unknown")
log_key = self._reply_key(conversation_key, content or "[未读]")
key = self._reply_key(conversation_key, content)
log_kwargs = {
"sender_name": sender,
"sender_id": peer_uid or conv_id or None,
"sender_avatar": sender_avatar or self._conv_meta.get(conv_id, {}).get("sender_avatar"),
"message": content or (raw_incoming if raw_incoming is not None else ""),
}
# 接收消息原始日志:WS content 原样落库(瘦推送已回 HTTP 补全为真实 content)
if self.received_log_fn and has_raw_ws:
recv_key = f"recv:mid:{server_message_id}" if server_message_id else f"recv:{log_key}"
if recv_key not in self._received_logged_keys:
self._received_logged_keys.add(recv_key)
message_type = ws_message_type
try:
message_type = int(message_type) if message_type is not None else None
except (TypeError, ValueError):
message_type = None
await self.received_log_fn(
sender_name=sender,
sender_id=peer_uid or conv_id or None,
sender_avatar=log_kwargs.get("sender_avatar"),
raw_content="" if raw_incoming is None else raw_incoming,
conversation_id=conv_id or None,
message_type=message_type,
server_message_id=server_message_id or None,
)
if log_key not in self._logged_keys and content:
self._logged_keys.add(log_key)
await self.log_fn(
**log_kwargs,
reply=None,
status="received",
)
try:
from .message_content import format_system_log_message, parse_stored_content
parsed = parse_stored_content(content)
msg_type = parsed.get("type") or "text"
detail = format_system_log_message(content)
if server_message_id:
detail = f"{detail} | mid={server_message_id}"
system_logger.record(
f"收到{'' if msg_type == 'text' else '['+msg_type+']'}消息:{sender}",
detail=detail,
level="info",
category="recv",
account_id=self.account_id,
)
except Exception as exc:
logger.debug(f"record recv system log failed: {exc}")
if key in self._replied_keys:
return
# WS 与 HTTP 轮询可能同时发现同一条消息。检查后立即占位(中间不 await),
# 防止延迟排队期间被重复加入发送队列。
self._replied_keys.add(key)
# 同账号、同会话只保留一个尚未发送的回复任务。后续来信只追加到
# 原任务详情,不改变它的发送时间、位置或已经匹配好的回复。
queue_merge_keys = self._reply_queue_merge_keys(conv_id, peer_uid)
if queue_merge_keys and self._running:
merge_result = await self._reply_queue.merge_pending(
queue_merge_keys,
lambda existing: self._merge_reply_queue_details(
existing,
incoming_content=content or "",
sender_name=sender,
sender_id=peer_uid or conv_id or "",
sender_avatar=log_kwargs.get("sender_avatar"),
conversation_id=conv_id,
),
)
if merge_result.get("status") == "merged":
if content:
self._conv_previews[sender] = content
message_count = int(merge_result.get("message_count") or 1)
logger.info(
"Merged message into queued reply for %s on account %s: "
"job=%s messages=%s position=%s",
sender,
self.account_id,
merge_result.get("job_id"),
message_count,
merge_result.get("position"),
)
system_logger.record(
"同一会话消息已合并到回复队列",
detail=(
f"{sender} 的新消息已并入原任务;当前共 {message_count} 条消息,"
"发送时间和队列位置保持不变。"
),
level="info",
category="send",
account_id=self.account_id,
)
return
# 收到新消息即尝试自动回复,不按消息类型/托管关系/内容形态过滤
replies = await self.match_reply(content if content != "[未读消息]" else "")
if not replies:
replies = await self.match_reply("")
if not replies:
await self.log_fn(
**log_kwargs,
reply=None,
status="ignored",
error="未配置任何自动回复规则,请在「自动回复规则」中添加至少一条启用规则",
)
if content:
self._conv_previews[sender] = content
return
# 冷却窗口:同一用户在设定时间内,无论发多少条消息,只自动回复一次(账号设置优先,否则全局)
peer_key = (peer_uid or conv_id or sender or "").strip()
cooldown = await self._resolve_cooldown_seconds()
if self._peer_in_cooldown(peer_key, cooldown):
logger.info(
f"Auto-reply to {sender} skipped: within {cooldown}s cooldown window"
)
if content:
self._conv_previews[sender] = content
system_logger.record(
"自动回复已跳过(冷却中)",
detail=f"{sender} 在 {cooldown} 秒冷却窗口内重复发送,未重复回复",
level="info",
category="send",
account_id=self.account_id,
)
return
# 提前标记回复时间,确保冷却窗口内(含延迟期间)的后续消息都被抑制
if peer_key and cooldown > 0:
self._last_reply_at[peer_key] = time.monotonic()
if content:
self._conv_previews[sender] = content
delay_seconds = await self._resolve_reply_delay_seconds()
if not self._running:
return
async def send_reply() -> None:
await self._send_auto_reply(
sender=sender,
content=content,
conv_id=conv_id,
replies=replies,
peer_key=peer_key,
cooldown=cooldown,
log_kwargs=log_kwargs,
)
if delay_seconds > 0:
position = await self._reply_queue.enqueue(
delay_seconds,
send_reply,
description=f"回复 {sender}",
details={
"sender_name": sender,
"sender_id": peer_uid or conv_id or None,
"sender_avatar": log_kwargs.get("sender_avatar"),
"conversation_id": conv_id or None,
"incoming_content": content or "",
"incoming_contents": [content or ""],
"message_count": 1,
"replies": list(replies),
},
merge_keys=queue_merge_keys,
)
logger.info(
"Queued reply to %s for account %s: position=%s interval=%ss",
sender,
self.account_id,
position,
delay_seconds,
)
system_logger.record(
"自动回复已进入账号队列",
detail=(
f"{sender} 当前排在第 {position} 位;账号生效间隔为 {delay_seconds} 秒,"
"账号内计时与排位独立;到点后再进入全局带宽队列逐条投递。"
),
level="info",
category="send",
account_id=self.account_id,
)
return
# 账号与系统均未配置排队间隔:跳过排队规则,保持原来的立即回复。
return send_reply
async def _send_auto_reply(
self,
*,
sender: str,
content: str,
conv_id: str,
replies: list[str],
peer_key: str,
cooldown: int,
log_kwargs: dict,
) -> None:
"""发送一项已匹配的自动回复任务,并记录原有消息/系统日志。"""
if not self._running:
return
reply_displays: list[str] = []
sent_any = False
send_error = ""
meta = self._conv_meta.get(conv_id, {})
for index, reply in enumerate(replies):
if not self._running:
send_error = self.last_error or "托管已停止,后续回复已取消"
break
if index > 0:
await asyncio.sleep(0.6)
if not self._running:
send_error = self.last_error or "托管已停止,后续回复已取消"
break
reply_display = format_reply_display(reply)
reply_displays.append(reply_display)
sent = False
if conv_id:
sent, resolved = await self._send_text(
conv_id,
reply,
conversation_short_id=str(meta.get("conversation_short_id") or ""),
)
if sent:
if resolved:
meta = {**meta, **resolved, "conversation_id": conv_id}
self._conv_meta[conv_id] = meta
else:
send_error = self.last_error or "IM API 发送失败"
else:
send_error = "缺少会话 ID,无法发送自动回复"
if sent:
sent_any = True
combined_display = " | ".join(reply_displays)
# 日志里存结构化内容(单条直接存 payload,多条用 {"messages":[...]} 包裹),
# 这样图片/表情等媒体回复会被前端渲染为真实媒体,而不是被压成 "图片" 占位文字。
reply_log_content = serialize_reply_log(replies)
if not sent_any:
logger.warning(
f"IM API send failed for [{sender}]: {send_error}; reply saved to log only"
)
# 发送彻底失败:清除冷却时间戳,避免把没收到回复的用户锁在冷却窗口内
if peer_key and cooldown > 0:
self._last_reply_at.pop(peer_key, None)
await self.log_fn(
**log_kwargs,
reply=reply_log_content,
status="replied" if sent_any else "failed",
error=None if sent_any else (send_error or "IM API 发送失败"),
)
if sent_any:
system_logger.record(
"自动回复成功",
detail=f"已回复 {sender}:{combined_display}",
level="success",
category="send",
account_id=self.account_id,
)
else:
system_logger.record(
"自动回复失败",
detail=f"回复 {sender} 失败:{send_error}(收到:{content})",
level="error",
category="send",
account_id=self.account_id,
)
logger.info(
f"Auto-reply to {sender}: {content!r} -> {combined_display!r} "
f"(sent={sent_any}, count={len(replies)})"
)
async def _index_conversations(self, conversations: list[dict]):
my_uid = int(self.session.my_uid or 0)
for raw in conversations:
conv = enrich_conversation_item(raw, my_uid)
conv_id = str(conv.get("conversation_id") or "")
name = (conv.get("sender_name") or "").strip()
avatar = str(conv.get("sender_avatar") or "").strip()
peer_uid = str(conv.get("peer_uid") or "")
if peer_uid and (is_generic_peer_name(name, peer_uid) or not avatar):
profile = await fetch_peer_profile(self.session, peer_uid, self.account_id)
if profile.get("nickname"):
name = profile["nickname"]
conv["sender_name"] = name
if profile.get("avatar_url"):
avatar = profile["avatar_url"]
conv["sender_avatar"] = avatar
if conv_id:
self._conv_meta[conv_id] = {
**conv,
"sender_name": name,
"sender_avatar": avatar or None,
"peer_uid": peer_uid,
}
if name:
self._conv_names[conv_id] = name
if peer_uid and name:
self._conv_names[peer_uid] = name
async def _poll_conversations(self):
controller = get_traffic_controller()
async with controller.background_slot(self.account_id, "conversation poll"):
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
unread_total = await http.get_unread_count()
if unread_total:
logger.info(f"IM unread total: {unread_total}")
conversations = await http.get_conversations()
await self._index_conversations(conversations)
# Message handling may wait in the global send lane. Do not keep one
# of the scarce background HTTP slots occupied while that happens.
for conv in conversations:
unread = int(conv.get("unread_count") or 0)
if unread > 0 or conv.get("content"):
await self._handle_incoming(conv)
async def _verify_account_uid(self):
"""启动时用 query/user 接口核验账号真实 UID,修正采集端可能取错的 my_uid/device_id。
采集端从 tea_cache 推断的 my_uid 可能是访客/对方 id,会导致会话列表为 0、
创建会话 INVALID_REQUEST。这里在建连前先校正,保证后续所有请求身份正确。
"""
if getattr(self.session, "uid_verified", False) and self.session.my_uid:
return
try:
from .auth import DouyinAuth
auth = DouyinAuth.from_im_session(self.session)
controller = get_traffic_controller()
async with controller.background_slot(self.account_id, "account UID verify"):
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
old = int(self.session.my_uid or 0)
resolved = await asyncio.to_thread(http._resolve_authoritative_uid, auth)
if resolved and old and int(resolved) != old:
system_logger.record(
"已自动校正账号 UID",
detail=f"采集端识别 UID={old},接口核验真实 UID={resolved},已修正后再建立私信连接。",
level="info",
category="system",
account_id=self.account_id,
)
except Exception as e:
logger.warning(f"启动核验账号 UID 失败(沿用采集值):{e}")
async def run(self):
"""主循环:WebSocket + HTTP 轮询"""
self._running = True
await self._reply_queue.start()
await self._verify_account_uid()
# ensure_frontier_ws 可能触发签名/HTTP(阻塞),放线程池避免多账号启动时卡死事件循环
controller = get_traffic_controller()
async with controller.background_slot(self.account_id, "frontier discovery"):
await asyncio.to_thread(ensure_frontier_ws, self.session)
has_ws = bool(self.session.frontier_ws_url())
cred_summary = format_session_credential_summary(self.session)
logger.info(cred_summary)
logger.info(
f"Starting IM direct service for account {self.account_id} "
f"(ws={'yes' if has_ws else 'no'})"
)
system_logger.record(
"私信托管已启动",
detail=f"实时接收通道:{'已就绪' if has_ws else '不可用(仅 HTTP 轮询)'}\n{cred_summary}",
level="success" if has_ws else "warning",
category="system",
account_id=self.account_id,
)
try:
from .emoji_pack import ensure_emoji_map, is_fresh
if not is_fresh():
async with controller.background_slot(self.account_id, "emoji preload"):
await asyncio.to_thread(ensure_emoji_map, self.session)
except Exception as e:
logger.debug(f"emoji map preload failed: {e}")
self._ws_client = DouyinImWsClient(
self.session, self._handle_incoming, account_id=self.account_id
)
await self._ws_client.start()
try:
await self._poll_conversations()
except Exception as e:
logger.warning(f"Initial conversation poll failed: {e}")
system_logger.record(
"首次会话轮询失败",
detail=f"{e}",
level="warning",
category="poll",
account_id=self.account_id,
)
loop_count = 0
while self._running:
# The initial poll above is authoritative. Sleep before the next
# recurring tick so startup cannot issue two back-to-back polls.
await asyncio.sleep(5)
if not self._running:
break
loop_count += 1
try:
if loop_count % 3 == 0:
await self._poll_conversations()
if loop_count % 6 == 0:
logger.info(f"IM direct tick #{loop_count} account={self.account_id}")
# 关注欢迎语:约每 60s 检测一次新粉丝(独立于私信轮询,失败不影响主循环)
if self.follow_tick and loop_count % 12 == 0:
try:
await self.follow_tick()
except Exception as e:
logger.error(f"follow welcome tick error: {e}")
except Exception as e:
logger.error(f"IM poll error: {e}")
system_logger.record(
"会话轮询出错",
detail=f"拉取会话/未读时出错:{e}",
level="error",
category="poll",
account_id=self.account_id,
)
async def stop(self):
self._running = False
await get_traffic_controller().send_queue.cancel_account(self.account_id)
await self._reply_queue.stop()
if self._ws_client:
await self._ws_client.stop()
def get_cached_conversations(self) -> list[dict]:
"""返回运行中缓存的会话(来自 WS / 轮询)。"""
results = []
seen = set()
for conv_id, meta in self._conv_meta.items():
name = (meta.get("sender_name") or "").strip()
key = conv_id or name
if not key or key in seen:
continue
seen.add(key)
results.append({
"conversation_id": conv_id,
"sender_name": name or f"会话{conv_id[-8:]}" if conv_id else "未知用户",
"sender_avatar": meta.get("sender_avatar") or None,
"sender_id": str(meta.get("peer_uid") or meta.get("sender_id") or conv_id or ""),
"peer_uid": str(meta.get("peer_uid") or ""),
"content": str(meta.get("content") or ""),
"unread_count": int(meta.get("unread_count") or 0),
})
return results
async def get_reply_queue_snapshot(self) -> list[dict]:
"""返回当前账号自动回复队列的可管理快照。"""
return await self._reply_queue.snapshot()
async def send_queued_reply_now(self, job_id: str) -> dict:
"""把指定自动回复任务移入账号紧急队列;实际发送仍由单消费者串行执行。"""
return await self._reply_queue.send_now(job_id)
async def _send_text(
self,
conversation_id: str,
content: str,
conversation_short_id: str = "",
) -> tuple[bool, Optional[dict]]:
"""发送一条私信;若因签名凭证失效(7911)失败,刷新 web_protect 后自动重试一次。
返回 (是否成功, 解析到的会话 meta)。失败原因写入 self.last_error。
"""
for attempt in range(2):
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
sent = await http.send_text_message(
conversation_id,
content,
conversation_short_id=conversation_short_id,
)
self.last_error = http.last_error
needs_refresh = http.last_send_needs_refresh
if sent:
resolved = http.last_send_meta.get(conversation_id)
self.session.conv_meta.update(http.session.conv_meta)
self._session_invalid_strikes = 0 # 发送成功 → 登录有效
return True, resolved
# 仅在“签名凭证失效”时刷新并重试一次
if attempt == 0 and needs_refresh and self.refresh_credentials:
logger.warning(
f"Send hit credential-expiry(7911) for {conversation_id}; "
"refreshing web_protect and retrying once..."
)
try:
refreshed = await self.refresh_credentials()
except Exception as e:
logger.warning(f"refresh_credentials raised: {e}")
refreshed = False
if refreshed:
continue
break
await self._note_session_invalid(self.last_error)
return False, None
async def _note_session_invalid(self, error: str) -> None:
"""根据发送失败原因判断 IM 是否已退出登录;连续 INVALID_REQUEST 即触发自动下线。
INVALID_REQUEST 来自 create_conversation/发送:会话/签名被抖音判为无效,强相关于「登录失效」。
而 8xxx/7xxx 等业务错误(关系/频控/内容)说明请求已到达抖音、登录仍有效,重置计数。
"""
err = error or ""
if "INVALID_REQUEST" not in err:
self._session_invalid_strikes = 0
return
self._session_invalid_strikes += 1
if self._session_invalid_strikes < 2 or self._session_invalid_fired:
return
self._session_invalid_fired = True
reason = "IM 会话失效(INVALID_REQUEST),登录可能已退出"
logger.warning(
f"Account {self.account_id} {reason};连续 {self._session_invalid_strikes} 次 -> 自动下线"
)
system_logger.record(
"IM 登录失效,自动下线",
detail=f"{reason}(连续 {self._session_invalid_strikes} 次发送返回 INVALID_REQUEST)。"
"请停止托管后用浏览器模式重新登录并打开私信页,再重新启动托管。",
level="error",
category="auth",
account_id=self.account_id,
)
self._running = False # 让主循环尽快退出
if self.on_session_invalid:
try:
await self.on_session_invalid(reason)
except Exception as e:
logger.error(f"on_session_invalid handler error: {e}")
async def send_message(self, conversation_id: str, content: str) -> bool:
"""手动发送私信"""
from .conv_util import normalize_conversation_id
from .auth import DouyinAuth
auth = DouyinAuth()
auth.perepare_auth(
self.session.cookie_header(),
self.session.web_protect_str,
self.session.keys_str,
)
if getattr(self.session, "uid_verified", False) and self.session.my_uid:
my_uid = self.session.my_uid
else:
my_uid = await asyncio.to_thread(lambda: auth.get_uid()) or self.session.my_uid
if my_uid:
conversation_id = normalize_conversation_id(conversation_id, my_uid)
meta = self._conv_meta.get(conversation_id, {})
sent, resolved = await self._send_text(
conversation_id,
content,
conversation_short_id=str(meta.get("conversation_short_id") or ""),
)
if sent and resolved:
self._conv_meta[conversation_id] = {
**meta,
**resolved,
"conversation_id": conversation_id,
}
return sent
+260
View File
@@ -0,0 +1,260 @@
import json
import re
import time
from dataclasses import dataclass, field
from typing import Any, Optional
from urllib.parse import parse_qs, unquote, urlparse
IM_TOKEN_COOKIES = ("sessionid", "sessionid_ss")
def is_frontier_ws_url(url: str) -> bool:
"""判断是否为抖音 IM frontier 长连接地址。
真实抓包里 host 可能是 frontier-im.douyin.com,也可能是
frontierNN-normal.zijieapi.com 这类内部别名,二者都要认。
"""
if not url or "token=" not in url:
return False
return "frontier-im.douyin.com" in url or ("frontier" in url and "zijieapi.com" in url)
@dataclass
class DouyinImSession:
"""抖音 IM 直连所需会话信息(从 Cookie + 浏览器抓包获得)"""
cookies: dict = field(default_factory=dict)
ws_urls: list = field(default_factory=list)
device_id: str = ""
web_id: str = ""
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
keys_str: str = ""
web_protect_str: str = ""
my_uid: int = 0
# my_uid 是否已用 query/user 接口核验过(采集端推断的 my_uid 可能取错 tea_cache id)
uid_verified: bool = False
conv_meta: dict = field(default_factory=dict)
# 方案 A:直接复用浏览器抓到的真实 frontier 连接凭证(绕开我们自己推导 token/access_key 不准的问题)
sdk_cert: str = "" # bd-ticket-guard 客户端证书(frontier sdk_cert / HTTP client-cert)
frontier_ts_sign: str = "" # 抓包得到的新鲜 ts_sign(覆盖 web_protect 里可能已过期的)
@classmethod
def from_storage_state(cls, data: dict, extra: Optional[dict] = None) -> "DouyinImSession":
extra = extra or {}
cookies = {}
cookie_items = data.get("cookies", [])
priority_names = set(IM_TOKEN_COOKIES)
def pick_best_cookie(name: str) -> str:
matches = [
c for c in cookie_items
if c.get("name") == name and c.get("value")
]
if not matches:
return ""
matches.sort(
key=lambda c: (
0 if ".douyin.com" in (c.get("domain") or "") else 1,
-len(c.get("value") or ""),
)
)
return matches[0].get("value") or ""
for item in cookie_items:
name = item.get("name")
if not name or any(c in name for c in "()[]{}'\"\n \t\\"):
continue
val = item.get("value") or ""
if name in priority_names:
best = pick_best_cookie(name)
if best:
cookies[name] = best
elif name not in cookies or len(val) > len(cookies.get(name, "")):
cookies[name] = val
device_id = extra.get("device_id") or cookies.get("device_id") or ""
web_id = extra.get("web_id") or ""
keys_str = extra.get("keys_str") or ""
web_protect_str = extra.get("web_protect_str") or ""
# my_uid 优先级:浏览器实时采集(extra) > storage_state 顶层(凭证采集工具手填/抓取)。
# 顶层 my_uid 让导入的明文数字 UID 直接生效,避免后端用加密 uid_tt 解析失败而回退联网查询。
def _as_uid(v) -> int:
try:
return int(str(v).strip())
except (TypeError, ValueError):
return 0
my_uid = _as_uid(extra.get("my_uid")) or _as_uid(data.get("my_uid"))
user_agent = str(extra.get("user_agent") or data.get("user_agent") or "").strip()
if not device_id or not web_id or not keys_str or not web_protect_str or not my_uid:
for origin in data.get("origins", []):
for entry in origin.get("localStorage", []):
name = entry.get("name", "")
value = entry.get("value", "")
if not value:
continue
if name == "security-sdk/s_sdk_crypt_sdk" and not keys_str:
keys_str = value
if name == "security-sdk/s_sdk_sign_data_key/web_protect" and not web_protect_str:
web_protect_str = value
if "tea_cache_tokens" in name and not web_id:
try:
parsed = json.loads(value)
web_id = str(
parsed.get("web_id")
or parsed.get("user_unique_id")
or ""
)
except Exception:
pass
if name == "web_runtime_security_uid" and not device_id:
if str(value or "").isdigit():
device_id = value
if "tea_cache_tokens" in name and not my_uid:
try:
parsed = json.loads(value)
uid = parsed.get("user_unique_id")
if uid and str(uid).isdigit():
my_uid = int(uid)
except Exception:
pass
if not my_uid:
for item in data.get("cookies", []):
cname = item.get("name") or ""
if cname in ("uid_tt", "uid_tt_ss") and item.get("value"):
try:
my_uid = int(item.get("value"))
break
except (TypeError, ValueError):
pass
if not device_id and my_uid:
device_id = str(my_uid)
elif not device_id and web_id:
device_id = web_id
ws_urls = list(extra.get("ws_urls") or [])
# 方案 A:凭证采集工具可携带浏览器抓到的真实 frontier 连接(含 token/sdk_cert/ts_sign)。
frontier_ws_url = str(
extra.get("frontier_ws_url") or data.get("frontier_ws_url") or ""
).strip()
sdk_cert = str(extra.get("sdk_cert") or data.get("sdk_cert") or "").strip()
frontier_ts_sign = str(
extra.get("frontier_ts_sign")
or data.get("frontier_ts_sign")
or data.get("ts_sign")
or ""
).strip()
if is_frontier_ws_url(frontier_ws_url):
# 真实抓包 URL 优先,放在最前面
ws_urls = [frontier_ws_url] + [u for u in ws_urls if u != frontier_ws_url]
# 从真实 URL 里补抽 sdk_cert / ts_sign(用户只贴了 URL 时)。
# 注意:不能用 parse_qs(它会把 + 解成空格,毁掉 base64 证书),用 unquote。
def _q(url: str, key: str) -> str:
m = re.search(rf"[?&]{re.escape(key)}=([^&\s]+)", url)
return unquote(m.group(1)) if m else ""
if not sdk_cert:
sdk_cert = _q(frontier_ws_url, "sdk_cert")
if not frontier_ts_sign:
frontier_ts_sign = _q(frontier_ws_url, "ts_sign")
return cls(
cookies=cookies,
ws_urls=ws_urls,
device_id=str(device_id or ""),
web_id=str(web_id or ""),
keys_str=keys_str,
web_protect_str=web_protect_str,
my_uid=my_uid,
user_agent=user_agent if user_agent else cls.user_agent,
sdk_cert=sdk_cert,
frontier_ts_sign=frontier_ts_sign,
)
def to_dict(self) -> dict:
return {
"cookies": self.cookies,
"ws_urls": self.ws_urls,
"device_id": self.device_id,
"web_id": self.web_id,
"user_agent": self.user_agent,
"keys_str": self.keys_str,
"web_protect_str": self.web_protect_str,
"my_uid": self.my_uid,
"conv_meta": self.conv_meta,
"sdk_cert": self.sdk_cert,
"frontier_ts_sign": self.frontier_ts_sign,
"saved_at": time.time(),
}
@classmethod
def from_dict(cls, data: dict) -> "DouyinImSession":
if not data:
return cls()
return cls(
cookies=data.get("cookies") or {},
ws_urls=data.get("ws_urls") or [],
device_id=str(data.get("device_id") or ""),
web_id=str(data.get("web_id") or ""),
user_agent=data.get("user_agent") or cls.user_agent,
keys_str=str(data.get("keys_str") or ""),
web_protect_str=str(data.get("web_protect_str") or ""),
my_uid=int(data.get("my_uid") or 0),
conv_meta=dict(data.get("conv_meta") or {}),
sdk_cert=str(data.get("sdk_cert") or ""),
frontier_ts_sign=str(data.get("frontier_ts_sign") or ""),
)
def cookie_header(self) -> str:
parts = []
for name, value in self.cookies.items():
if name and value is not None:
parts.append(f"{name}={value}")
return "; ".join(parts)
def has_login(self) -> bool:
login_keys = {
"sessionid",
"sessionid_ss",
"sid_tt",
"sid_guard",
"passport_auth_status",
"odin_tt",
}
return any(self.cookies.get(k) for k in login_keys)
def can_direct_im(self) -> bool:
"""是否具备 Cookie 直连 IM 的最低条件(无需浏览器)"""
return self.has_login() and bool(
self.cookies.get("sessionid") or self.cookies.get("sessionid_ss")
)
def frontier_ws_url(self) -> Optional[str]:
"""仅返回 IM frontier 地址,忽略浏览器抓到的 bytelink 等无关 WS。"""
for url in self.ws_urls:
if is_frontier_ws_url(url):
return url
return None
def sanitize_ws_urls(self) -> None:
self.ws_urls = [url for url in self.ws_urls if is_frontier_ws_url(url)]
def common_params(self) -> dict[str, str]:
return {
"aid": "6383",
"app_name": "douyin_web",
"device_platform": "webapp",
"channel": "channel_pc_web",
"pc_client_type": "1",
"version_code": "170400",
"version_name": "17.4.0",
"device_id": self.device_id or self.web_id or "",
"webid": self.web_id or self.device_id or "",
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,60 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: Request.proto
# Protobuf Python Version: 5.27.1
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
5,
27,
1,
'',
'Request.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rRequest.proto\"&\n\x08\x45xtValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x94\x04\n\x07Request\x12\x0b\n\x03\x63md\x18\x01 \x01(\x05\x12\x13\n\x0bsequence_id\x18\x02 \x01(\x03\x12\x13\n\x0bsdk_version\x18\x03 \x01(\t\x12\r\n\x05token\x18\x04 \x01(\t\x12\r\n\x05refer\x18\x05 \x01(\x05\x12\x12\n\ninbox_type\x18\x06 \x01(\x03\x12\x14\n\x0c\x62uild_number\x18\x07 \x01(\t\x12\x1a\n\x04\x62ody\x18\x08 \x01(\x0b\x32\x0c.RequestBody\x12\x11\n\tdevice_id\x18\t \x01(\t\x12\x0f\n\x07\x63hannel\x18\n \x01(\t\x12\x17\n\x0f\x64\x65vice_platform\x18\x0b \x01(\t\x12\x13\n\x0b\x64\x65vice_type\x18\x0c \x01(\t\x12\x12\n\nos_version\x18\r \x01(\t\x12\x14\n\x0cversion_code\x18\x0e \x01(\t\x12&\n\x07headers\x18\x0f \x03(\x0b\x32\x15.Request.HeadersEntry\x12\x11\n\tconfig_id\x18\x10 \x01(\x05\x12\x1e\n\ntoken_info\x18\x11 \x01(\x0b\x32\n.TokenInfo\x12\x11\n\tauth_type\x18\x12 \x01(\x05\x12\x0b\n\x03\x62iz\x18\x15 \x01(\t\x12\x0e\n\x06\x61\x63\x63\x65ss\x18\x16 \x01(\t\x12\x0f\n\x07ts_sign\x18\x17 \x01(\t\x12\x10\n\x08sdk_cert\x18\x18 \x01(\t\x12\x14\n\x0creuqest_sign\x18\x19 \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xeb\x01\n\x0bRequestBody\x12\x34\n\x11send_message_body\x18\x64 \x01(\x0b\x32\x17.SendMessageRequestBodyH\x00\x12H\n\x1b\x63reate_conversation_v2_body\x18\xe1\x04 \x01(\x0b\x32 .CreateConversationV2RequestBodyH\x00\x12T\n\"get_conversation_info_list_v2_body\x18\xe2\x04 \x01(\x0b\x32%.GetConversationInfoListV2RequestBodyH\x00\x42\x06\n\x04\x62ody\"\xb8\x02\n\x16SendMessageRequestBody\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x02 \x01(\x05\x12\x1d\n\x15\x63onversation_short_id\x18\x03 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x04 \x01(\t\x12\x16\n\x03\x65xt\x18\x05 \x03(\x0b\x32\t.ExtValue\x12\x14\n\x0cmessage_type\x18\x06 \x01(\x05\x12\x0e\n\x06ticket\x18\x07 \x01(\t\x12\x19\n\x11\x63lient_message_id\x18\x08 \x01(\t\x12\x17\n\x0fmentioned_users\x18\t \x03(\x03\x12\x1a\n\x12ignore_badge_count\x18\n \x01(\x08\x12,\n\x0cref_msg_info\x18\x0b \x01(\x0b\x32\x16.ReferencedMessageInfo\"y\n\x15ReferencedMessageInfo\x12\x1b\n\x13original_message_id\x18\x01 \x01(\x03\x12\x1f\n\x17original_message_sender\x18\x02 \x01(\t\x12\"\n\x1aoriginal_message_timestamp\x18\x03 \x01(\x03\"^\n\tTokenInfo\x12\x0f\n\x07mark_id\x18\x01 \x01(\x05\x12\x0c\n\x04type\x18\x02 \x01(\x05\x12\x0e\n\x06\x61pp_id\x18\x03 \x01(\x05\x12\x0f\n\x07user_id\x18\x04 \x01(\x03\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\"\xa2\x02\n\x1f\x43reateConversationV2RequestBody\x12\x19\n\x11\x63onversation_type\x18\x01 \x01(\x05\x12\x14\n\x0cparticipants\x18\x02 \x03(\x03\x12\x12\n\npersistent\x18\x03 \x01(\x08\x12\x15\n\ridempotent_id\x18\x04 \x01(\t\x12\x0c\n\x04name\x18\x05 \x01(\t\x12\x12\n\navatar_url\x18\x06 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x07 \x01(\t\x12=\n\x07\x62iz_ext\x18\x08 \x03(\x0b\x32,.CreateConversationV2RequestBody.BizExtEntry\x1a-\n\x0b\x42izExtEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n$GetConversationInfoListV2RequestBody\x12\x38\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32*.GetConversationInfoListV2ResponseBodyData\"~\n)GetConversationInfoListV2ResponseBodyData\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x1d\n\x15\x63onversation_short_id\x18\x02 \x01(\x03\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x62\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'Request_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
DESCRIPTOR._loaded_options = None
_globals['_REQUEST_HEADERSENTRY']._loaded_options = None
_globals['_REQUEST_HEADERSENTRY']._serialized_options = b'8\001'
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._loaded_options = None
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_options = b'8\001'
_globals['_EXTVALUE']._serialized_start=17
_globals['_EXTVALUE']._serialized_end=55
_globals['_REQUEST']._serialized_start=58
_globals['_REQUEST']._serialized_end=590
_globals['_REQUEST_HEADERSENTRY']._serialized_start=544
_globals['_REQUEST_HEADERSENTRY']._serialized_end=590
_globals['_REQUESTBODY']._serialized_start=593
_globals['_REQUESTBODY']._serialized_end=828
_globals['_SENDMESSAGEREQUESTBODY']._serialized_start=831
_globals['_SENDMESSAGEREQUESTBODY']._serialized_end=1143
_globals['_REFERENCEDMESSAGEINFO']._serialized_start=1145
_globals['_REFERENCEDMESSAGEINFO']._serialized_end=1266
_globals['_TOKENINFO']._serialized_start=1268
_globals['_TOKENINFO']._serialized_end=1362
_globals['_CREATECONVERSATIONV2REQUESTBODY']._serialized_start=1365
_globals['_CREATECONVERSATIONV2REQUESTBODY']._serialized_end=1655
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_start=1610
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_end=1655
_globals['_GETCONVERSATIONINFOLISTV2REQUESTBODY']._serialized_start=1657
_globals['_GETCONVERSATIONINFOLISTV2REQUESTBODY']._serialized_end=1753
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODYDATA']._serialized_start=1755
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODYDATA']._serialized_end=1881
# @@protoc_insertion_point(module_scope)
@@ -0,0 +1,46 @@
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# NO CHECKED-IN PROTOBUF GENCODE
# source: Response.proto
# Protobuf Python Version: 5.27.1
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder
_runtime_version.ValidateProtobufRuntimeVersion(
_runtime_version.Domain.PUBLIC,
5,
27,
1,
'',
'Response.proto'
)
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eResponse.proto\"\x82\x01\n\x08Response\x12\x0b\n\x03\x63md\x18\x01 \x01(\x05\x12\x13\n\x0bsequence_id\x18\x02 \x01(\x03\x12\x12\n\nerror_desc\x18\x03 \x01(\t\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\x12\n\ninbox_type\x18\x05 \x01(\x03\x12\x1b\n\x04\x62ody\x18\x06 \x01(\x0b\x32\r.ResponseBody\"\xf8\x01\n\x0cResponseBody\x12\x30\n\x12new_message_notify\x18\xf4\x03 \x01(\x0b\x32\x11.NewMessageNotifyH\x00\x12N\n\x1b\x63reate_conversation_v2_body\x18\xe1\x04 \x01(\x0b\x32&.GetConversationInfoListV2ResponseBodyH\x00\x12^\n+get_conversation_info_list_v2_response_body\x18\xe2\x04 \x01(\x0b\x32&.GetConversationInfoListV2ResponseBodyH\x00\x42\x06\n\x04\x62ody\"z\n\x10NewMessageNotify\x12\x17\n\x0f\x63onversation_id\x18\x02 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x12\x13\n\x0bnotify_type\x18\x04 \x01(\x05\x12\x1d\n\x07message\x18\x05 \x01(\x0b\x32\x0c.MessageBody\"\xd1\x01\n\x0bMessageBody\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x02 \x01(\x05\x12\x19\n\x11server_message_id\x18\x03 \x01(\x03\x12\x1d\n\x15index_in_conversation\x18\x04 \x01(\x03\x12\x1d\n\x15\x63onversation_short_id\x18\x05 \x01(\x03\x12\x14\n\x0cmessage_type\x18\x06 \x01(\x05\x12\x0e\n\x06sender\x18\x07 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x08 \x01(\t\"g\n%GetConversationInfoListV2ResponseBody\x12>\n\x16\x63onversation_info_list\x18\x01 \x03(\x0b\x32\x1e.GetConversationInfoV2Response\"\x82\x01\n\x1dGetConversationInfoV2Response\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x1d\n\x15\x63onversation_short_id\x18\x02 \x01(\x03\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x12\x0e\n\x06ticket\x18\x04 \x01(\tb\x06proto3')
_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'Response_pb2', _globals)
if not _descriptor._USE_C_DESCRIPTORS:
DESCRIPTOR._loaded_options = None
_globals['_RESPONSE']._serialized_start=19
_globals['_RESPONSE']._serialized_end=149
_globals['_RESPONSEBODY']._serialized_start=152
_globals['_RESPONSEBODY']._serialized_end=400
_globals['_NEWMESSAGENOTIFY']._serialized_start=402
_globals['_NEWMESSAGENOTIFY']._serialized_end=524
_globals['_MESSAGEBODY']._serialized_start=527
_globals['_MESSAGEBODY']._serialized_end=736
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODY']._serialized_start=738
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODY']._serialized_end=841
_globals['_GETCONVERSATIONINFOV2RESPONSE']._serialized_start=844
_globals['_GETCONVERSATIONINFOV2RESPONSE']._serialized_end=974
# @@protoc_insertion_point(module_scope)
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,167 @@
ea = function() {
return (ea = Object.assign || function(e) {
for (var t, n = 1, r = arguments.length; n < r; n++)
for (var o in t = arguments[n])
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
return e
}
).apply(this, arguments)
}
var H = function(e) {
for (var t, n = e.toString(), r = [], o = 0; o < n.length; o++)
0 <= (t = n.charCodeAt(o)) && t <= 127 ? r.push(t) : 128 <= t && t <= 2047 ? (r.push(192 | 31 & t >> 6),
r.push(128 | 63 & t)) : (2048 <= t && t <= 55295 || 57344 <= t && t <= 65535) && (r.push(224 | 15 & t >> 12),
r.push(128 | 63 & t >> 6),
r.push(128 | 63 & t));
for (var i = 0; i < r.length; i++)
r[i] &= 255;
return r
}
V = function(e) {
var t = []
, n = [];
if (void 0 === e)
return "";
n = H(e);
for (var r = 0, o = n.length; r < o; ++r)
t.push((5 ^ n[r]).toString(16));
return t.join("")
}
function generate_account_sdk_source_info(){
let browserInfo = {
"hardwareConcurrency": 20,
"webdriver": false,
"chromedriver": false,
"shelldriver": false,
"plugins": 5,
"permissions": [
{
"name": "notifications",
"state": "prompt"
}
],
"innerHeight": 1442,
"innerWidth": 1166,
"outerHeight": 1552,
"outerWidth": 2560,
"stoargeStatus": {
"indexedDB": {
"idb": "object",
"open": "function",
"indexedDB": "object",
"IDBKeyRange": "function",
"openDatabase": "undefined",
"isSafari": false,
"hasFetch": false
},
"localStorage": {
"isSupportLStorage": true,
"size": 46382,
"write": true
},
"storageQuotaStatus": {
"usage": 149822,
"quota": 128849645568,
"isPrivate": false
}
},
"notificationPermission": "default",
"performance": {
// "timeOrigin": 1723036093298,
"timeOrigin": new Date().getTime(),
"usedJSHeapSize": 137509473,
"navigationTiming": {
"decodedBodySize": 633258,
"entryType": "navigation",
"initiatorType": "navigation",
"name": "https://www.douyin.com/?recommend=1",
"renderBlockingStatus": "non-blocking",
"serverTiming": "inner,tt_agw,cdn-cache,edge,origin",
"guleStart": 1496.2999999523163,
"guleDuration": 14.799999952316284
}
}
}
let data = {
request_host: "www.douyin.com",
request_pathname: '/'
}
return V(JSON.stringify(ea(ea({}, browserInfo || {}), data)))
}
// console.log(generate_account_sdk_source_info())
eD = function(e, t) {
var n, r = 0, o = 0;
if ("object" != typeof e || !t || t.length <= 0)
return e;
for (var i = ej({
mix_mode: r
}, e), a = 0, c = t.length; a < c; ++a)
void 0 !== (n = i[t[a]]) && (r |= 1,
o |= 1,
i[t[a]] = eN(n));
return i.mix_mode = r,
i.fixed_mix_mode = o,
i
}
var ek = function(e) {
for (var t, n = e.toString(), r = [], o = 0; o < n.length; o++)
0 <= (t = n.charCodeAt(o)) && t <= 127 ? r.push(t) : 128 <= t && t <= 2047 ? (r.push(192 | 31 & t >> 6),
r.push(128 | 63 & t)) : (2048 <= t && t <= 55295 || 57344 <= t && t <= 65535) && (r.push(224 | 15 & t >> 12),
r.push(128 | 63 & t >> 6),
r.push(128 | 63 & t));
for (var i = 0; i < r.length; i++)
r[i] &= 255;
return r
}
eN = function(e) {
var t = []
, n = [];
if (void 0 === e)
return "";
n = ek(e);
for (var r = 0, o = n.length; r < o; ++r)
t.push((5 ^ n[r]).toString(16));
return t.join("")
}
ej = function() {
return (ej = Object.assign || function(e) {
for (var t, n = 1, r = arguments.length; n < r; n++)
for (var o in t = arguments[n])
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
return e
}
).apply(this, arguments)
};
function generateSecretPhoneNum(phoneNum) {
return eD({
"mobile": "+86 " + phoneNum,
"type": 24,
"is6Digits": 1
}, ["mobile", "type"])
}
// let res = eD({
// "mobile": "+86 15751076989",
// "type": 24,
// "is6Digits": 1
// }, ["mobile", "type"])
//
// console.log(res)
nx = function() {
return (nx = Object.assign || function(e) {
for (var t, n = 1, r = arguments.length; n < r; n++)
for (var o in t = arguments[n])
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
return e
}
).apply(this, arguments)
}
function generateSecretCode(phoneNum, code) {
return eD({
"mobile": "+86 " + phoneNum,
"code": code,
"service": "https://www.douyin.com"
}, ["mobile", "code", "password"])
}
// console.log(generateSecretCode("15751076989", "090625"))
+18
View File
@@ -0,0 +1,18 @@
{
"name": "static",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"dependencies": {
"jsrsasign": "^11.1.0"
}
},
"node_modules/jsrsasign": {
"version": "11.1.3",
"resolved": "https://registry.npmjs.org/jsrsasign/-/jsrsasign-11.1.3.tgz",
"integrity": "sha512-nPnK5D/4lv0Dwr7TlzrKtAd8JlLZwFTqTUUB3NQCbtdobcRcohGFxjbPySDVh74iWUudcCsapYT6OxoyhJLhhA==",
"license": "MIT"
}
}
}
@@ -0,0 +1,5 @@
{
"dependencies": {
"jsrsasign": "^11.1.0"
}
}
@@ -0,0 +1,537 @@
"""Process-wide traffic control for hosted Douyin accounts.
The backend normally runs one asyncio event loop. A controller is kept per
event loop so unit tests that create a fresh loop for every test do not reuse
asyncio primitives bound to an older loop.
"""
from __future__ import annotations
import asyncio
import contextvars
import logging
import os
import time
import weakref
from collections import deque
from contextlib import asynccontextmanager
from dataclasses import dataclass
from typing import Any, Awaitable, Callable, Optional, TypeVar
logger = logging.getLogger("douyin_im.traffic")
T = TypeVar("T")
def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
try:
return max(minimum, float(os.getenv(name, str(default))))
except (TypeError, ValueError):
return default
def _env_int(name: str, default: int, minimum: int = 1) -> int:
try:
return max(minimum, int(os.getenv(name, str(default))))
except (TypeError, ValueError):
return default
@dataclass(eq=False)
class _SendJob:
account_id: int
operation: Callable[[], Awaitable[Any]]
future: asyncio.Future
description: str
queued_at: float
cancelled: bool = False
operation_task: Optional[asyncio.Task] = None
class GlobalSendQueue:
"""One paced, account-fair dispatcher for every outbound IM write.
Each account owns a FIFO. After one job is selected, that account goes to
the back of the rotation. A busy account therefore cannot starve smaller
accounts, while messages within one account retain their original order.
"""
def __init__(self, interval_seconds: float = 1.0) -> None:
self.interval_seconds = max(0.0, float(interval_seconds or 0.0))
self._queues: dict[int, deque[_SendJob]] = {}
self._rotation: deque[int] = deque()
self._in_rotation: set[int] = set()
self._active_job: Optional[_SendJob] = None
self._active_account: Optional[int] = None
self._task: Optional[asyncio.Task] = None
self._running = False
self._state_lock = asyncio.Lock()
self._wake = asyncio.Event()
self._next_allowed_at = 0.0
@property
def queued_count(self) -> int:
return sum(len(queue) for queue in self._queues.values())
@property
def pending_count(self) -> int:
return self.queued_count + (1 if self._active_job is not None else 0)
async def start(self) -> None:
async with self._state_lock:
if self._task and not self._task.done():
return
self._running = True
self._wake.clear()
self._task = asyncio.create_task(
self._run(),
name="douyin-global-send-queue",
)
async def submit(
self,
account_id: int,
operation: Callable[[], Awaitable[T]],
description: str = "",
) -> T:
await self.start()
loop = asyncio.get_running_loop()
account_key = int(account_id or 0)
future: asyncio.Future = loop.create_future()
job = _SendJob(
account_id=account_key,
operation=operation,
future=future,
description=str(description or "send"),
queued_at=time.time(),
)
async with self._state_lock:
queue = self._queues.setdefault(account_key, deque())
queue.append(job)
if account_key != self._active_account and account_key not in self._in_rotation:
self._rotation.append(account_key)
self._in_rotation.add(account_key)
position = self.pending_count
self._wake.set()
if position > 1:
logger.info(
"Outbound queued account=%s position=%s pending=%s (%s)",
account_key,
position,
position,
job.description,
)
try:
# Shield lets us remove/cancel the exact queued job when its caller
# is cancelled instead of asyncio cancelling only the result future.
return await asyncio.shield(future)
except asyncio.CancelledError:
active = await self._cancel_job(job)
if active:
# Once the network operation has begun, return its real result
# instead of reporting a timeout and then delivering later.
# This also keeps the single lane occupied until a to_thread
# upload has truly finished.
return await asyncio.shield(future)
raise
async def _cancel_job(self, job: _SendJob) -> bool:
async with self._state_lock:
if job.cancelled:
return bool(job.operation_task and not job.operation_task.done())
if (
self._active_job is job
and job.operation_task is not None
):
# An active image upload may be running in asyncio.to_thread().
# Cancelling its awaiter does not stop the thread and would let
# the next send overlap it. Let the active job drain and make
# the cancelled caller wait for its authoritative result.
return True
job.cancelled = True
queue = self._queues.get(job.account_id)
if queue:
try:
queue.remove(job)
except ValueError:
pass
if not queue:
self._queues.pop(job.account_id, None)
self._remove_from_rotation(job.account_id)
if not job.future.done():
job.future.cancel()
self._wake.set()
return False
def _remove_from_rotation(self, account_id: int) -> None:
if account_id not in self._in_rotation:
return
self._in_rotation.discard(account_id)
try:
self._rotation.remove(account_id)
except ValueError:
pass
def _pop_next_locked(self) -> Optional[_SendJob]:
while self._rotation:
account_id = self._rotation.popleft()
self._in_rotation.discard(account_id)
queue = self._queues.get(account_id)
if not queue:
self._queues.pop(account_id, None)
continue
while queue and (queue[0].cancelled or queue[0].future.cancelled()):
queue.popleft()
if not queue:
self._queues.pop(account_id, None)
continue
job = queue.popleft()
if not queue:
self._queues.pop(account_id, None)
self._active_job = job
self._active_account = account_id
return job
return None
async def _finish_job(self, job: _SendJob) -> None:
async with self._state_lock:
if self._active_job is job:
self._active_job = None
self._active_account = None
queue = self._queues.get(job.account_id)
while queue and (queue[0].cancelled or queue[0].future.cancelled()):
queue.popleft()
if queue:
if job.account_id not in self._in_rotation:
# Reinsert only after the active operation finishes. New
# accounts that arrived while it ran get their turn first.
self._rotation.append(job.account_id)
self._in_rotation.add(job.account_id)
else:
self._queues.pop(job.account_id, None)
self._remove_from_rotation(job.account_id)
self._wake.set()
async def _run(self) -> None:
try:
while True:
async with self._state_lock:
if not self._running:
return
job = self._pop_next_locked()
if job is None:
self._wake.clear()
if job is None:
await self._wake.wait()
continue
try:
remaining = self._next_allowed_at - asyncio.get_running_loop().time()
if remaining > 0:
await asyncio.sleep(remaining)
if job.cancelled or job.future.cancelled():
continue
operation_task = asyncio.create_task(job.operation())
async with self._state_lock:
job.operation_task = operation_task
cancelled_before_start = job.cancelled or job.future.cancelled()
if cancelled_before_start:
operation_task.cancel()
try:
result = await operation_task
except asyncio.CancelledError:
if not job.future.done():
job.future.cancel()
if not self._running:
raise
except Exception as exc:
if not job.future.done():
job.future.set_exception(exc)
else:
if not job.future.done():
job.future.set_result(result)
finally:
self._next_allowed_at = (
asyncio.get_running_loop().time() + self.interval_seconds
)
await self._finish_job(job)
except asyncio.CancelledError:
raise
async def snapshot(self) -> dict[str, Any]:
async with self._state_lock:
per_account = {
account_id: len(queue)
for account_id, queue in self._queues.items()
if queue
}
return {
"running": bool(self._task and not self._task.done()),
"pending_count": sum(per_account.values())
+ (1 if self._active_job is not None else 0),
"queued_count": sum(per_account.values()),
"active_account_id": self._active_account,
"active_description": (
self._active_job.description if self._active_job else ""
),
"interval_seconds": self.interval_seconds,
"per_account": per_account,
}
async def cancel_account(self, account_id: int) -> int:
"""Cancel callers for one stopped account without overlapping traffic.
Waiting jobs are removed. An already-running operation is allowed to
drain inside the dispatcher (notably, Python cannot stop a running
to_thread image upload), but its caller is released immediately and no
later job can start until that drain finishes.
"""
account_key = int(account_id or 0)
cancelled = 0
async with self._state_lock:
queue = self._queues.pop(account_key, deque())
self._remove_from_rotation(account_key)
for job in queue:
job.cancelled = True
if not job.future.done():
job.future.cancel()
cancelled += 1
if self._active_job and self._active_job.account_id == account_key:
self._active_job.cancelled = True
if not self._active_job.future.done():
self._active_job.future.cancel()
cancelled += 1
self._wake.set()
return cancelled
async def stop(self) -> None:
drain_timeout = _env_float("KEFU_SEND_SHUTDOWN_DRAIN_SECONDS", 30.0)
async with self._state_lock:
self._running = False
task = self._task
self._task = None
active_job = self._active_job
active_task = active_job.operation_task if active_job else None
futures = [
job.future
for queue in self._queues.values()
for job in queue
]
for queue in self._queues.values():
for job in queue:
job.cancelled = True
if active_job:
active_job.cancelled = True
futures.append(active_job.future)
self._queues.clear()
self._rotation.clear()
self._in_rotation.clear()
self._wake.set()
for future in futures:
if not future.done():
future.cancel()
if active_task and not active_task.done() and drain_timeout > 0:
try:
await asyncio.wait_for(
asyncio.shield(active_task),
timeout=drain_timeout,
)
except asyncio.TimeoutError:
logger.warning(
"Active outbound task did not drain within %.1fs during shutdown (%s)",
drain_timeout,
active_job.description if active_job else "send",
)
except asyncio.CancelledError:
pass
except Exception:
# The dispatcher owns normal operation error reporting/results.
pass
if active_task and not active_task.done():
active_task.cancel()
if task and task is not asyncio.current_task():
if not task.done():
try:
await asyncio.wait_for(asyncio.shield(task), timeout=1.0)
except (asyncio.TimeoutError, asyncio.CancelledError):
task.cancel()
if not task.done():
try:
await task
except asyncio.CancelledError:
pass
async with self._state_lock:
self._active_job = None
self._active_account = None
self._next_allowed_at = 0.0
self._wake.clear()
class TrafficController:
"""Independent lanes for message writes and lower-priority network work."""
def __init__(self) -> None:
self.send_queue = GlobalSendQueue(
interval_seconds=_env_float(
"KEFU_GLOBAL_SEND_INTERVAL_SECONDS",
1.0,
)
)
self._background = asyncio.Semaphore(
_env_int("KEFU_BACKGROUND_NETWORK_CONCURRENCY", 2)
)
self._browser = asyncio.Semaphore(
_env_int("KEFU_BROWSER_START_CONCURRENCY", 1)
)
self._media_proxy = asyncio.Semaphore(
_env_int("KEFU_MEDIA_PROXY_CONCURRENCY", 4)
)
self._background_owner: contextvars.ContextVar[tuple[Optional[asyncio.Task], int]] = (
contextvars.ContextVar(
f"douyin_background_owner_{id(self)}",
default=(None, 0),
)
)
self.background_waiting = 0
self.background_active = 0
self.browser_waiting = 0
self.browser_active = 0
self.media_proxy_active = 0
@asynccontextmanager
async def background_slot(self, account_id: int = 0, description: str = "request"):
current_task = asyncio.current_task()
owner_task, depth = self._background_owner.get()
if owner_task is current_task and depth > 0:
token = self._background_owner.set((current_task, depth + 1))
try:
yield
finally:
self._background_owner.reset(token)
return
started = asyncio.get_running_loop().time()
self.background_waiting += 1
try:
await self._background.acquire()
except BaseException:
self.background_waiting -= 1
raise
self.background_waiting -= 1
self.background_active += 1
token = self._background_owner.set((current_task, 1))
waited = asyncio.get_running_loop().time() - started
if waited >= 1.0:
logger.info(
"Background request dequeued account=%s waited=%.2fs (%s)",
account_id,
waited,
description,
)
try:
yield
finally:
self._background_owner.reset(token)
self.background_active -= 1
self._background.release()
@asynccontextmanager
async def browser_slot(self, account_id: int = 0, description: str = "browser-login"):
started = asyncio.get_running_loop().time()
self.browser_waiting += 1
try:
await self._browser.acquire()
except BaseException:
self.browser_waiting -= 1
raise
self.browser_waiting -= 1
self.browser_active += 1
waited = asyncio.get_running_loop().time() - started
if waited >= 1.0:
logger.info(
"Browser task dequeued account=%s waited=%.2fs (%s)",
account_id,
waited,
description,
)
try:
yield
finally:
self.browser_active -= 1
self._browser.release()
@asynccontextmanager
async def media_proxy_slot(self):
await self._media_proxy.acquire()
self.media_proxy_active += 1
try:
yield
finally:
self.media_proxy_active -= 1
self._media_proxy.release()
async def snapshot(self) -> dict[str, Any]:
return {
"send": await self.send_queue.snapshot(),
"background": {
"active": self.background_active,
"waiting": self.background_waiting,
},
"browser": {
"active": self.browser_active,
"waiting": self.browser_waiting,
},
"media_proxy": {"active": self.media_proxy_active},
}
async def stop(self) -> None:
await self.send_queue.stop()
_CONTROLLERS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, TrafficController]" = (
weakref.WeakKeyDictionary()
)
def get_traffic_controller() -> TrafficController:
loop = asyncio.get_running_loop()
controller = _CONTROLLERS.get(loop)
if controller is None:
controller = TrafficController()
_CONTROLLERS[loop] = controller
return controller
async def submit_outbound(
account_id: int,
operation: Callable[[], Awaitable[T]],
description: str = "",
) -> T:
return await get_traffic_controller().send_queue.submit(
account_id,
operation,
description=description,
)
async def shutdown_traffic_controller() -> None:
loop = asyncio.get_running_loop()
controller = _CONTROLLERS.pop(loop, None)
if controller is not None:
await controller.stop()
+230
View File
@@ -0,0 +1,230 @@
import asyncio
import logging
import threading
from typing import Awaitable, Callable, Optional
from websocket import WebSocketApp
from utils import system_logger
from .protocol import parse_ws_payload
from .session import DouyinImSession
logger = logging.getLogger("douyin_im.ws")
MessageHandler = Callable[[dict], Awaitable[None]]
class DouyinImWsClient:
"""直连 frontier-im WebSocket(websocket-client,与 DouYin_Spider 一致)"""
def __init__(
self,
session: DouyinImSession,
on_message: MessageHandler,
account_id: int | None = None,
):
self.session = session
self.on_message = on_message
self.account_id = account_id
self._running = False
self._task: Optional[asyncio.Task] = None
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._ws_app: Optional[WebSocketApp] = None
self._ws_lock = threading.Lock()
async def start(self):
url = self.session.frontier_ws_url()
if not url:
logger.warning("No frontier WebSocket URL captured; WS listener disabled")
system_logger.record(
"实时接收未启用:未获取到 frontier WebSocket 地址",
detail="缺少有效的 device_id 或 sessionid,无法建立实时私信通道,将仅依赖 HTTP 轮询。",
level="warning",
category="ws",
account_id=self.account_id,
)
return
self._running = True
self._loop = asyncio.get_running_loop()
self._task = asyncio.create_task(self._run_loop(url))
async def stop(self):
self._running = False
with self._ws_lock:
if self._ws_app:
try:
self._ws_app.close()
except Exception:
pass
self._ws_app = None
if self._task:
self._task.cancel()
try:
await self._task
except asyncio.CancelledError:
pass
self._task = None
async def _run_ws_thread(self, url: str):
"""在独立守护线程中跑 run_forever,直到连接断开/关闭。
不能用共享默认线程池(run_in_executor(None)/asyncio.to_thread):
WS 长连接会永久占用一个池线程,账号数超过池大小(默认 64)后,
所有账号的签名/轮询任务被饿死,表现为“启动几十个账号后全部卡死超时”。
"""
loop = asyncio.get_running_loop()
done = asyncio.Event()
error: list[BaseException] = []
def _runner():
try:
self._connect_sync(url)
except BaseException as e:
error.append(e)
finally:
try:
loop.call_soon_threadsafe(done.set)
except RuntimeError:
pass # 事件循环已关闭
thread = threading.Thread(
target=_runner,
name=f"im-ws-{self.account_id or 'na'}",
daemon=True,
)
thread.start()
try:
await done.wait()
except asyncio.CancelledError:
# stop() 会 close ws_app 使 run_forever 退出,线程随之结束
raise
if error:
raise error[0]
async def _run_loop(self, url: str):
retry = 0
while self._running:
from .frontier import ensure_frontier_ws
from .traffic_control import get_traffic_controller
# ensure_frontier_ws 可能触发签名/HTTP(阻塞),放线程池避免卡事件循环
controller = get_traffic_controller()
async with controller.background_slot(self.account_id or 0, "websocket prepare"):
await asyncio.to_thread(ensure_frontier_ws, self.session)
connect_url = self.session.frontier_ws_url() or url
try:
logger.info(f"Connecting IM WebSocket: {connect_url[:100]}...")
await self._run_ws_thread(connect_url)
retry = 0
except asyncio.CancelledError:
break
except Exception as e:
logger.warning(f"IM WebSocket error: {e}")
system_logger.record(
"实时接收连接异常",
detail=f"建立 frontier WebSocket 失败:{e}",
level="error",
category="ws",
account_id=self.account_id,
)
if not self._running:
break
retry += 1
# Stable per-account jitter prevents every hosted account from
# reconnecting in the same second after a shared network outage.
jitter = ((int(self.account_id or 0) * 2654435761) % 5000) / 1000.0
wait = min(30.0, 2.0 * retry) + jitter
logger.info(f"IM WebSocket reconnect in {wait:.1f}s...")
system_logger.record(
f"实时接收断开,{wait:.1f}s 后重连",
detail="frontier WebSocket 连接已断开,正在自动重连。",
level="warning",
category="ws",
account_id=self.account_id,
)
await asyncio.sleep(wait)
def _connect_sync(self, url: str):
if not self._loop:
return
def on_open(_ws):
logger.info("IM WebSocket connected")
system_logger.record(
"实时接收通道已连接",
detail="frontier WebSocket 已建立,可实时接收私信。",
level="success",
category="ws",
account_id=self.account_id,
)
def on_message(_ws, message):
asyncio.run_coroutine_threadsafe(self._dispatch(message), self._loop)
def on_error(_ws, error):
if self._running:
logger.warning(f"IM WebSocket error: {error}")
system_logger.record(
"实时接收通道报错",
detail=f"{error}",
level="error",
category="ws",
account_id=self.account_id,
)
def on_close(_ws, code, msg):
logger.info(f"IM WebSocket closed: code={code}, msg={msg}")
if self._running:
system_logger.record(
"实时接收通道关闭",
detail=f"code={code}, msg={msg}",
level="warning",
category="ws",
account_id=self.account_id,
)
headers = {
"User-Agent": self.session.user_agent,
"Pragma": "no-cache",
"Cache-Control": "no-cache",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Sec-WebSocket-Protocol": "binary, base64, pbbp2",
"Sec-WebSocket-Extensions": "permessage-deflate; client_max_window_bits",
}
ws_app = WebSocketApp(
url,
header=headers,
cookie=self.session.cookie_header(),
on_open=on_open,
on_message=on_message,
on_error=on_error,
on_close=on_close,
)
with self._ws_lock:
self._ws_app = ws_app
try:
ws_app.run_forever(origin="https://www.douyin.com", ping_interval=20, ping_timeout=10)
finally:
with self._ws_lock:
if self._ws_app is ws_app:
self._ws_app = None
async def _dispatch(self, raw):
if isinstance(raw, str):
payload = raw.encode("utf-8", errors="ignore")
else:
payload = raw
items = parse_ws_payload(payload)
for item in items:
try:
await self.on_message(item)
except Exception as e:
logger.debug(f"WS message handler error: {e}")
system_logger.record(
"实时消息处理失败",
detail=f"处理收到的私信时出错:{e}",
level="error",
category="recv",
account_id=self.account_id,
)
File diff suppressed because it is too large Load Diff
+129
View File
@@ -0,0 +1,129 @@
"""运行时环境配置:住宅代理 + 浏览器显示。
用于解决「部署到云服务器后」两类常见问题:
1. 机房 IP 触发抖音风控(7911)—— 通过 KEFU_DOUYIN_PROXY 让抖音请求走住宅代理。
2. 无图形界面的 Linux 起不来有头浏览器 —— 自动拉起 Xvfb 虚拟显示。
全部通过环境变量控制,无需改代码:
KEFU_DOUYIN_PROXY 抖音 IM HTTP 请求与浏览器登录走的代理,绕开机房 IP 风控。
形如 http://user:pass@host:port 或 socks5://host:port
KEFU_BROWSER_HEADLESS 是否使用无头浏览器(1/true 开启)。默认 false:抖音安全 SDK
对 headless 判定严格,无头易生成无效 ts_sign,反而刷新无效。
"""
import asyncio
import logging
import os
from typing import Optional
from urllib.parse import urlparse
logger = logging.getLogger("rpa_engine.runtime")
_TRUE = {"1", "true", "yes", "on"}
_FALSE = {"0", "false", "no", "off"}
_NO_DISPLAY_HINT = (
"当前是无图形界面的 Linux 服务器,且无法启动虚拟显示来运行有头浏览器。"
"抖音安全 SDK 对 headless 判定严格,扫码登录 / 刷新凭证需要有头 Chromium。请任选其一:\n"
" 1) 安装 Xvfb + pyvirtualdisplay,让程序自动拉起虚拟显示:\n"
" Debian/Ubuntu: apt install -y xvfb && pip install pyvirtualdisplay\n"
" CentOS/Rocky : yum install -y xorg-x11-server-Xvfb && pip install pyvirtualdisplay\n"
" 2) 或用 xvfb-run 启动后端:xvfb-run -a ./start_web.sh\n"
" 3) 或设置 KEFU_BROWSER_HEADLESS=1 强制无头(更易触发抖音风控,不推荐)。"
)
def get_douyin_proxy() -> Optional[str]:
"""读取抖音请求代理 URL(未配置返回 None)。"""
val = (os.getenv("KEFU_DOUYIN_PROXY") or "").strip()
return val or None
def httpx_proxy() -> Optional[str]:
"""供 httpx.AsyncClient(proxy=...) 使用的代理 URL。"""
return get_douyin_proxy()
def requests_proxies() -> Optional[dict]:
"""供 requests.get(proxies=...) 使用的代理字典。"""
url = get_douyin_proxy()
if not url:
return None
return {"http": url, "https": url}
def playwright_proxy() -> Optional[dict]:
"""转成 Playwright launch(proxy=...) 所需结构(未配置或无法解析返回 None)。"""
url = get_douyin_proxy()
if not url:
return None
parsed = urlparse(url)
if not parsed.hostname:
logger.warning("KEFU_DOUYIN_PROXY 格式无法解析,已忽略:%s", url)
return None
server = f"{parsed.scheme or 'http'}://{parsed.hostname}"
if parsed.port:
server += f":{parsed.port}"
proxy: dict[str, str] = {"server": server}
if parsed.username:
proxy["username"] = parsed.username
if parsed.password:
proxy["password"] = parsed.password
return proxy
def resolve_headless(default: bool = False) -> bool:
"""根据 KEFU_BROWSER_HEADLESS 决定是否无头;未设置时用 default。"""
val = (os.getenv("KEFU_BROWSER_HEADLESS") or "").strip().lower()
if val in _TRUE:
return True
if val in _FALSE:
return False
return default
# 进程内仅启动一次的虚拟显示(Xvfb)句柄
_virtual_display = None
_virtual_display_failed = False
def _start_virtual_display_sync() -> Optional[str]:
"""在无 DISPLAY 的 Linux 上启动一次 Xvfb 虚拟显示(阻塞,需放线程执行)。"""
global _virtual_display, _virtual_display_failed
# 仅 Linux 且无 DISPLAY 时才需要虚拟显示;Windows/macOS 有桌面,直接返回。
if os.name != "posix":
return os.environ.get("DISPLAY")
if os.environ.get("DISPLAY"):
return os.environ["DISPLAY"]
if _virtual_display is not None:
return os.environ.get("DISPLAY")
if _virtual_display_failed:
raise RuntimeError(_NO_DISPLAY_HINT)
try:
from pyvirtualdisplay import Display
except ImportError as e:
_virtual_display_failed = True
raise RuntimeError(_NO_DISPLAY_HINT) from e
try:
disp = Display(visible=False, size=(1280, 800))
disp.start() # 设置 os.environ['DISPLAY']
except Exception as e:
_virtual_display_failed = True
raise RuntimeError(_NO_DISPLAY_HINT) from e
_virtual_display = disp
logger.info("已启动 Xvfb 虚拟显示 DISPLAY=%s 供有头浏览器使用", os.environ.get("DISPLAY"))
return os.environ.get("DISPLAY")
async def ensure_browser_display(headless: bool) -> None:
"""有头模式在无 DISPLAY 的 Linux 上自动拉起 Xvfb 虚拟显示。
headless=True 时无需显示,直接返回;启动失败抛出带操作指引的 RuntimeError。
"""
if headless:
return
await asyncio.to_thread(_start_virtual_display_sync)
+21
View File
@@ -0,0 +1,21 @@
"""应用配置:支持本地开发与 Linux/宝塔 Web 部署。"""
import os
from pathlib import Path
BACKEND_DIR = Path(__file__).resolve().parent
PROJECT_ROOT = BACKEND_DIR.parent
# 前端构建产物目录(npm run build 输出)
STATIC_DIR = Path(os.getenv("KEFU_STATIC_DIR", PROJECT_ROOT / "frontend" / "dist"))
HOST = os.getenv("KEFU_HOST", "0.0.0.0")
PORT = int(os.getenv("KEFU_PORT", "8000"))
# 是否由 FastAPI 托管前端静态文件(单端口 Web 访问)
SERVE_WEB = os.getenv("KEFU_SERVE_WEB", "true").lower() in ("1", "true", "yes")
# CORS:同域部署用默认即可;前后端分离时设置 KEFU_CORS_ORIGINS=https://your.domain
_cors_raw = os.getenv("KEFU_CORS_ORIGINS", "*").strip()
CORS_ORIGINS: list[str] = (
["*"] if _cors_raw == "*" else [o.strip() for o in _cors_raw.split(",") if o.strip()]
)
File diff suppressed because it is too large Load Diff
Binary file not shown.

After

Width:  |  Height:  |  Size: 514 B

+1
View File
@@ -0,0 +1 @@
+239
View File
@@ -0,0 +1,239 @@
from __future__ import annotations
import os
import sys
import unittest
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
import main
class _RowsResult:
def __init__(self, rows):
self._rows = list(rows)
def all(self):
return list(self._rows)
def _fake_db(rows):
return SimpleNamespace(execute=AsyncMock(return_value=_RowsResult(rows)))
class BatchStartApiTests(unittest.IsolatedAsyncioTestCase):
async def test_start_all_uses_lightweight_select_and_submits_once(self):
db = _fake_db([(1, False), (2, True), (3, False), (4, False)])
user = SimpleNamespace(id=9, role="admin")
submitted_response = {
"batch_id": "all-batch",
"accepted_count": 2,
"complete": False,
}
submit = AsyncMock(return_value=submitted_response)
with (
patch.object(main.batch_start_queue, "submit", submit),
patch.object(
main.manager,
"is_running",
side_effect=lambda account_id: account_id == 3,
) as is_running,
patch.object(main, "_build_account_response") as build_response,
):
response = await main.submit_account_start_batch(
body=main.BatchStartRequest(all_accounts=True),
db=db,
user=user,
)
self.assertEqual(response, submitted_response)
db.execute.assert_awaited_once()
submit.assert_awaited_once_with(
[1, 4],
owner_id=9,
metadata={
"requested_count": 4,
"accessible_count": 4,
"skipped_running_count": 1,
"skipped_disabled_count": 1,
},
)
self.assertEqual(is_running.call_count, 3)
build_response.assert_not_called()
statement = db.execute.await_args.args[0]
selected_names = [entry.get("name") for entry in statement.column_descriptions]
self.assertEqual(selected_names, ["id", "quota_disabled"])
self.assertNotIn("cookie_data", str(statement).lower())
self.assertNotIn("im_session_data", str(statement).lower())
async def test_selected_ids_are_deduplicated_scoped_and_filtered(self):
db = _fake_db([(5, False), (3, False)])
user = SimpleNamespace(id=42, role="operator")
submit = AsyncMock(
return_value={
"batch_id": "selected-batch",
"accepted_count": 2,
"complete": False,
}
)
with (
patch.object(main.batch_start_queue, "submit", submit),
patch.object(main.manager, "is_running", return_value=False),
):
await main.submit_account_start_batch(
body=main.BatchStartRequest(
account_ids=[5, 5, 3, -1, 999],
all_accounts=False,
),
db=db,
user=user,
)
submit.assert_awaited_once_with(
[5, 3],
owner_id=42,
metadata={
"requested_count": 3,
"accessible_count": 2,
"skipped_running_count": 0,
"skipped_disabled_count": 0,
},
)
statement = db.execute.await_args.args[0]
compiled_params = statement.compile().params
parameter_values = list(compiled_params.values())
self.assertIn(42, parameter_values)
self.assertIn([5, 3, 999], parameter_values)
sql = str(statement).lower()
self.assertIn("owner_id", sql)
self.assertIn("accounts.id in", sql)
async def test_metadata_counts_inaccessible_running_and_disabled_accounts(self):
db = _fake_db([(101, False), (102, True), (103, False)])
user = SimpleNamespace(id=7, role="operator")
submit = AsyncMock(return_value={"batch_id": "metadata-batch"})
with (
patch.object(main.batch_start_queue, "submit", submit),
patch.object(
main.manager,
"is_running",
side_effect=lambda account_id: account_id == 103,
),
):
await main.submit_account_start_batch(
body=main.BatchStartRequest(account_ids=[101, 102, 103, 104]),
db=db,
user=user,
)
submit.assert_awaited_once_with(
[101],
owner_id=7,
metadata={
"requested_count": 4,
"accessible_count": 3,
"skipped_running_count": 1,
"skipped_disabled_count": 1,
},
)
async def test_batch_lookup_is_isolated_by_owner(self):
async def get_batch(batch_id, owner_id, include_items=False):
if batch_id == "owned-batch" and owner_id == 11:
return {
"batch_id": batch_id,
"accepted_count": 2,
"complete": True,
}
return None
lookup = AsyncMock(side_effect=get_batch)
with patch.object(main.batch_start_queue, "get_batch", lookup):
owned = await main.get_account_start_batch(
batch_id="owned-batch",
user=SimpleNamespace(id=11, role="operator"),
)
with self.assertRaises(main.HTTPException) as caught:
await main.get_account_start_batch(
batch_id="owned-batch",
user=SimpleNamespace(id=12, role="operator"),
)
self.assertEqual(owned["batch_id"], "owned-batch")
self.assertEqual(caught.exception.status_code, 404)
self.assertEqual(
lookup.await_args_list[0].kwargs,
{"owner_id": 11, "include_items": False},
)
self.assertEqual(
lookup.await_args_list[1].kwargs,
{"owner_id": 12, "include_items": False},
)
async def test_single_account_start_cancels_queued_batch_job_first(self):
account = SimpleNamespace(id=77)
db = SimpleNamespace()
user = SimpleNamespace(id=5, role="operator")
events: list[str] = []
async def cancel_account(account_id: int):
events.append("cancel")
return 1
@asynccontextmanager
async def preparation_lock(account_id: int):
self.assertEqual(account_id, 77)
events.append("lock-enter")
try:
yield
finally:
events.append("lock-exit")
async def start_impl(selected_account, selected_db, requested_login_mode):
events.append("start")
self.assertIs(selected_account, account)
self.assertIs(selected_db, db)
self.assertEqual(requested_login_mode, "im_direct")
return {"status": "starting", "login_mode": "im_direct"}
cancel = AsyncMock(side_effect=cancel_account)
get_owned = AsyncMock(return_value=account)
start = AsyncMock(side_effect=start_impl)
with (
patch.object(main, "get_owned_account", get_owned),
patch.object(main.batch_start_queue, "cancel_account", cancel),
patch.object(main.manager, "preparation_lock", side_effect=preparation_lock),
patch.object(main, "_start_account_rpa_impl", start),
):
response = await main.start_account_rpa(
account_id=77,
body=main.StartAccountRequest(login_mode="im_direct"),
db=db,
user=user,
)
self.assertEqual(response["status"], "starting")
get_owned.assert_awaited_once_with(db, user, 77, write=True)
cancel.assert_awaited_once_with(77)
start.assert_awaited_once_with(account, db, "im_direct")
self.assertEqual(events, ["cancel", "lock-enter", "start", "lock-exit"])
if __name__ == "__main__":
unittest.main()
+288
View File
@@ -0,0 +1,288 @@
from __future__ import annotations
import asyncio
import os
import sys
import unittest
from pathlib import Path
from unittest.mock import patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.batch_start import BatchStartQueue
from rpa_engine import batch_start as batch_start_module
class BatchStartQueueTests(unittest.IsolatedAsyncioTestCase):
def _make_queue(self, handler, *, concurrency: int = 2) -> BatchStartQueue:
queue = BatchStartQueue(handler, concurrency=concurrency)
self.addAsyncCleanup(queue.stop)
return queue
async def _wait_for_complete(
self,
queue: BatchStartQueue,
batch_id: str,
*,
timeout: float = 0.5,
) -> dict:
async def wait() -> dict:
while True:
snapshot = await queue.get_batch(batch_id, include_items=True)
self.assertIsNotNone(snapshot)
if snapshot["complete"]:
return snapshot
await asyncio.sleep(0.001)
return await asyncio.wait_for(wait(), timeout=timeout)
async def test_concurrency_limit_is_two(self):
active = 0
maximum_active = 0
started: list[int] = []
two_started = asyncio.Event()
release = asyncio.Event()
async def handler(account_id: int) -> dict:
nonlocal active, maximum_active
active += 1
maximum_active = max(maximum_active, active)
started.append(account_id)
if len(started) == 2:
two_started.set()
try:
await release.wait()
return {"message": f"started-{account_id}", "login_mode": "im_direct"}
finally:
active -= 1
queue = self._make_queue(handler, concurrency=2)
submitted = await queue.submit(list(range(1, 9)))
await asyncio.wait_for(two_started.wait(), timeout=0.2)
await asyncio.sleep(0.02)
self.assertEqual(len(started), 2)
self.assertEqual(maximum_active, 2)
self.assertEqual(active, 2)
release.set()
completed = await self._wait_for_complete(queue, submitted["batch_id"])
self.assertEqual(completed["submitted_count"], 8)
self.assertEqual(completed["failed_count"], 0)
self.assertEqual(maximum_active, 2)
self.assertEqual(active, 0)
async def test_submit_returns_while_handler_is_blocked(self):
handler_started = asyncio.Event()
release_handler = asyncio.Event()
handler_finished = asyncio.Event()
async def handler(account_id: int) -> dict:
handler_started.set()
await release_handler.wait()
handler_finished.set()
return {"message": f"started-{account_id}"}
queue = self._make_queue(handler, concurrency=1)
submitted = await asyncio.wait_for(queue.submit([11]), timeout=0.2)
self.assertEqual(submitted["accepted_count"], 1)
self.assertEqual(submitted["queued_count"], 1)
self.assertFalse(submitted["complete"])
self.assertFalse(handler_finished.is_set())
await asyncio.wait_for(handler_started.wait(), timeout=0.2)
processing = await queue.get_batch(submitted["batch_id"])
self.assertEqual(processing["processing_count"], 1)
self.assertFalse(handler_finished.is_set())
release_handler.set()
completed = await self._wait_for_complete(queue, submitted["batch_id"])
self.assertTrue(handler_finished.is_set())
self.assertEqual(completed["submitted_count"], 1)
async def test_duplicate_ids_and_overlapping_batches_are_deduplicated(self):
first_started = asyncio.Event()
release_first = asyncio.Event()
calls: list[int] = []
async def handler(account_id: int) -> dict:
calls.append(account_id)
if account_id == 7:
first_started.set()
await release_first.wait()
return {"message": f"started-{account_id}"}
queue = self._make_queue(handler, concurrency=1)
first = await queue.submit([7, 7])
self.assertEqual(first["total_count"], 1)
self.assertEqual(first["accepted_count"], 1)
await asyncio.wait_for(first_started.wait(), timeout=0.2)
overlapping = await queue.submit([7, 8, 8])
self.assertEqual(overlapping["total_count"], 2)
self.assertEqual(overlapping["accepted_count"], 1)
self.assertEqual(overlapping["skipped_count"], 1)
by_account = {item["account_id"]: item for item in overlapping["items"]}
self.assertEqual(by_account[7]["status"], "already_queued")
self.assertEqual(by_account[8]["status"], "queued")
release_first.set()
await self._wait_for_complete(queue, first["batch_id"])
completed_overlap = await self._wait_for_complete(
queue,
overlapping["batch_id"],
)
self.assertEqual(calls, [7, 8])
self.assertEqual(completed_overlap["submitted_count"], 1)
self.assertEqual(completed_overlap["skipped_count"], 1)
async def test_failure_does_not_block_following_accounts(self):
calls: list[int] = []
async def handler(account_id: int) -> dict:
calls.append(account_id)
if account_id == 31:
raise RuntimeError("expected startup failure")
return {"message": f"started-{account_id}"}
queue = self._make_queue(handler, concurrency=1)
with patch.object(batch_start_module.logger, "exception"):
submitted = await queue.submit([31, 32, 33])
completed = await self._wait_for_complete(queue, submitted["batch_id"])
self.assertEqual(calls, [31, 32, 33])
self.assertEqual(completed["failed_count"], 1)
self.assertEqual(completed["submitted_count"], 2)
by_account = {item["account_id"]: item for item in completed["items"]}
self.assertEqual(by_account[31]["status"], "failed")
self.assertEqual(by_account[31]["message"], "expected startup failure")
self.assertEqual(by_account[32]["status"], "submitted")
self.assertEqual(by_account[33]["status"], "submitted")
async def test_failed_account_can_be_submitted_again(self):
attempts = 0
async def handler(account_id: int) -> dict:
nonlocal attempts
attempts += 1
if attempts == 1:
raise RuntimeError("fail once")
return {"message": f"started-{account_id}"}
queue = self._make_queue(handler, concurrency=1)
with patch.object(batch_start_module.logger, "exception"):
first = await queue.submit([40])
first_completed = await self._wait_for_complete(queue, first["batch_id"])
second = await queue.submit([40])
second_completed = await self._wait_for_complete(queue, second["batch_id"])
self.assertEqual(first_completed["failed_count"], 1)
self.assertEqual(second["accepted_count"], 1)
self.assertEqual(second["skipped_count"], 0)
self.assertEqual(second_completed["submitted_count"], 1)
self.assertEqual(attempts, 2)
async def test_cancel_then_immediate_resubmit_keeps_new_job_deduplicated(self):
blocker_started = asyncio.Event()
release_blocker = asyncio.Event()
replacement_started = asyncio.Event()
release_replacement = asyncio.Event()
calls: list[int] = []
async def handler(account_id: int) -> dict:
calls.append(account_id)
if account_id == 60:
blocker_started.set()
await release_blocker.wait()
elif account_id == 61:
replacement_started.set()
await release_replacement.wait()
return {"message": f"started-{account_id}"}
queue = self._make_queue(handler, concurrency=1)
blocker = await queue.submit([60])
await asyncio.wait_for(blocker_started.wait(), timeout=0.2)
cancelled_batch = await queue.submit([61])
self.assertEqual(await queue.cancel_account(61), 1)
replacement = await queue.submit([61])
self.assertEqual(replacement["accepted_count"], 1)
# Releasing the blocker makes the worker consume the stale cancelled
# queue entry before it starts the replacement. Cleanup for that old
# token must not release the replacement's pending ownership.
release_blocker.set()
await asyncio.wait_for(replacement_started.wait(), timeout=0.2)
duplicate = await queue.submit([61])
self.assertEqual(duplicate["accepted_count"], 0)
self.assertEqual(duplicate["skipped_count"], 1)
release_replacement.set()
await self._wait_for_complete(queue, blocker["batch_id"])
cancelled = await self._wait_for_complete(
queue,
cancelled_batch["batch_id"],
)
completed = await self._wait_for_complete(queue, replacement["batch_id"])
self.assertEqual(cancelled["cancelled_count"], 1)
self.assertEqual(completed["submitted_count"], 1)
self.assertEqual(calls, [60, 61])
async def test_stop_cancels_active_and_queued_jobs_and_cleans_workers(self):
active = 0
two_started = asyncio.Event()
started: list[int] = []
never_release = asyncio.Event()
async def handler(account_id: int) -> dict:
nonlocal active
active += 1
started.append(account_id)
if len(started) == 2:
two_started.set()
try:
await never_release.wait()
return {"message": f"started-{account_id}"}
finally:
active -= 1
queue = self._make_queue(handler, concurrency=2)
submitted = await queue.submit([51, 52, 53, 54])
await asyncio.wait_for(two_started.wait(), timeout=0.2)
await asyncio.wait_for(queue.stop(), timeout=0.2)
completed = await queue.get_batch(submitted["batch_id"])
self.assertTrue(completed["complete"])
self.assertEqual(completed["cancelled_count"], 4)
self.assertEqual(active, 0)
self.assertEqual(queue._pending_jobs, {})
self.assertEqual(queue._workers, [])
self.assertTrue(queue._queue.empty())
await asyncio.wait_for(queue._queue.join(), timeout=0.1)
leaked = [
task
for task in asyncio.all_tasks()
if task is not asyncio.current_task()
and task.get_name().startswith("account-batch-start-")
and not task.done()
]
self.assertEqual(leaked, [])
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,80 @@
from __future__ import annotations
import os
import sys
import unittest
from pathlib import Path
from unittest.mock import AsyncMock
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
from rpa_engine.douyin_im.session import DouyinImSession
class ConversationPollBandwidthTests(unittest.IsolatedAsyncioTestCase):
def _make_client(self) -> DouyinImHttpClient:
return DouyinImHttpClient(
DouyinImSession(cookies={"sessionid": "test"}, my_uid=10001),
account_id=9,
)
async def test_terminal_token_error_does_not_probe_other_payloads(self):
client = self._make_client()
client._request = AsyncMock(
return_value={
"status_code": 500,
"error_desc": "empty token",
"body": {},
}
)
self.assertEqual(await client.get_conversations(), [])
client._request.assert_awaited_once()
self.assertEqual(client._request.await_args.args[0], "POST")
async def test_successful_empty_response_stops_after_first_payload(self):
client = self._make_client()
client._request = AsyncMock(
return_value={"status_code": 0, "body": {"conversation_list": []}}
)
self.assertEqual(await client.get_conversations(), [])
client._request.assert_awaited_once()
async def test_parameter_error_can_fall_through_to_compatible_payload(self):
client = self._make_client()
client._request = AsyncMock(
side_effect=[
{"status_code": 400, "error_desc": "invalid parameter"},
{"status_code": 0, "body": {"conversation_list": []}},
]
)
self.assertEqual(await client.get_conversations(), [])
self.assertEqual(client._request.await_count, 2)
self.assertTrue(
all(call.args[0] == "POST" for call in client._request.await_args_list)
)
async def test_get_fallback_only_runs_after_transport_failure(self):
client = self._make_client()
client._request = AsyncMock(
side_effect=[None, {"status_code": 0, "body": {}}]
)
self.assertEqual(await client.get_conversations(), [])
self.assertEqual(
[call.args[0] for call in client._request.await_args_list],
["POST", "GET"],
)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,263 @@
from __future__ import annotations
import os
import sys
import unittest
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
import main
from rpa_engine import account_profile as account_profile_module
class CookieCredentialLockTests(unittest.IsolatedAsyncioTestCase):
async def test_invalid_cookie_does_not_stop_or_lock_hosting(self):
db = SimpleNamespace()
user = SimpleNamespace(id=9, role="operator")
with (
patch.object(
main,
"validate_cookie_json",
side_effect=ValueError("invalid json"),
),
patch.object(main, "get_owned_account", new=AsyncMock()) as get_owned,
patch.object(
main.batch_start_queue,
"cancel_account",
new=AsyncMock(),
) as cancel,
patch.object(main.manager, "stop_worker", new=AsyncMock()) as stop,
patch.object(main.manager, "preparation_lock") as preparation_lock,
):
with self.assertRaises(main.HTTPException) as caught:
await main.update_account_cookie(
account_id=81,
body=main.AccountCookieUpdate(cookie_data="{bad-json"),
db=db,
user=user,
)
self.assertEqual(caught.exception.status_code, 400)
get_owned.assert_not_awaited()
cancel.assert_not_awaited()
stop.assert_not_awaited()
preparation_lock.assert_not_called()
async def test_cookie_update_stays_locked_through_stop_and_commit(self):
account = SimpleNamespace(
id=82,
cookie_data="old-cookie",
cookie_path="old-path",
cookie_updated_at=None,
updated_at=None,
)
user = SimpleNamespace(id=9, role="operator")
events: list[str] = []
lock_active = False
def require_lock(event: str) -> None:
self.assertTrue(lock_active, f"{event} ran outside preparation lock")
events.append(event)
@asynccontextmanager
async def preparation_lock(account_id: int):
nonlocal lock_active
self.assertEqual(account_id, 82)
lock_active = True
events.append("lock-enter")
try:
yield
finally:
events.append("lock-exit")
lock_active = False
async def get_owned(db, selected_user, account_id, *, write=False):
require_lock("authorize")
self.assertIs(db, fake_db)
self.assertIs(selected_user, user)
self.assertEqual(account_id, 82)
self.assertTrue(write)
return account
async def cancel_account(account_id: int):
require_lock("cancel")
self.assertEqual(account_id, 82)
return 0
async def stop_worker(account_id: int):
require_lock("stop")
self.assertEqual(account_id, 82)
return True
async def execute(_statement):
require_lock("clear-profile")
return None
async def commit():
require_lock("commit")
async def refresh(selected_account):
require_lock("refresh")
self.assertIs(selected_account, account)
async def apply_profile(db, selected_account, cookie_data):
require_lock("sync-profile")
self.assertIs(db, fake_db)
self.assertIs(selected_account, account)
self.assertIn("sessionid", cookie_data)
fake_db = SimpleNamespace(
execute=AsyncMock(side_effect=execute),
commit=AsyncMock(side_effect=commit),
refresh=AsyncMock(side_effect=refresh),
)
def write_cookie(account_id: int, cookie_data: str) -> str:
require_lock("write-cookie")
self.assertEqual(account_id, 82)
self.assertIn("sessionid", cookie_data)
return "new-cookie-path"
with (
patch.object(main, "validate_cookie_json", return_value={"cookies": [{"name": "sessionid", "value": "new"}]}),
patch.object(main, "get_owned_account", new=AsyncMock(side_effect=get_owned)) as get_owned_mock,
patch.object(main.manager, "preparation_lock", side_effect=preparation_lock),
patch.object(main.batch_start_queue, "cancel_account", new=AsyncMock(side_effect=cancel_account)) as cancel,
patch.object(main.manager, "stop_worker", new=AsyncMock(side_effect=stop_worker)) as stop,
patch.object(main, "write_cookie_file", side_effect=write_cookie),
patch.object(account_profile_module, "apply_douyin_profile", new=AsyncMock(side_effect=apply_profile)),
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value={"account_id": 82})) as build_response,
):
response = await main.update_account_cookie(
account_id=82,
body=main.AccountCookieUpdate(cookie_data="valid-cookie"),
db=fake_db,
user=user,
)
self.assertEqual(response, {"account_id": 82})
self.assertEqual(
events,
[
"lock-enter",
"authorize",
"cancel",
"stop",
"write-cookie",
"clear-profile",
"sync-profile",
"commit",
"refresh",
"lock-exit",
],
)
get_owned_mock.assert_awaited_once()
cancel.assert_awaited_once_with(82)
stop.assert_awaited_once_with(82)
self.assertEqual(account.cookie_path, "new-cookie-path")
build_response.assert_awaited_once()
async def test_cookie_delete_stays_locked_through_stop_and_commit(self):
account = SimpleNamespace(
id=83,
cookie_data="old-cookie",
cookie_path="old-path",
cookie_updated_at=object(),
im_session_data="old-im-session",
updated_at=None,
)
user = SimpleNamespace(id=9, role="operator")
events: list[str] = []
lock_active = False
def require_lock(event: str) -> None:
self.assertTrue(lock_active, f"{event} ran outside preparation lock")
events.append(event)
@asynccontextmanager
async def preparation_lock(account_id: int):
nonlocal lock_active
self.assertEqual(account_id, 83)
lock_active = True
events.append("lock-enter")
try:
yield
finally:
events.append("lock-exit")
lock_active = False
async def get_owned(_db, _user, account_id, *, write=False):
require_lock("authorize")
self.assertEqual(account_id, 83)
self.assertTrue(write)
return account
async def cancel_account(_account_id: int):
require_lock("cancel")
return 0
async def stop_worker(_account_id: int):
require_lock("stop")
return True
async def execute(_statement):
require_lock("clear-profile")
async def commit():
require_lock("commit")
fake_db = SimpleNamespace(
execute=AsyncMock(side_effect=execute),
commit=AsyncMock(side_effect=commit),
)
def clear_cookie(account_id: int) -> None:
require_lock("clear-cookie-file")
self.assertEqual(account_id, 83)
with (
patch.object(main, "get_owned_account", new=AsyncMock(side_effect=get_owned)),
patch.object(main.manager, "preparation_lock", side_effect=preparation_lock),
patch.object(main.batch_start_queue, "cancel_account", new=AsyncMock(side_effect=cancel_account)),
patch.object(main.manager, "stop_worker", new=AsyncMock(side_effect=stop_worker)),
patch.object(main, "clear_cookie_file", side_effect=clear_cookie),
):
response = await main.delete_account_cookie(
account_id=83,
db=fake_db,
user=user,
)
self.assertEqual(response["message"], "Cookie cleared successfully.")
self.assertEqual(
events,
[
"lock-enter",
"authorize",
"cancel",
"stop",
"clear-cookie-file",
"clear-profile",
"commit",
"lock-exit",
],
)
self.assertIsNone(account.cookie_data)
self.assertIsNone(account.cookie_path)
self.assertIsNone(account.cookie_updated_at)
self.assertIsNone(account.im_session_data)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,202 @@
from __future__ import annotations
import os
import sys
import unittest
from datetime import datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
import main
class _ProfileResult:
def __init__(self, profile):
self.profile = profile
def scalar_one_or_none(self):
return self.profile
class DesktopLoginSecUserIdTests(unittest.IsolatedAsyncioTestCase):
@staticmethod
def _account(*, cookie_data: str | None = "cookie-json"):
return SimpleNamespace(
id=501,
cookie_data=cookie_data,
cookie_path=None,
cookie_updated_at=datetime(2026, 7, 22, 8, 0, 0),
im_session_data=None,
)
async def test_verified_sec_user_id_returns_credential(self):
account = self._account()
profile = SimpleNamespace(
sec_user_id=" MS4wLjAB-valid ",
synced_at=account.cookie_updated_at + timedelta(seconds=1),
)
db = SimpleNamespace(execute=AsyncMock(return_value=_ProfileResult(profile)))
response = main.AccountCookieResponse(account_id=501, cookie_data="cookie-json")
with (
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value=response)) as build,
):
result = await main.get_desktop_login_credential(
account_id=501,
db=db,
user=SimpleNamespace(id=9, role="operator"),
)
self.assertIs(result, response)
db.execute.assert_awaited_once()
build.assert_awaited_once_with(account, "cookie-json")
async def test_missing_or_blank_sec_user_id_is_rejected_without_cookie(self):
for missing_value in (None, "", " "):
with self.subTest(sec_user_id=missing_value):
account = self._account()
profile = SimpleNamespace(
sec_user_id=missing_value,
synced_at=account.cookie_updated_at,
)
db = SimpleNamespace(
execute=AsyncMock(return_value=_ProfileResult(profile))
)
with (
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
):
with self.assertRaises(main.HTTPException) as caught:
await main.get_desktop_login_credential(
account_id=501,
db=db,
user=SimpleNamespace(id=9, role="operator"),
)
self.assertEqual(caught.exception.status_code, 409)
self.assertIn("缺少 sec_user_id", str(caught.exception.detail))
build.assert_not_awaited()
async def test_missing_profile_or_stale_identity_is_rejected(self):
account = self._account()
stale_profile = SimpleNamespace(
sec_user_id="MS4wLjAB-old",
synced_at=account.cookie_updated_at - timedelta(seconds=1),
)
for profile in (None, stale_profile):
with self.subTest(profile=profile):
db = SimpleNamespace(
execute=AsyncMock(return_value=_ProfileResult(profile))
)
with (
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
):
with self.assertRaises(main.HTTPException) as caught:
await main.get_desktop_login_credential(
account_id=501,
db=db,
user=SimpleNamespace(id=9, role="operator"),
)
self.assertEqual(caught.exception.status_code, 409)
build.assert_not_awaited()
async def test_profile_check_failure_is_unknown_and_fails_closed(self):
account = self._account()
db = SimpleNamespace(execute=AsyncMock(side_effect=RuntimeError("db locked")))
with (
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
patch.object(main.logger, "exception") as log_exception,
):
with self.assertRaises(main.HTTPException) as caught:
await main.get_desktop_login_credential(
account_id=501,
db=db,
user=SimpleNamespace(id=9, role="operator"),
)
self.assertEqual(caught.exception.status_code, 503)
self.assertNotIn("缺少 sec_user_id", str(caught.exception.detail))
build.assert_not_awaited()
log_exception.assert_called_once()
async def test_no_cookie_keeps_existing_no_login_state_flow(self):
account = self._account(cookie_data=None)
db = SimpleNamespace(execute=AsyncMock())
response = main.AccountCookieResponse(account_id=501, cookie_data=None)
with (
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
patch.object(main, "_get_account_cookie_data", return_value=None),
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value=response)) as build,
):
result = await main.get_desktop_login_credential(
account_id=501,
db=db,
user=SimpleNamespace(id=9, role="operator"),
)
self.assertIs(result, response)
db.execute.assert_not_awaited()
build.assert_awaited_once_with(account, None)
async def test_normal_cookie_management_endpoint_is_not_guarded(self):
account = self._account()
db = SimpleNamespace(execute=AsyncMock())
response = main.AccountCookieResponse(account_id=501, cookie_data="cookie-json")
with (
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
patch.object(main, "_build_cookie_response", new=AsyncMock(return_value=response)) as build,
):
result = await main.get_account_cookie(
account_id=501,
purpose="management",
db=db,
user=SimpleNamespace(id=9, role="operator"),
)
self.assertIs(result, response)
db.execute.assert_not_awaited()
build.assert_awaited_once_with(account, "cookie-json")
async def test_legacy_cookie_request_is_guarded_for_existing_desktop_clients(self):
account = self._account()
profile = SimpleNamespace(
sec_user_id=" ",
synced_at=account.cookie_updated_at,
)
db = SimpleNamespace(execute=AsyncMock(return_value=_ProfileResult(profile)))
with (
patch.object(main, "get_owned_account", new=AsyncMock(return_value=account)),
patch.object(main, "_build_cookie_response", new=AsyncMock()) as build,
):
with self.assertRaises(main.HTTPException) as caught:
await main.get_account_cookie(
account_id=501,
purpose=None,
db=db,
user=SimpleNamespace(id=9, role="operator"),
)
self.assertEqual(caught.exception.status_code, 409)
self.assertIn("缺少 sec_user_id", str(caught.exception.detail))
build.assert_not_awaited()
if __name__ == "__main__":
unittest.main()
+154
View File
@@ -0,0 +1,154 @@
from __future__ import annotations
import os
import sys
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
import main
class _FakeResponse:
def __init__(
self,
*,
final_host: str,
headers: dict[str, str] | None = None,
chunks: tuple[bytes, ...] = (),
) -> None:
self.url = SimpleNamespace(host=final_host)
self.headers = headers or {}
self._chunks = chunks
self.raise_for_status = MagicMock()
self.iteration_started = False
async def aiter_bytes(self):
self.iteration_started = True
for chunk in self._chunks:
yield chunk
class _AsyncContext:
def __init__(self, value) -> None:
self.value = value
async def __aenter__(self):
return self.value
async def __aexit__(self, exc_type, exc, traceback):
return False
class _FakeAsyncClient:
def __init__(self, response: _FakeResponse) -> None:
self.response = response
self.stream_calls: list[tuple[tuple, dict]] = []
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc, traceback):
return False
def stream(self, *args, **kwargs):
self.stream_calls.append((args, kwargs))
return _AsyncContext(self.response)
class MediaProxyHostGuardTests(unittest.TestCase):
def test_allows_each_root_domain_and_its_subdomains(self):
for allowed in main._MEDIA_PROXY_HOSTS:
with self.subTest(host=allowed):
self.assertTrue(main._is_allowed_media_host(allowed))
self.assertTrue(main._is_allowed_media_host(f"cdn.images.{allowed}"))
self.assertTrue(main._is_allowed_media_host("CDN.DOUYINPIC.COM."))
def test_rejects_empty_suffix_tricks_and_similar_domains(self):
rejected = (
"",
".",
"douyin.com.evil",
"evildouyin.com",
"byteimg.com.evil.example",
"evilbyteimg.com",
"ibyteimg.comevil",
"douyin.co",
"douyinpic.co",
"douyin-static.com",
"snssdk.example",
)
for hostname in rejected:
with self.subTest(host=hostname):
self.assertFalse(main._is_allowed_media_host(hostname))
class MediaProxyResponseGuardTests(unittest.IsolatedAsyncioTestCase):
async def _call_proxy(self, response: _FakeResponse):
client = _FakeAsyncClient(response)
constructor = MagicMock(return_value=client)
with patch.object(main.httpx, "AsyncClient", constructor):
result = await main.proxy_media(
url="https://cdn.douyinpic.com/media/test.jpg"
)
return result, client, constructor
async def test_rejects_redirect_to_non_allowlisted_final_domain(self):
response = _FakeResponse(
final_host="attacker.example",
headers={"content-type": "image/jpeg"},
chunks=(b"not-read",),
)
with self.assertRaises(main.HTTPException) as caught:
await self._call_proxy(response)
self.assertEqual(caught.exception.status_code, 400)
self.assertFalse(response.iteration_started)
async def test_rejects_oversized_content_length_before_streaming(self):
response = _FakeResponse(
final_host="cdn.douyinpic.com",
headers={"content-length": "9", "content-type": "image/jpeg"},
chunks=(b"not-read",),
)
with (
patch.object(main, "_MEDIA_PROXY_MAX_BYTES", 8),
self.assertRaises(main.HTTPException) as caught,
):
await self._call_proxy(response)
self.assertEqual(caught.exception.status_code, 413)
self.assertFalse(response.iteration_started)
async def test_rejects_stream_when_actual_bytes_exceed_limit(self):
response = _FakeResponse(
final_host="cdn.douyinpic.com",
headers={"content-length": "0", "content-type": "image/jpeg"},
chunks=(b"12345", b"6789"),
)
with (
patch.object(main, "_MEDIA_PROXY_MAX_BYTES", 8),
self.assertRaises(main.HTTPException) as caught,
):
await self._call_proxy(response)
self.assertEqual(caught.exception.status_code, 413)
self.assertTrue(response.iteration_started)
if __name__ == "__main__":
unittest.main()
+458
View File
@@ -0,0 +1,458 @@
import asyncio
import sys
import unittest
from datetime import datetime
from pathlib import Path
BACKEND_DIR = Path(__file__).resolve().parents[1]
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im.reply_queue import AccountReplyQueue
class AccountReplyQueueTests(unittest.IsolatedAsyncioTestCase):
async def _start_queue(self, account_id: int, **kwargs) -> AccountReplyQueue:
queue = AccountReplyQueue(account_id=account_id, **kwargs)
await queue.start()
self.addAsyncCleanup(queue.stop)
return queue
async def _wait_until_idle(
self,
queue: AccountReplyQueue,
timeout: float = 0.5,
) -> None:
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
while queue.pending_count and loop.time() < deadline:
await asyncio.sleep(0.002)
self.assertEqual(queue.pending_count, 0)
async def test_one_account_runs_three_jobs_at_successive_fifo_slots(self):
queue = await self._start_queue(account_id=101)
interval = 0.05
loop = asyncio.get_running_loop()
started_at = loop.time()
calls: list[tuple[int, float]] = []
finished = asyncio.Event()
def callback_for(index: int):
async def callback() -> None:
calls.append((index, loop.time() - started_at))
if len(calls) == 3:
finished.set()
return callback
for index in range(3):
await queue.enqueue(interval, callback_for(index), description=str(index))
await asyncio.wait_for(finished.wait(), timeout=0.75)
await self._wait_until_idle(queue)
self.assertEqual([index for index, _ in calls], [0, 1, 2])
elapsed = [timestamp for _, timestamp in calls]
for timestamp, expected in zip(elapsed, (interval, interval * 2, interval * 3)):
self.assertGreaterEqual(timestamp, expected - 0.015)
self.assertLess(timestamp, expected + 0.15)
self.assertGreaterEqual(elapsed[1] - elapsed[0], interval - 0.02)
self.assertGreaterEqual(elapsed[2] - elapsed[1], interval - 0.02)
async def test_separate_account_queues_reach_first_slot_without_blocking(self):
first_queue = await self._start_queue(account_id=201)
second_queue = await self._start_queue(account_id=202)
interval = 0.04
loop = asyncio.get_running_loop()
started_at = loop.time()
first_started = asyncio.Event()
release_first = asyncio.Event()
second_finished = asyncio.Event()
timestamps: dict[str, float] = {}
async def first_callback() -> None:
timestamps["first"] = loop.time() - started_at
first_started.set()
await release_first.wait()
async def second_callback() -> None:
timestamps["second"] = loop.time() - started_at
second_finished.set()
await asyncio.gather(
first_queue.enqueue(interval, first_callback, description="first account"),
second_queue.enqueue(interval, second_callback, description="second account"),
)
await asyncio.wait_for(first_started.wait(), timeout=0.4)
await asyncio.wait_for(second_finished.wait(), timeout=0.4)
self.assertFalse(release_first.is_set())
self.assertLess(abs(timestamps["first"] - timestamps["second"]), 0.05)
release_first.set()
await self._wait_until_idle(first_queue)
await self._wait_until_idle(second_queue)
async def test_stop_before_due_discards_pending_job(self):
queue = await self._start_queue(account_id=301)
callback_called = asyncio.Event()
async def callback() -> None:
callback_called.set()
await queue.enqueue(0.12, callback, description="must be discarded")
await asyncio.sleep(0.02)
self.assertEqual(queue.pending_count, 1)
await queue.stop()
self.assertEqual(queue.pending_count, 0)
await asyncio.sleep(0.13)
self.assertFalse(callback_called.is_set())
async def test_callback_failure_does_not_block_following_job(self):
errors: list[tuple[str, str]] = []
def on_error(description: str, exc: BaseException) -> None:
errors.append((description, type(exc).__name__))
queue = await self._start_queue(account_id=401, on_error=on_error)
calls: list[str] = []
second_finished = asyncio.Event()
async def failing_callback() -> None:
calls.append("first")
raise ValueError("expected test failure")
async def following_callback() -> None:
calls.append("second")
second_finished.set()
with self.assertLogs("douyin_im.reply_queue", level="ERROR"):
await queue.enqueue(0.03, failing_callback, description="first")
await queue.enqueue(0.03, following_callback, description="second")
await asyncio.wait_for(second_finished.wait(), timeout=0.5)
await self._wait_until_idle(queue)
self.assertEqual(calls, ["first", "second"])
self.assertEqual(errors, [("first", "ValueError")])
async def test_snapshot_exposes_waiting_job_details(self):
queue = await self._start_queue(account_id=501)
async def callback() -> None:
pass
await queue.enqueue(
0.2,
callback,
description="回复 测试用户",
details={
"sender_name": "测试用户",
"conversation_id": "conv-501",
"incoming_content": "你好",
"replies": ["您好"],
},
)
items = await queue.snapshot()
self.assertEqual(len(items), 1)
self.assertEqual(items[0]["position"], 1)
self.assertEqual(items[0]["status"], "waiting")
self.assertEqual(items[0]["sender_name"], "测试用户")
self.assertEqual(items[0]["incoming_content"], "你好")
self.assertEqual(items[0]["replies"], ["您好"])
self.assertNotIn("callback", items[0])
async def test_send_now_wakes_first_job_and_moves_later_slots_forward(self):
queue = await self._start_queue(account_id=502)
interval = 0.2
sent = asyncio.Event()
async def first_callback() -> None:
sent.set()
async def noop() -> None:
pass
await queue.enqueue(interval, first_callback, description="first")
await queue.enqueue(interval, noop, description="second")
await queue.enqueue(interval, noop, description="third")
before = await queue.snapshot()
result = await queue.send_now(before[0]["job_id"])
self.assertEqual(result["status"], "accepted")
self.assertEqual(result["shifted_count"], 2)
await asyncio.wait_for(sent.wait(), timeout=0.1)
await asyncio.sleep(0)
after = await queue.snapshot()
self.assertEqual([item["description"] for item in after], ["second", "third"])
for old, new in zip(before[1:], after):
old_due = datetime.fromisoformat(old["scheduled_at"]).timestamp()
new_due = datetime.fromisoformat(new["scheduled_at"]).timestamp()
self.assertAlmostEqual(old_due - new_due, interval, delta=0.05)
async def test_send_now_middle_runs_first_and_only_shifts_jobs_behind_it(self):
queue = await self._start_queue(account_id=503)
interval = 0.12
calls: list[str] = []
all_sent = asyncio.Event()
def callback_for(name: str):
async def callback() -> None:
calls.append(name)
if len(calls) == 3:
all_sent.set()
return callback
for name in ("first", "middle", "last"):
await queue.enqueue(interval, callback_for(name), description=name)
before = await queue.snapshot()
result = await queue.send_now(before[1]["job_id"])
self.assertEqual(result["shifted_count"], 1)
await asyncio.wait_for(all_sent.wait(), timeout=0.6)
self.assertEqual(calls, ["middle", "first", "last"])
first_before = datetime.fromisoformat(before[0]["scheduled_at"]).timestamp()
# The first normal job keeps its original slot; this is also validated by execution order.
self.assertGreater(first_before, datetime.now().timestamp() - 1)
async def test_send_now_does_not_overlap_active_send(self):
queue = await self._start_queue(account_id=504)
first_started = asyncio.Event()
release_first = asyncio.Event()
second_started = asyncio.Event()
async def first_callback() -> None:
first_started.set()
await release_first.wait()
async def second_callback() -> None:
second_started.set()
await queue.enqueue(0.01, first_callback, description="first")
await queue.enqueue(0.2, second_callback, description="second")
await asyncio.wait_for(first_started.wait(), timeout=0.2)
items = await queue.snapshot()
second = next(item for item in items if item["description"] == "second")
result = await queue.send_now(second["job_id"])
self.assertEqual(result["status"], "accepted")
await asyncio.sleep(0.03)
self.assertFalse(second_started.is_set())
release_first.set()
await asyncio.wait_for(second_started.wait(), timeout=0.2)
async def test_send_now_tail_releases_slot_for_next_enqueue(self):
queue = await self._start_queue(account_id=505)
interval = 0.2
async def noop() -> None:
pass
for name in ("first", "second", "tail"):
await queue.enqueue(interval, noop, description=name)
before = await queue.snapshot()
old_tail_due = datetime.fromisoformat(before[2]["scheduled_at"]).timestamp()
await queue.send_now(before[2]["job_id"])
await queue.enqueue(interval, noop, description="new-tail")
after = await queue.snapshot()
new_tail = next(item for item in after if item["description"] == "new-tail")
new_tail_due = datetime.fromisoformat(new_tail["scheduled_at"]).timestamp()
self.assertAlmostEqual(new_tail_due, old_tail_due, delta=0.05)
async def test_repeated_send_now_is_idempotent(self):
queue = await self._start_queue(account_id=506)
calls = 0
finished = asyncio.Event()
async def callback() -> None:
nonlocal calls
calls += 1
finished.set()
await queue.enqueue(0.2, callback, description="once")
item = (await queue.snapshot())[0]
first = await queue.send_now(item["job_id"])
second = await queue.send_now(item["job_id"])
self.assertEqual(first["status"], "accepted")
self.assertIn(second["status"], {"already_requested", "already_sending"})
await asyncio.wait_for(finished.wait(), timeout=0.1)
await asyncio.sleep(0.02)
self.assertEqual(calls, 1)
async def test_merge_pending_keeps_one_job_slot_and_callback(self):
queue = await self._start_queue(account_id=507)
calls = 0
finished = asyncio.Event()
async def callback() -> None:
nonlocal calls
calls += 1
finished.set()
await queue.enqueue(
0.3,
callback,
description="conversation",
details={
"incoming_content": "first",
"incoming_contents": ["first"],
"message_count": 1,
"replies": ["one reply"],
},
merge_key="conv:507",
)
before = (await queue.snapshot())[0]
def append_second(details):
details["incoming_content"] = "second"
details["incoming_contents"].append("second")
details["message_count"] = 2
return details
result = await queue.merge_pending("conv:507", append_second)
after = await queue.snapshot()
self.assertEqual(result["status"], "merged")
self.assertEqual(result["position"], 1)
self.assertEqual(len(after), 1)
self.assertEqual(after[0]["job_id"], before["job_id"])
self.assertEqual(after[0]["incoming_contents"], ["first", "second"])
self.assertEqual(after[0]["replies"], ["one reply"])
old_due = datetime.fromisoformat(before["scheduled_at"]).timestamp()
new_due = datetime.fromisoformat(after[0]["scheduled_at"]).timestamp()
self.assertAlmostEqual(old_due, new_due, delta=0.02)
# Management snapshots must not expose the queue's mutable nested list.
after[0]["incoming_contents"].append("external mutation")
self.assertEqual(
(await queue.snapshot())[0]["incoming_contents"],
["first", "second"],
)
await queue.send_now(before["job_id"])
await asyncio.wait_for(finished.wait(), timeout=0.2)
self.assertEqual(calls, 1)
async def test_urgent_job_can_still_merge_before_sending(self):
queue = await self._start_queue(account_id=508)
blocker_started = asyncio.Event()
release_blocker = asyncio.Event()
merged_sent = asyncio.Event()
merged_calls = 0
async def blocker() -> None:
blocker_started.set()
await release_blocker.wait()
async def merged_callback() -> None:
nonlocal merged_calls
merged_calls += 1
merged_sent.set()
await queue.enqueue(0.01, blocker, description="blocker")
await queue.enqueue(
0.3,
merged_callback,
description="mergeable",
details={"incoming_content": "first", "incoming_contents": ["first"]},
merge_key="conv:508",
)
await asyncio.wait_for(blocker_started.wait(), timeout=0.2)
mergeable = next(
item for item in await queue.snapshot() if item["description"] == "mergeable"
)
await queue.send_now(mergeable["job_id"])
def append_second(details):
details["incoming_contents"].append("second")
details["message_count"] = 2
return details
result = await queue.merge_pending("conv:508", append_second)
self.assertEqual(result["status"], "merged")
self.assertEqual(result["queue_status"], "ready")
release_blocker.set()
await asyncio.wait_for(merged_sent.wait(), timeout=0.2)
self.assertEqual(merged_calls, 1)
async def test_active_job_is_sealed_against_merge(self):
queue = await self._start_queue(account_id=509)
started = asyncio.Event()
release = asyncio.Event()
async def callback() -> None:
started.set()
await release.wait()
await queue.enqueue(
0.01,
callback,
details={"incoming_content": "first", "incoming_contents": ["first"]},
merge_key="conv:509",
)
await asyncio.wait_for(started.wait(), timeout=0.2)
result = await queue.merge_pending("conv:509", lambda details: details)
self.assertEqual(result["status"], "not_found")
release.set()
async def test_merge_keys_bridge_partial_peer_and_conversation_ids(self):
queue = await self._start_queue(account_id=510)
async def callback() -> None:
pass
await queue.enqueue(
0.3,
callback,
details={"incoming_contents": ["first"], "message_count": 1},
merge_keys=("peer:123",),
)
def append_second(details):
details["incoming_contents"].append("second")
details["message_count"] = 2
return details
result = await queue.merge_pending(
("conv:510", "peer:123"),
append_second,
)
self.assertEqual(result["status"], "merged")
self.assertEqual(queue.pending_count, 1)
self.assertEqual(
(await queue.snapshot())[0]["incoming_contents"],
["first", "second"],
)
async def test_different_conversation_ids_do_not_merge_on_shared_peer(self):
queue = await self._start_queue(account_id=511)
async def callback() -> None:
pass
await queue.enqueue(
0.3,
callback,
details={"incoming_contents": ["first"]},
merge_keys=("conv:first", "peer:123"),
)
result = await queue.merge_pending(
("conv:second", "peer:123"),
lambda details: details,
)
self.assertEqual(result["status"], "not_found")
self.assertEqual(queue.pending_count, 1)
if __name__ == "__main__":
unittest.main()
+165
View File
@@ -0,0 +1,165 @@
from __future__ import annotations
import os
import sys
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
import main
def _queue_item(account_id: int, job_id: str = "job-1") -> dict:
return {
"job_id": job_id,
"account_id": account_id,
"position": 1,
"status": "waiting",
"expedited": False,
"description": "回复 测试用户",
"sender_name": "测试用户",
"sender_id": "peer-1",
"sender_avatar": None,
"conversation_id": "conv-1",
"incoming_content": "你好",
"incoming_contents": ["你好", "第二条"],
"message_count": 2,
"replies": ["您好"],
"interval_seconds": 60,
"enqueued_at": "2026-07-20T10:00:00+00:00",
"scheduled_at": "2026-07-20T10:01:00+00:00",
"remaining_seconds": 60,
}
class _FakeService:
def __init__(self, account_id: int, items=None, action=None):
self.account_id = account_id
self._running = True
self.items = list(items or [])
self.action = action or {
"status": "accepted",
"job_id": "job-1",
"shifted_count": 2,
}
async def get_reply_queue_snapshot(self):
return list(self.items)
async def send_queued_reply_now(self, job_id: str):
return {**self.action, "job_id": job_id}
class ReplyQueueApiTests(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
self.original_workers = main.manager.workers
main.manager.workers = {}
async def asyncTearDown(self):
main.manager.workers = self.original_workers
async def test_summary_filters_accounts_by_owner_scope(self):
service_one = _FakeService(1, [_queue_item(1)])
service_two = _FakeService(2, [_queue_item(2, "job-2")])
main.manager.workers = {
1: SimpleNamespace(is_running=True, _im_service=service_one),
2: SimpleNamespace(is_running=True, _im_service=service_two),
}
with patch.object(main, "owned_account_ids", AsyncMock(return_value={1})):
response = await main.get_reply_queue_summaries(
db=object(),
user=SimpleNamespace(id=10, role="operator"),
)
self.assertEqual(response.total_pending, 1)
self.assertEqual([item.account_id for item in response.items], [1])
async def test_offline_account_detail_returns_empty_snapshot(self):
account = SimpleNamespace(id=1, reply_delay_seconds=0)
with (
patch.object(main, "get_owned_account", AsyncMock(return_value=account)),
patch.object(main, "_global_reply_delay_seconds", return_value=0),
):
response = await main.get_account_reply_queue(
account_id=1,
db=object(),
user=SimpleNamespace(id=10, role="operator"),
)
self.assertFalse(response.running)
self.assertEqual(response.pending_count, 0)
self.assertEqual(response.items, [])
async def test_detail_preserves_merged_incoming_messages(self):
account = SimpleNamespace(id=1, reply_delay_seconds=60)
service = _FakeService(1, [_queue_item(1)])
main.manager.workers = {
1: SimpleNamespace(is_running=True, _im_service=service),
}
with (
patch.object(main, "get_owned_account", AsyncMock(return_value=account)),
patch.object(main, "_resolve_effective_reply_delay", return_value=60),
):
response = await main.get_account_reply_queue(
account_id=1,
db=object(),
user=SimpleNamespace(id=10, role="operator"),
)
self.assertEqual(response.pending_count, 1)
self.assertEqual(response.items[0].incoming_contents, ["你好", "第二条"])
self.assertEqual(response.items[0].message_count, 2)
self.assertEqual(response.items[0].replies, ["您好"])
async def test_send_now_returns_shifted_count(self):
service = _FakeService(1)
main.manager.workers = {
1: SimpleNamespace(is_running=True, _im_service=service),
}
with (
patch.object(main, "get_owned_account", AsyncMock(return_value=SimpleNamespace(id=1))),
patch.object(main.system_logger, "record"),
):
response = await main.send_account_queued_reply_now(
account_id=1,
job_id="job-1",
db=object(),
user=SimpleNamespace(id=10, role="operator"),
)
self.assertEqual(response.status, "accepted")
self.assertEqual(response.shifted_count, 2)
async def test_cross_account_or_finished_job_returns_not_found(self):
service = _FakeService(1, action={"status": "not_found"})
main.manager.workers = {
1: SimpleNamespace(is_running=True, _im_service=service),
}
with patch.object(
main,
"get_owned_account",
AsyncMock(return_value=SimpleNamespace(id=1)),
):
with self.assertRaises(main.HTTPException) as caught:
await main.send_account_queued_reply_now(
account_id=1,
job_id="other-account-job",
db=object(),
user=SimpleNamespace(id=10, role="operator"),
)
self.assertEqual(caught.exception.status_code, 404)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,421 @@
from __future__ import annotations
import asyncio
import os
import sys
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from auth.system_settings import SystemSettingsData, set_cached_settings
from rpa_engine.douyin_im.service import DouyinImService
from rpa_engine.playwright_worker import DouyinWorker
class _RecordingQueue:
def __init__(self) -> None:
self.jobs: list[tuple[float, object, str, dict, frozenset[str]]] = []
async def enqueue(
self,
delay_seconds,
callback,
description="",
details=None,
merge_key="",
merge_keys=None,
) -> int:
keys = merge_keys if merge_keys is not None else [merge_key]
normalized_keys = frozenset(str(key) for key in keys if str(key or "").strip())
self.jobs.append(
(delay_seconds, callback, description, dict(details or {}), normalized_keys)
)
return len(self.jobs)
async def merge_pending(self, merge_key, details_merger):
incoming_keys = (
frozenset([merge_key])
if isinstance(merge_key, str)
else frozenset(merge_key)
)
for index, job in enumerate(self.jobs):
existing_conversations = {
key for key in job[4] if key.startswith("conv:")
}
incoming_conversations = {
key for key in incoming_keys if key.startswith("conv:")
}
same_conversation = bool(
existing_conversations & incoming_conversations
)
different_explicit_conversations = bool(
existing_conversations
and incoming_conversations
and not same_conversation
)
same_peer = bool(
{key for key in job[4] if key.startswith("peer:")}
& {key for key in incoming_keys if key.startswith("peer:")}
)
if not same_conversation and (different_explicit_conversations or not same_peer):
continue
merged_details = details_merger(dict(job[3]))
self.jobs[index] = (
job[0],
job[1],
job[2],
merged_details,
frozenset(job[4] | incoming_keys),
)
return {
"status": "merged",
"job_id": f"job-{index + 1}",
"position": index + 1,
"message_count": merged_details.get("message_count", 1),
}
return {"status": "not_found"}
def _build_service(delay_seconds: int = 60):
match_reply = AsyncMock(return_value=["自动回复"])
log_fn = AsyncMock()
service = DouyinImService(
session=SimpleNamespace(my_uid="999"),
match_reply=match_reply,
log_fn=log_fn,
account_id=1,
)
service._running = True
service._reply_queue = _RecordingQueue()
service._resolve_peer_profile = AsyncMock(return_value=("张三", "", "123"))
service._resolve_cooldown_seconds = AsyncMock(return_value=0)
service._resolve_reply_delay_seconds = AsyncMock(return_value=delay_seconds)
return service, match_reply, log_fn
class ReplyQueueIntegrationTests(unittest.IsolatedAsyncioTestCase):
async def test_same_message_from_ws_and_poll_is_queued_once(self):
service, match_reply, _ = _build_service()
message = {
"conversation_id": "conv-1",
"sender_uid": "123",
"sender_name": "张三",
"content": "你好",
"server_message_id": "mid-1",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await asyncio.gather(
service._handle_incoming(dict(message)),
service._handle_incoming(dict(message)),
)
self.assertEqual(len(service._reply_queue.jobs), 1)
self.assertEqual(match_reply.await_count, 1)
details = service._reply_queue.jobs[0][3]
self.assertEqual(details["sender_name"], "张三")
self.assertEqual(details["conversation_id"], "conv-1")
self.assertEqual(details["incoming_content"], "你好")
self.assertEqual(details["replies"], ["自动回复"])
async def test_same_conversation_with_different_message_ids_merges(self):
service, match_reply, _ = _build_service()
base = {
"conversation_id": "conv-1",
"sender_uid": "123",
"sender_name": "张三",
"content": "你好",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming({**base, "server_message_id": "mid-1"})
await service._handle_incoming({**base, "server_message_id": "mid-2"})
self.assertEqual(len(service._reply_queue.jobs), 1)
self.assertEqual(match_reply.await_count, 1)
details = service._reply_queue.jobs[0][3]
self.assertEqual(details["incoming_contents"], ["你好", "你好"])
self.assertEqual(details["message_count"], 2)
self.assertEqual(details["replies"], ["自动回复"])
async def test_pending_conversation_merges_before_cooldown_filter(self):
service, match_reply, _ = _build_service()
service._resolve_cooldown_seconds = AsyncMock(return_value=120)
base = {
"conversation_id": "conv-1",
"sender_uid": "123",
"sender_name": "张三",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming(
{**base, "content": "first", "server_message_id": "mid-1"}
)
await service._handle_incoming(
{
**base,
"conversation_id": "conv-1",
"content": "second",
"server_message_id": "mid-2",
}
)
self.assertEqual(len(service._reply_queue.jobs), 1)
self.assertEqual(match_reply.await_count, 1)
details = service._reply_queue.jobs[0][3]
self.assertEqual(details["incoming_contents"], ["first", "second"])
self.assertEqual(details["message_count"], 2)
async def test_http_previews_from_different_conversations_do_not_collide(self):
service, _, _ = _build_service()
base = {
"sender_uid": "",
"sender_name": "同名用户",
"content": "相同内容",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming({**base, "conversation_id": "conv-a"})
await service._handle_incoming({**base, "conversation_id": "conv-b"})
self.assertEqual(len(service._reply_queue.jobs), 2)
async def test_missing_conversation_id_uses_peer_uid_for_merge(self):
service, match_reply, _ = _build_service()
base = {
"conversation_id": "",
"sender_uid": "123",
"sender_name": "张三",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming(
{**base, "content": "first", "server_message_id": "mid-1"}
)
await service._handle_incoming(
{
**base,
"conversation_id": "conv-1",
"content": "second",
"server_message_id": "mid-2",
}
)
self.assertEqual(len(service._reply_queue.jobs), 1)
self.assertEqual(match_reply.await_count, 1)
self.assertEqual(
service._reply_queue.jobs[0][3]["incoming_contents"],
["first", "second"],
)
async def test_peer_alias_added_later_merges_with_conversation_only_job(self):
service, match_reply, _ = _build_service()
async def resolve_profile(conv_id, sender_uid, sender, sender_avatar):
return "张三", "", str(sender_uid or "")
service._resolve_peer_profile = AsyncMock(side_effect=resolve_profile)
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming(
{
"conversation_id": "conv-1",
"sender_uid": "",
"sender_name": "张三",
"content": "first",
"server_message_id": "mid-1",
}
)
await service._handle_incoming(
{
"conversation_id": "conv-1",
"sender_uid": "123",
"sender_name": "张三",
"content": "second",
"server_message_id": "mid-2",
}
)
self.assertEqual(len(service._reply_queue.jobs), 1)
self.assertEqual(match_reply.await_count, 1)
self.assertEqual(
service._reply_queue.jobs[0][3]["incoming_contents"],
["first", "second"],
)
async def test_missing_conversation_and_peer_ids_do_not_merge_by_name(self):
service, match_reply, _ = _build_service()
service._resolve_peer_profile = AsyncMock(return_value=("同名用户", "", ""))
base = {
"conversation_id": "",
"sender_uid": "",
"sender_name": "同名用户",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming(
{**base, "content": "first", "server_message_id": "mid-1"}
)
await service._handle_incoming(
{**base, "content": "second", "server_message_id": "mid-2"}
)
self.assertEqual(len(service._reply_queue.jobs), 2)
self.assertEqual(match_reply.await_count, 2)
async def test_slow_first_message_keeps_arrival_order(self):
service, _, _ = _build_service()
async def resolve_profile(conv_id, sender_uid, sender, sender_avatar):
if conv_id == "conv-first":
await asyncio.sleep(0.03)
return "先到", "", "101"
return "后到", "", "102"
service._resolve_peer_profile = AsyncMock(side_effect=resolve_profile)
first = {
"conversation_id": "conv-first",
"sender_uid": "101",
"sender_name": "先到",
"content": "第一条",
"server_message_id": "mid-first",
}
second = {
"conversation_id": "conv-second",
"sender_uid": "102",
"sender_name": "后到",
"content": "第二条",
"server_message_id": "mid-second",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
first_task = asyncio.create_task(service._handle_incoming(first))
await asyncio.sleep(0)
second_task = asyncio.create_task(service._handle_incoming(second))
await asyncio.gather(first_task, second_task)
descriptions = [job[2] for job in service._reply_queue.jobs]
self.assertEqual(descriptions, ["回复 先到", "回复 后到"])
async def test_zero_effective_delay_skips_queue_and_sends_immediately(self):
service, _, _ = _build_service(delay_seconds=0)
service._send_auto_reply = AsyncMock()
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming(
{
"conversation_id": "conv-1",
"sender_uid": "123",
"sender_name": "张三",
"content": "你好",
"server_message_id": "mid-1",
}
)
self.assertEqual(len(service._reply_queue.jobs), 0)
service._send_auto_reply.assert_awaited_once()
async def test_zero_delay_keeps_separate_immediate_replies(self):
service, _, _ = _build_service(delay_seconds=0)
service._send_auto_reply = AsyncMock()
base = {
"conversation_id": "conv-1",
"sender_uid": "123",
"sender_name": "张三",
}
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._handle_incoming(
{**base, "content": "first", "server_message_id": "mid-1"}
)
await service._handle_incoming(
{**base, "content": "second", "server_message_id": "mid-2"}
)
self.assertEqual(len(service._reply_queue.jobs), 0)
self.assertEqual(service._send_auto_reply.await_count, 2)
async def test_session_stop_during_first_payload_cancels_remaining_payloads(self):
service, _, log_fn = _build_service()
async def fail_and_stop(*args, **kwargs):
service.last_error = "INVALID_REQUEST"
service._running = False
return False, None
service._send_text = AsyncMock(side_effect=fail_and_stop)
with unittest.mock.patch(
"rpa_engine.douyin_im.service.system_logger.record", Mock()
):
await service._send_auto_reply(
sender="张三",
content="你好",
conv_id="conv-1",
replies=["第一段", "第二段"],
peer_key="123",
cooldown=0,
log_kwargs={
"sender_name": "张三",
"sender_id": "123",
"sender_avatar": None,
"message": "你好",
},
)
self.assertEqual(service._send_text.await_count, 1)
log_fn.assert_awaited_once()
class ReplyDelayResolutionTests(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
self.worker = DouyinWorker(account_id=1)
async def test_account_value_overrides_system_default(self):
set_cached_settings(SystemSettingsData(auto_reply_delay_seconds=60))
self.worker.get_reply_delay = AsyncMock(return_value=15)
self.assertEqual(await self.worker.resolve_reply_delay_seconds(), 15)
async def test_unset_account_uses_system_default(self):
set_cached_settings(SystemSettingsData(auto_reply_delay_seconds=60))
self.worker.get_reply_delay = AsyncMock(return_value=None)
self.assertEqual(await self.worker.resolve_reply_delay_seconds(), 60)
async def test_both_unset_skip_queue_rule(self):
set_cached_settings(SystemSettingsData(auto_reply_delay_seconds=0))
self.worker.get_reply_delay = AsyncMock(return_value=None)
self.assertEqual(await self.worker.resolve_reply_delay_seconds(), 0)
if __name__ == "__main__":
unittest.main()
+676
View File
@@ -0,0 +1,676 @@
from __future__ import annotations
import os
import sys
import unittest
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im.session import DouyinImSession
from rpa_engine import account_profile as account_profile_module
from rpa_engine.playwright_worker import DouyinWorker
class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
async def test_im_direct_missing_sec_user_id_exits_before_online_state(self):
worker = DouyinWorker(account_id=301, login_mode="im_direct")
worker._require_sec_user_id = AsyncMock(return_value=False)
worker._load_user_agent = AsyncMock(return_value="test-agent")
im_session = DouyinImSession(
cookies={"sessionid": "test-session"},
my_uid=30101,
)
worker._build_im_session_from_storage = AsyncMock(return_value=im_session)
worker._persist_im_session = AsyncMock()
worker._run_im_direct_service = AsyncMock()
with patch(
"rpa_engine.playwright_worker.validate_im_session",
new=AsyncMock(return_value=(True, "ready")),
) as validate:
started, reason = await worker._try_cookie_only_im_start(
{"cookies": [{"name": "sessionid", "value": "test-session"}]}
)
self.assertFalse(started)
self.assertIn("sec_user_id", reason)
worker._require_sec_user_id.assert_awaited_once()
require_call = worker._require_sec_user_id.await_args
self.assertTrue(require_call.kwargs["refresh_if_missing"])
self.assertTrue(require_call.kwargs["refresh_if_stale"])
worker._build_im_session_from_storage.assert_awaited_once()
validate.assert_awaited_once_with(im_session)
worker._persist_im_session.assert_not_awaited()
worker._run_im_direct_service.assert_not_awaited()
async def test_running_guard_precedes_disabled_follow_welcome_setting(self):
worker = DouyinWorker(account_id=302, login_mode="im_direct")
worker.is_running = True
worker._im_service = SimpleNamespace(_running=True)
worker._require_sec_user_id = AsyncMock(return_value=False)
worker.get_db = AsyncMock()
await worker.follow_welcome_tick()
worker._require_sec_user_id.assert_awaited_once()
require_call = worker._require_sec_user_id.await_args
self.assertFalse(require_call.kwargs.get("refresh_if_missing", False))
# The identity guard must run before the account's follow-welcome flag
# is queried. Otherwise accounts with that feature disabled could stay
# hosted indefinitely without a sec_user_id.
worker.get_db.assert_not_awaited()
async def test_blank_sec_user_id_is_missing_and_stops_hosting(self):
worker = DouyinWorker(account_id=303, login_mode="im_direct")
worker._load_sec_user_id = AsyncMock(return_value=" ")
worker._refresh_sec_user_id = AsyncMock(return_value="\t")
worker._stop_for_missing_sec_user_id = AsyncMock()
accepted = await worker._require_sec_user_id(
"直连启动前",
refresh_if_missing=True,
)
self.assertFalse(accepted)
worker._load_sec_user_id.assert_awaited()
worker._refresh_sec_user_id.assert_awaited_once_with()
worker._stop_for_missing_sec_user_id.assert_awaited_once_with("直连启动前")
async def test_existing_sec_user_id_does_not_refresh_or_stop(self):
worker = DouyinWorker(account_id=304, login_mode="im_direct")
worker._load_sec_user_id = AsyncMock(return_value="MS4wLjABAAAA-valid")
worker._refresh_sec_user_id = AsyncMock()
worker._stop_for_missing_sec_user_id = AsyncMock()
accepted = await worker._require_sec_user_id(
"运行中",
refresh_if_missing=True,
)
self.assertTrue(accepted)
worker._load_sec_user_id.assert_awaited_once_with()
worker._refresh_sec_user_id.assert_not_awaited()
worker._stop_for_missing_sec_user_id.assert_not_awaited()
async def test_identity_read_error_does_not_claim_field_is_missing(self):
worker = DouyinWorker(account_id=307, login_mode="im_direct")
worker._load_sec_user_id = AsyncMock(
side_effect=RuntimeError("database temporarily unavailable")
)
worker._refresh_sec_user_id = AsyncMock()
worker._stop_for_missing_sec_user_id = AsyncMock()
with self.assertRaisesRegex(RuntimeError, "database temporarily unavailable"):
await worker._require_sec_user_id(
"托管运行中",
refresh_if_missing=True,
)
worker._refresh_sec_user_id.assert_not_awaited()
worker._stop_for_missing_sec_user_id.assert_not_awaited()
async def test_unknown_profile_refresh_error_does_not_stop_as_missing(self):
worker = DouyinWorker(account_id=308, login_mode="im_direct")
worker._load_sec_user_id = AsyncMock(return_value="")
worker._refresh_sec_user_id = AsyncMock(
side_effect=RuntimeError("暂时无法核验 sec_user_id,请稍后重试")
)
worker._stop_for_missing_sec_user_id = AsyncMock()
with self.assertRaisesRegex(RuntimeError, "暂时无法核验"):
await worker._require_sec_user_id(
"启动托管时",
refresh_if_missing=True,
)
worker._refresh_sec_user_id.assert_awaited_once_with()
worker._stop_for_missing_sec_user_id.assert_not_awaited()
async def test_stale_cookie_forces_refresh_instead_of_using_cached_identity(self):
worker = DouyinWorker(account_id=309, login_mode="im_direct")
worker._load_sec_user_id = AsyncMock(return_value="cached-sec-user-id")
worker._sec_user_id_is_stale = AsyncMock(return_value=True)
worker._refresh_sec_user_id = AsyncMock(return_value="fresh-sec-user-id")
worker._stop_for_missing_sec_user_id = AsyncMock()
resolved = await worker._require_sec_user_id(
"启动托管时",
refresh_if_missing=True,
refresh_if_stale=True,
)
self.assertEqual(resolved, "fresh-sec-user-id")
worker._sec_user_id_is_stale.assert_awaited_once_with()
worker._refresh_sec_user_id.assert_awaited_once_with()
worker._stop_for_missing_sec_user_id.assert_not_awaited()
async def test_profile_helper_uses_uid_fallback_for_sec_user_id(self):
fetch_detail = AsyncMock(
return_value={
"uid": "123456789",
"nickname": "fallback-user",
"sec_user_id": "",
"fetched": True,
"message": "",
}
)
run_in_thread = AsyncMock(return_value=("MS4wLjABAAAA-fallback", True))
with (
patch.object(
account_profile_module,
"fetch_douyin_profile_detail",
fetch_detail,
),
patch.object(
account_profile_module.asyncio,
"to_thread",
run_in_thread,
),
):
detail = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
"cookie-json",
"test-agent",
)
self.assertEqual(detail["sec_user_id"], "MS4wLjABAAAA-fallback")
self.assertEqual(detail["sec_user_id_status"], "found")
fetch_detail.assert_awaited_once_with("cookie-json", "test-agent")
run_in_thread.assert_awaited_once_with(
account_profile_module._resolve_sec_user_id_by_uid_sync,
"cookie-json",
"test-agent",
"123456789",
)
async def test_profile_transport_failure_remains_unknown_not_missing(self):
fetch_detail = AsyncMock(
return_value={
"uid": "",
"sec_user_id": "",
"fetched": False,
"message": "profile request timed out",
}
)
run_in_thread = AsyncMock()
with (
patch.object(
account_profile_module,
"fetch_douyin_profile_detail",
fetch_detail,
),
patch.object(
account_profile_module.asyncio,
"to_thread",
run_in_thread,
),
):
detail = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
"cookie-json",
"test-agent",
)
self.assertEqual(detail["sec_user_id_status"], "unknown")
run_in_thread.assert_not_awaited()
async def test_confirmed_empty_uid_fallback_is_missing(self):
fetch_detail = AsyncMock(
return_value={
"uid": "987654321",
"sec_user_id": "",
"fetched": True,
"message": "",
}
)
run_in_thread = AsyncMock(return_value=("", True))
with (
patch.object(
account_profile_module,
"fetch_douyin_profile_detail",
fetch_detail,
),
patch.object(
account_profile_module.asyncio,
"to_thread",
run_in_thread,
),
):
detail = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
"cookie-json",
"test-agent",
)
self.assertEqual(detail["sec_user_id"], "")
self.assertEqual(detail["sec_user_id_status"], "missing")
run_in_thread.assert_awaited_once()
async def test_stop_for_missing_sec_user_id_is_offline_and_idempotent(self):
worker = DouyinWorker(account_id=310, login_mode="im_direct")
worker.is_running = True
worker.stopping = False
service = SimpleNamespace(_running=True)
worker._im_service = service
worker.update_account_status = AsyncMock()
with patch(
"rpa_engine.playwright_worker.system_logger.record"
) as record_system_log:
await worker._stop_for_missing_sec_user_id("托管运行中")
await worker._stop_for_missing_sec_user_id("重复检查")
self.assertTrue(worker.stopping)
self.assertFalse(worker.is_running)
self.assertFalse(service._running)
worker.update_account_status.assert_awaited_once()
status_call = worker.update_account_status.await_args
self.assertEqual(status_call.args[0], "offline")
self.assertIn("sec_user_id", status_call.kwargs["error_msg"])
self.assertIn("托管已自动退出", status_call.kwargs["error_msg"])
record_system_log.assert_called_once()
async def test_cookie_uid_without_valid_profile_payload_stays_unknown(self):
auth = SimpleNamespace(
cookie={},
msToken="test-ms-token",
get_uid=lambda: "cookie-only-uid",
)
payloads = [
{"status_code": 0, "data": {"status": "ok"}},
{"status_code": 0, "message": "success"},
{"status_code": 0, "data": {}},
]
responses = [SimpleNamespace(json=lambda value=value: value) for value in payloads]
with (
patch.object(
account_profile_module,
"_build_auth",
return_value=(auth, "test-agent"),
),
patch.object(
account_profile_module.requests,
"get",
side_effect=responses,
) as request_get,
patch.object(
account_profile_module,
"generate_a_bogus",
return_value="a-bogus",
),
patch.object(
account_profile_module,
"generate_webid",
return_value="web-id",
),
patch.object(
account_profile_module,
"_requests_proxies",
return_value=None,
),
):
raw_detail = account_profile_module.fetch_douyin_profile_detail_sync(
"cookie-json",
"test-agent",
)
self.assertEqual(request_get.call_count, 3)
self.assertEqual(raw_detail["uid"], "cookie-only-uid")
self.assertFalse(raw_detail["fetched"])
self.assertFalse(raw_detail["profile_response_valid"])
with patch.object(
account_profile_module,
"fetch_douyin_profile_detail",
new=AsyncMock(return_value=raw_detail),
):
checked = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
"cookie-json",
"test-agent",
)
self.assertEqual(checked["sec_user_id_status"], "unknown")
async def test_invalid_uid_fallback_payloads_stay_unknown(self):
auth = SimpleNamespace(cookie={}, msToken="test-ms-token")
cases = (
(
"nonzero-status",
{
"status_code": 10007,
"data": {
"user": {
"uid": "fallback-uid",
"sec_uid": "must-not-be-used",
}
},
},
),
(
"no-user-node",
{"status_code": 0, "data": {"status": "ok"}},
),
)
for label, payload in cases:
with self.subTest(label=label):
with (
patch.object(
account_profile_module,
"_build_auth",
return_value=(auth, "test-agent"),
),
patch.object(
account_profile_module,
"_douyin_get_json",
return_value=payload,
),
patch.object(
account_profile_module,
"generate_webid",
return_value="web-id",
),
):
resolved, request_completed = (
account_profile_module._resolve_sec_user_id_by_uid_sync(
"cookie-json",
"test-agent",
"fallback-uid",
)
)
self.assertEqual(resolved, "")
self.assertFalse(request_completed)
fetch_detail = AsyncMock(
return_value={
"uid": "fallback-uid",
"sec_user_id": "",
"fetched": True,
"message": "",
}
)
run_in_thread = AsyncMock(
return_value=(resolved, request_completed)
)
with (
patch.object(
account_profile_module,
"fetch_douyin_profile_detail",
fetch_detail,
),
patch.object(
account_profile_module.asyncio,
"to_thread",
run_in_thread,
),
):
checked = await account_profile_module.fetch_douyin_profile_detail_with_sec_user_id(
"cookie-json",
"test-agent",
)
self.assertEqual(checked["sec_user_id_status"], "unknown")
async def test_unknown_profile_sync_preserves_cached_identity_and_videos(self):
old_synced_at = object()
profile = SimpleNamespace(
sec_user_id="cached-sec-user-id",
synced_at=old_synced_at,
sync_message="previous message",
)
account = SimpleNamespace(
id=411,
user_agent="test-agent",
douyin_uid="cached-uid",
username="cached-user",
avatar_url="cached-avatar",
)
scalar_result = SimpleNamespace(scalar_one_or_none=lambda: profile)
db = SimpleNamespace(
execute=AsyncMock(return_value=scalar_result),
commit=AsyncMock(),
add=MagicMock(),
)
unknown_detail = {
"uid": "cookie-uid",
"sec_user_id": "",
"sec_user_id_status": "unknown",
"fetched": False,
"message": "profile endpoint temporarily unavailable",
}
cached_response = {
"account_id": 411,
"sec_user_id": "cached-sec-user-id",
"synced_at": "cached-time",
"videos": [{"aweme_id": "cached-video"}],
}
fetch_videos = AsyncMock()
apply_profile = AsyncMock()
load_cached = AsyncMock(return_value=dict(cached_response))
with (
patch.object(
account_profile_module,
"fetch_douyin_profile_detail_with_sec_user_id",
new=AsyncMock(return_value=unknown_detail),
),
patch.object(
account_profile_module,
"fetch_douyin_user_videos",
fetch_videos,
),
patch.object(
account_profile_module,
"apply_douyin_profile",
apply_profile,
),
patch.object(
account_profile_module,
"load_account_profile_from_db",
load_cached,
),
):
result = await account_profile_module.sync_account_profile_to_db(
db,
account,
"cookie-json",
)
self.assertEqual(profile.sec_user_id, "cached-sec-user-id")
self.assertIs(profile.synced_at, old_synced_at)
fetch_videos.assert_not_awaited()
apply_profile.assert_not_awaited()
load_cached.assert_awaited_once_with(db, account)
self.assertEqual(db.execute.await_count, 1)
self.assertNotIn("delete", str(db.execute.await_args.args[0]).lower())
self.assertEqual(result["videos"], cached_response["videos"])
self.assertEqual(
result["message"],
"profile endpoint temporarily unavailable",
)
async def test_browser_login_missing_sec_user_id_closes_browser_and_skips_im(self):
worker = DouyinWorker(account_id=305, login_mode="browser")
worker.is_running = True
events: list[str] = []
async def reject_identity(*_args, **_kwargs):
events.append("require-sec-user-id")
return False
worker._load_user_agent = AsyncMock(return_value="test-agent")
worker._probe_existing_login = AsyncMock(return_value=True)
worker._finalize_login_session = AsyncMock()
worker._require_sec_user_id = AsyncMock(side_effect=reject_identity)
worker._setup_im_network_listener = AsyncMock()
worker._navigate_to_message_center = AsyncMock()
worker._harvest_im_credentials = AsyncMock()
worker._persist_cookies = AsyncMock()
im_session = DouyinImSession(
cookies={"sessionid": "test-session"},
my_uid=30501,
)
worker._build_im_session = AsyncMock(return_value=im_session)
worker._persist_im_session = AsyncMock()
worker._run_im_direct_service = AsyncMock()
worker._close_browser_only = AsyncMock()
page = SimpleNamespace()
context = SimpleNamespace(
add_init_script=AsyncMock(),
new_page=AsyncMock(return_value=page),
)
browser = SimpleNamespace(new_context=AsyncMock(return_value=context))
playwright = SimpleNamespace()
playwright_starter = SimpleNamespace(
start=AsyncMock(return_value=playwright),
)
@asynccontextmanager
async def browser_slot(account_id: int, description: str):
self.assertEqual(account_id, 305)
self.assertEqual(description, "browser credential harvest")
events.append("browser-acquired")
try:
yield
finally:
events.append("browser-released")
controller = SimpleNamespace(browser_slot=browser_slot)
with (
patch(
"rpa_engine.playwright_worker.async_playwright",
return_value=playwright_starter,
),
patch(
"rpa_engine.playwright_worker._launch_chromium",
new=AsyncMock(return_value=browser),
),
patch(
"rpa_engine.playwright_worker.get_traffic_controller",
return_value=controller,
),
patch(
"rpa_engine.playwright_worker.validate_im_session",
new=AsyncMock(return_value=(True, "ready")),
),
):
await worker._run_browser_im_flow(
storage_state=None,
cookie_info={
"cookie_valid": False,
"has_sessionid": False,
"reason": "no saved login",
},
)
worker._finalize_login_session.assert_awaited_once_with()
worker._require_sec_user_id.assert_awaited_once()
require_call = worker._require_sec_user_id.await_args
self.assertTrue(require_call.kwargs["force_refresh"])
self.assertEqual(
events,
["browser-acquired", "browser-released", "require-sec-user-id"],
)
worker._setup_im_network_listener.assert_awaited_once_with()
worker._navigate_to_message_center.assert_awaited_once_with()
worker._harvest_im_credentials.assert_awaited_once_with(timeout=25)
worker._persist_cookies.assert_awaited_once_with()
worker._build_im_session.assert_awaited_once_with()
worker._persist_im_session.assert_not_awaited()
worker._run_im_direct_service.assert_not_awaited()
worker._close_browser_only.assert_awaited_once_with()
async def test_browser_login_with_sec_user_id_continues_to_im(self):
worker = DouyinWorker(account_id=306, login_mode="browser")
worker.is_running = True
worker._load_user_agent = AsyncMock(return_value="test-agent")
worker._probe_existing_login = AsyncMock(return_value=True)
worker._finalize_login_session = AsyncMock()
worker._require_sec_user_id = AsyncMock(return_value=True)
worker._setup_im_network_listener = AsyncMock()
worker._navigate_to_message_center = AsyncMock()
worker._harvest_im_credentials = AsyncMock()
worker._persist_cookies = AsyncMock()
im_session = DouyinImSession(
cookies={"sessionid": "test-session"},
my_uid=30601,
)
worker._build_im_session = AsyncMock(return_value=im_session)
worker._persist_im_session = AsyncMock()
worker._run_im_direct_service = AsyncMock()
worker._close_browser_only = AsyncMock()
page = SimpleNamespace()
context = SimpleNamespace(
add_init_script=AsyncMock(),
new_page=AsyncMock(return_value=page),
)
browser = SimpleNamespace(new_context=AsyncMock(return_value=context))
playwright_starter = SimpleNamespace(
start=AsyncMock(return_value=SimpleNamespace()),
)
@asynccontextmanager
async def browser_slot(_account_id: int, _description: str):
yield
controller = SimpleNamespace(browser_slot=browser_slot)
with (
patch(
"rpa_engine.playwright_worker.async_playwright",
return_value=playwright_starter,
),
patch(
"rpa_engine.playwright_worker._launch_chromium",
new=AsyncMock(return_value=browser),
),
patch(
"rpa_engine.playwright_worker.get_traffic_controller",
return_value=controller,
),
patch(
"rpa_engine.playwright_worker.validate_im_session",
new=AsyncMock(return_value=(True, "ready")),
),
):
await worker._run_browser_im_flow(
storage_state=None,
cookie_info={
"cookie_valid": False,
"has_sessionid": False,
"reason": "no saved login",
},
)
worker._require_sec_user_id.assert_awaited_once()
self.assertTrue(worker._require_sec_user_id.await_args.kwargs["force_refresh"])
worker._setup_im_network_listener.assert_awaited_once_with()
worker._navigate_to_message_center.assert_awaited_once_with()
worker._harvest_im_credentials.assert_awaited_once_with(timeout=25)
worker._build_im_session.assert_awaited_once_with()
worker._persist_im_session.assert_awaited_once_with(
im_session,
status="online",
clear_error=True,
)
worker._run_im_direct_service.assert_awaited_once_with(im_session)
worker._close_browser_only.assert_awaited()
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,284 @@
from __future__ import annotations
import asyncio
import os
import sys
import unittest
from contextlib import asynccontextmanager
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ["KEFU_DB_TYPE"] = "sqlite"
os.environ["KEFU_DATABASE_URL"] = ""
os.environ["KEFU_DB_PATH"] = str(BACKEND_DIR / "kefu.db")
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im import http_client as http_client_module
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
from rpa_engine.douyin_im.session import DouyinImSession
from rpa_engine.playwright_worker import DouyinWorker
class SendTextMessageEntryTests(unittest.IsolatedAsyncioTestCase):
def _make_client(self, account_id: int = 41) -> DouyinImHttpClient:
session = DouyinImSession(
cookies={"sessionid": "test-session"},
my_uid=10001,
conv_meta={"existing": {"ticket": "old-ticket"}},
)
return DouyinImHttpClient(session, account_id=account_id)
async def test_public_entry_submits_the_whole_send_to_outbound_queue(self):
client = self._make_client(account_id=73)
submit = AsyncMock(return_value=True)
with patch(
"rpa_engine.douyin_im.traffic_control.submit_outbound",
submit,
):
sent = await client.send_text_message(
"0:1:10001:20002",
"hello",
conversation_short_id="short-1",
)
self.assertTrue(sent)
submit.assert_awaited_once()
account_id, operation = submit.await_args.args[:2]
self.assertEqual(account_id, 73)
self.assertTrue(callable(operation))
self.assertIn("20002", submit.await_args.kwargs["description"])
async def test_bypass_path_does_not_submit_again(self):
client = self._make_client()
client._resolve_authoritative_uid = MagicMock(return_value=0)
submit = AsyncMock(return_value=True)
with patch(
"rpa_engine.douyin_im.traffic_control.submit_outbound",
submit,
):
sent = await client.send_text_message(
"0:1:10001:20002",
"hello",
_bypass_global_queue=True,
)
self.assertFalse(sent)
submit.assert_not_awaited()
client._resolve_authoritative_uid.assert_called_once()
async def test_queued_client_result_state_is_copied_to_calling_client(self):
client = self._make_client(account_id=88)
queued_meta = {
"0:1:10001:20002": {
"ticket": "fresh-ticket",
"conversation_short_id": "fresh-short-id",
}
}
queued_client = SimpleNamespace(
send_text_message=AsyncMock(return_value=False),
last_send_meta=queued_meta,
last_error="credential expired",
last_send_needs_refresh=True,
last_request_debug="response status=401",
)
queued_context = MagicMock()
queued_context.__aenter__ = AsyncMock(return_value=queued_client)
queued_context.__aexit__ = AsyncMock(return_value=None)
queued_factory = MagicMock(return_value=queued_context)
async def execute_submission(account_id, operation, description=""):
self.assertEqual(account_id, 88)
self.assertIn("20002", description)
return await operation()
submit = AsyncMock(side_effect=execute_submission)
with (
patch(
"rpa_engine.douyin_im.traffic_control.submit_outbound",
submit,
),
patch.object(
http_client_module,
"DouyinImHttpClient",
queued_factory,
),
):
sent = await client.send_text_message(
"0:1:10001:20002",
"queued hello",
conversation_short_id="short-before-send",
)
self.assertFalse(sent)
submit.assert_awaited_once()
queued_factory.assert_called_once_with(client.session, account_id=88)
queued_client.send_text_message.assert_awaited_once_with(
"0:1:10001:20002",
"queued hello",
conversation_short_id="short-before-send",
_bypass_global_queue=True,
)
self.assertEqual(client.last_send_meta, queued_meta)
self.assertIsNot(client.last_send_meta, queued_meta)
self.assertEqual(client.last_error, "credential expired")
self.assertTrue(client.last_send_needs_refresh)
self.assertEqual(client.last_request_debug, "response status=401")
class WorkerLifecycleTests(unittest.IsolatedAsyncioTestCase):
async def test_start_saves_task_and_stop_waits_until_it_is_done(self):
worker = DouyinWorker(account_id=919, login_mode="im_direct")
loop_started = asyncio.Event()
loop_finished = asyncio.Event()
never_finishes_without_cancellation = asyncio.Event()
async def controlled_run_loop() -> None:
loop_started.set()
try:
await never_finishes_without_cancellation.wait()
finally:
loop_finished.set()
worker._run_loop = controlled_run_loop
worker.update_account_status = AsyncMock()
worker.cleanup = AsyncMock()
await worker.start()
task = worker._task
self.assertIsInstance(task, asyncio.Task)
self.assertEqual(task.get_name(), "douyin-worker-919")
self.assertFalse(task.done())
await asyncio.wait_for(loop_started.wait(), timeout=0.2)
# Starting an already-running worker must retain the same owned task.
await worker.start()
self.assertIs(worker._task, task)
await asyncio.wait_for(worker.stop(), timeout=0.5)
self.assertTrue(task.done())
self.assertTrue(task.cancelled())
self.assertTrue(loop_finished.is_set())
self.assertIsNone(worker._task)
self.assertTrue(worker.stopping)
self.assertFalse(worker.is_running)
worker.update_account_status.assert_awaited_once_with("offline")
worker.cleanup.assert_awaited_once_with()
leaked = [
running
for running in asyncio.all_tasks()
if running is not asyncio.current_task()
and running.get_name() == "douyin-worker-919"
and not running.done()
]
self.assertEqual(leaked, [])
async def test_browser_flow_closes_resources_before_releasing_slot_on_error(self):
worker = DouyinWorker(account_id=920, login_mode="browser")
events: list[str] = []
# A truthy browser resource makes the flow's finally block responsible
# for cleanup even though credential preparation fails midway.
worker.page = object()
async def failing_prepare(storage_state, cookie_info):
events.append("prepare")
raise RuntimeError("credential harvest failed")
async def close_browser_only():
events.append("close")
worker.page = None
@asynccontextmanager
async def browser_slot(account_id, description):
self.assertEqual(account_id, 920)
self.assertEqual(description, "browser credential harvest")
events.append("acquire")
try:
yield
finally:
events.append("release")
controller = SimpleNamespace(browser_slot=browser_slot)
worker._prepare_browser_im_flow = AsyncMock(side_effect=failing_prepare)
worker._close_browser_only = AsyncMock(side_effect=close_browser_only)
with patch(
"rpa_engine.playwright_worker.get_traffic_controller",
return_value=controller,
):
with self.assertRaisesRegex(RuntimeError, "credential harvest failed"):
await worker._run_browser_im_flow(
storage_state={"cookies": []},
cookie_info={"has_sessionid": True},
)
self.assertEqual(events, ["acquire", "prepare", "close", "release"])
worker._prepare_browser_im_flow.assert_awaited_once_with(
{"cookies": []},
{"has_sessionid": True},
)
worker._close_browser_only.assert_awaited_once_with()
async def test_close_browser_only_closes_every_resource_and_clears_references(self):
worker = DouyinWorker(account_id=921, login_mode="browser")
events: list[str] = []
async def record(name: str, *, fail: bool = False) -> None:
events.append(name)
if fail:
raise RuntimeError(f"{name} close failed")
async def close_page() -> None:
await record("page")
async def close_context() -> None:
await record("context", fail=True)
async def close_browser() -> None:
await record("browser")
async def stop_playwright() -> None:
await record("playwright")
page = SimpleNamespace(
close=AsyncMock(side_effect=close_page),
)
context = SimpleNamespace(
# One failed close must not keep the remaining processes alive.
close=AsyncMock(side_effect=close_context),
)
browser = SimpleNamespace(
close=AsyncMock(side_effect=close_browser),
)
playwright = SimpleNamespace(
stop=AsyncMock(side_effect=stop_playwright),
)
worker.page = page
worker.context = context
worker.browser = browser
worker.playwright = playwright
await worker._close_browser_only()
self.assertEqual(events, ["page", "context", "browser", "playwright"])
page.close.assert_awaited_once_with()
context.close.assert_awaited_once_with()
browser.close.assert_awaited_once_with()
playwright.stop.assert_awaited_once_with()
self.assertIsNone(worker.page)
self.assertIsNone(worker.context)
self.assertIsNone(worker.browser)
self.assertIsNone(worker.playwright)
if __name__ == "__main__":
unittest.main()
+495
View File
@@ -0,0 +1,495 @@
import asyncio
import os
import sys
import unittest
from pathlib import Path
from unittest.mock import patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im.traffic_control import (
GlobalSendQueue,
TrafficController,
get_traffic_controller,
)
class GlobalSendQueueTests(unittest.IsolatedAsyncioTestCase):
async def _make_queue(self, interval_seconds: float = 0.0) -> GlobalSendQueue:
queue = GlobalSendQueue(interval_seconds=interval_seconds)
self.addAsyncCleanup(queue.stop)
return queue
async def _wait_for_queued_count(
self,
queue: GlobalSendQueue,
expected: int,
timeout: float = 0.5,
) -> None:
loop = asyncio.get_running_loop()
deadline = loop.time() + timeout
while loop.time() < deadline:
if (await queue.snapshot())["queued_count"] == expected:
return
await asyncio.sleep(0.001)
self.assertEqual((await queue.snapshot())["queued_count"], expected)
async def test_global_send_concurrency_is_one_across_accounts(self):
queue = await self._make_queue()
active = 0
maximum_active = 0
async def operation(index: int) -> int:
nonlocal active, maximum_active
active += 1
maximum_active = max(maximum_active, active)
await asyncio.sleep(0.015)
active -= 1
return index
tasks = [
asyncio.create_task(
queue.submit(
account_id=(index % 3) + 1,
operation=lambda index=index: operation(index),
description=f"send-{index}",
)
)
for index in range(9)
]
results = await asyncio.wait_for(asyncio.gather(*tasks), timeout=1.0)
self.assertEqual(sorted(results), list(range(9)))
self.assertEqual(maximum_active, 1)
self.assertEqual(active, 0)
async def test_same_account_keeps_fifo_order(self):
queue = await self._make_queue()
calls: list[int] = []
async def operation(index: int) -> int:
calls.append(index)
await asyncio.sleep(0)
return index
tasks = [
asyncio.create_task(
queue.submit(
account_id=101,
operation=lambda index=index: operation(index),
description=str(index),
)
)
for index in range(6)
]
results = await asyncio.wait_for(asyncio.gather(*tasks), timeout=0.5)
self.assertEqual(calls, list(range(6)))
self.assertEqual(results, list(range(6)))
async def test_busy_accounts_are_dispatched_round_robin(self):
queue = await self._make_queue()
calls: list[str] = []
first_started = asyncio.Event()
release_first = asyncio.Event()
async def first_operation() -> str:
calls.append("a1")
first_started.set()
await release_first.wait()
return "a1"
async def operation(name: str) -> str:
calls.append(name)
await asyncio.sleep(0)
return name
first = asyncio.create_task(queue.submit(1, first_operation, "a1"))
await asyncio.wait_for(first_started.wait(), timeout=0.2)
# Account 1 is deliberately noisy. Accounts 2 and 3 must each get a
# turn before account 1 is allowed to dispatch its next waiting job.
remaining_specs = [
(1, "a2"),
(1, "a3"),
(2, "b1"),
(2, "b2"),
(3, "c1"),
]
remaining = [
asyncio.create_task(
queue.submit(
account_id,
lambda name=name: operation(name),
description=name,
)
)
for account_id, name in remaining_specs
]
await self._wait_for_queued_count(queue, expected=len(remaining))
release_first.set()
results = await asyncio.wait_for(
asyncio.gather(first, *remaining),
timeout=0.5,
)
self.assertEqual(calls, ["a1", "b1", "c1", "a2", "b2", "a3"])
self.assertCountEqual(results, ["a1", "a2", "a3", "b1", "b2", "c1"])
async def test_send_starts_are_separated_by_configured_interval(self):
interval = 0.08
queue = await self._make_queue(interval_seconds=interval)
loop = asyncio.get_running_loop()
started_at: list[float] = []
async def operation() -> None:
started_at.append(loop.time())
tasks = [
asyncio.create_task(queue.submit(index + 1, operation, str(index)))
for index in range(3)
]
await asyncio.wait_for(asyncio.gather(*tasks), timeout=0.7)
self.assertEqual(len(started_at), 3)
gaps = [later - earlier for earlier, later in zip(started_at, started_at[1:])]
for gap in gaps:
# The Windows selector clock may wake one ~15.6 ms tick early.
self.assertGreaterEqual(gap, interval - 0.025)
async def test_operation_failure_does_not_block_later_job(self):
queue = await self._make_queue()
calls: list[str] = []
async def failing_operation() -> None:
calls.append("failed")
raise ValueError("expected failure")
async def following_operation() -> str:
calls.append("continued")
return "ok"
failed = asyncio.create_task(queue.submit(1, failing_operation, "failed"))
continued = asyncio.create_task(queue.submit(2, following_operation, "continued"))
results = await asyncio.wait_for(
asyncio.gather(failed, continued, return_exceptions=True),
timeout=0.2,
)
self.assertIsInstance(results[0], ValueError)
self.assertEqual(str(results[0]), "expected failure")
self.assertEqual(results[1], "ok")
self.assertEqual(calls, ["failed", "continued"])
async def test_cancelling_waiting_submit_prevents_operation(self):
queue = await self._make_queue()
first_started = asyncio.Event()
release_first = asyncio.Event()
cancelled_operation_called = asyncio.Event()
async def blocking_operation() -> None:
first_started.set()
await release_first.wait()
async def must_not_run() -> None:
cancelled_operation_called.set()
blocker = asyncio.create_task(queue.submit(1, blocking_operation, "blocker"))
await asyncio.wait_for(first_started.wait(), timeout=0.2)
waiting = asyncio.create_task(queue.submit(2, must_not_run, "cancelled"))
await self._wait_for_queued_count(queue, expected=1)
waiting.cancel()
with self.assertRaises(asyncio.CancelledError):
await waiting
release_first.set()
await asyncio.wait_for(blocker, timeout=0.2)
await asyncio.sleep(0.02)
self.assertFalse(cancelled_operation_called.is_set())
self.assertEqual((await queue.snapshot())["pending_count"], 0)
async def test_cancelling_active_submit_waits_for_real_result_without_overlap(self):
queue = await self._make_queue()
active = 0
maximum_active = 0
first_started = asyncio.Event()
release_first = asyncio.Event()
second_started = asyncio.Event()
async def first_operation() -> str:
nonlocal active, maximum_active
active += 1
maximum_active = max(maximum_active, active)
first_started.set()
await release_first.wait()
active -= 1
return "delivered"
async def second_operation() -> str:
nonlocal active, maximum_active
active += 1
maximum_active = max(maximum_active, active)
second_started.set()
await asyncio.sleep(0)
active -= 1
return "next"
first = asyncio.create_task(queue.submit(1, first_operation, "active"))
await asyncio.wait_for(first_started.wait(), timeout=0.2)
second = asyncio.create_task(queue.submit(2, second_operation, "next"))
await self._wait_for_queued_count(queue, expected=1)
first.cancel()
await asyncio.sleep(0.02)
# Once the network operation has begun, cancelling its caller must not
# release the single lane or report a false cancellation to the caller.
self.assertFalse(first.done())
self.assertFalse(second_started.is_set())
self.assertEqual(active, 1)
release_first.set()
self.assertEqual(await asyncio.wait_for(first, timeout=0.2), "delivered")
self.assertEqual(await asyncio.wait_for(second, timeout=0.2), "next")
self.assertTrue(second_started.is_set())
self.assertEqual(maximum_active, 1)
self.assertEqual(active, 0)
async def test_cancel_account_removes_waiting_jobs_without_running_them(self):
queue = await self._make_queue()
blocker_started = asyncio.Event()
release_blocker = asyncio.Event()
removed_calls: list[str] = []
async def blocking_operation() -> None:
blocker_started.set()
await release_blocker.wait()
async def removed_operation(name: str) -> None:
removed_calls.append(name)
blocker = asyncio.create_task(queue.submit(1, blocking_operation, "blocker"))
await asyncio.wait_for(blocker_started.wait(), timeout=0.2)
removed = [
asyncio.create_task(
queue.submit(
77,
lambda name=name: removed_operation(name),
name,
)
)
for name in ("waiting-1", "waiting-2")
]
await self._wait_for_queued_count(queue, expected=2)
self.assertEqual(await queue.cancel_account(77), 2)
results = await asyncio.wait_for(
asyncio.gather(*removed, return_exceptions=True),
timeout=0.1,
)
snapshot = await queue.snapshot()
self.assertTrue(all(isinstance(result, asyncio.CancelledError) for result in results))
self.assertEqual(removed_calls, [])
self.assertNotIn(77, snapshot["per_account"])
self.assertEqual(snapshot["queued_count"], 0)
self.assertEqual(snapshot["active_account_id"], 1)
release_blocker.set()
await asyncio.wait_for(blocker, timeout=0.2)
await asyncio.sleep(0)
self.assertEqual((await queue.snapshot())["pending_count"], 0)
async def test_cancel_account_releases_active_caller_but_drains_lane(self):
queue = await self._make_queue()
active = 0
maximum_active = 0
active_started = asyncio.Event()
release_active = asyncio.Event()
active_drained = asyncio.Event()
next_started = asyncio.Event()
calls: list[str] = []
async def active_operation() -> str:
nonlocal active, maximum_active
active += 1
maximum_active = max(maximum_active, active)
calls.append("active-start")
active_started.set()
await release_active.wait()
calls.append("active-drained")
active -= 1
active_drained.set()
return "discarded-result"
async def next_operation() -> str:
nonlocal active, maximum_active
active += 1
maximum_active = max(maximum_active, active)
calls.append("next")
next_started.set()
await asyncio.sleep(0)
active -= 1
return "next-result"
active_submit = asyncio.create_task(
queue.submit(88, active_operation, "active-account")
)
await asyncio.wait_for(active_started.wait(), timeout=0.2)
next_submit = asyncio.create_task(queue.submit(89, next_operation, "other-account"))
await self._wait_for_queued_count(queue, expected=1)
self.assertEqual(await queue.cancel_account(88), 1)
cancelled_result = await asyncio.wait_for(
asyncio.gather(active_submit, return_exceptions=True),
timeout=0.1,
)
self.assertIsInstance(cancelled_result[0], asyncio.CancelledError)
self.assertTrue(active_submit.cancelled())
self.assertFalse(active_drained.is_set())
self.assertFalse(next_started.is_set())
self.assertEqual(active, 1)
# The cancelled account's network operation still owns the lane until
# it actually drains; only then can a different account begin.
release_active.set()
self.assertEqual(
await asyncio.wait_for(next_submit, timeout=0.2),
"next-result",
)
self.assertTrue(active_drained.is_set())
self.assertEqual(calls, ["active-start", "active-drained", "next"])
self.assertEqual(maximum_active, 1)
self.assertEqual(active, 0)
# A fresh submission proves the dispatcher remains usable after the
# cancelled active job and its queued successor have both completed.
async def recovered_operation() -> str:
calls.append("recovered")
return "recovered-result"
self.assertEqual(
await asyncio.wait_for(
queue.submit(90, recovered_operation, "recovered"),
timeout=0.2,
),
"recovered-result",
)
self.assertEqual(calls[-1], "recovered")
self.assertEqual((await queue.snapshot())["pending_count"], 0)
class BackgroundTrafficLimitTests(unittest.IsolatedAsyncioTestCase):
async def test_background_slot_respects_configured_concurrency_limit(self):
with patch.dict(
os.environ,
{"KEFU_BACKGROUND_NETWORK_CONCURRENCY": "2"},
):
controller = TrafficController()
self.addAsyncCleanup(controller.stop)
active = 0
maximum_active = 0
entered = 0
all_finished = asyncio.Event()
async def request(account_id: int) -> None:
nonlocal active, maximum_active, entered
async with controller.background_slot(account_id, "poll"):
active += 1
entered += 1
maximum_active = max(maximum_active, active)
await asyncio.sleep(0.02)
active -= 1
if entered == 8 and active == 0:
all_finished.set()
tasks = [asyncio.create_task(request(index + 1)) for index in range(8)]
await asyncio.wait_for(asyncio.gather(*tasks), timeout=0.5)
self.assertEqual(entered, 8)
self.assertEqual(maximum_active, 2)
self.assertEqual(active, 0)
self.assertEqual(controller.background_active, 0)
self.assertEqual(controller.background_waiting, 0)
async def test_nested_background_slot_in_same_task_is_reentrant(self):
with patch.dict(
os.environ,
{"KEFU_BACKGROUND_NETWORK_CONCURRENCY": "1"},
):
controller = TrafficController()
self.addAsyncCleanup(controller.stop)
tasks_seen: list[asyncio.Task] = []
async def nested_request() -> None:
tasks_seen.append(asyncio.current_task())
async with controller.background_slot(1, "outer"):
self.assertEqual(controller.background_active, 1)
self.assertEqual(controller.background_waiting, 0)
self.assertEqual(controller._background._value, 0)
tasks_seen.append(asyncio.current_task())
async with controller.background_slot(1, "inner"):
tasks_seen.append(asyncio.current_task())
self.assertEqual(controller.background_active, 1)
self.assertEqual(controller.background_waiting, 0)
self.assertEqual(controller._background._value, 0)
self.assertEqual(controller.background_active, 1)
self.assertEqual(controller._background._value, 0)
self.assertEqual(controller.background_active, 0)
self.assertEqual(controller.background_waiting, 0)
self.assertEqual(controller._background._value, 1)
await asyncio.wait_for(nested_request(), timeout=0.2)
self.assertEqual(len(tasks_seen), 3)
self.assertTrue(all(task is tasks_seen[0] for task in tasks_seen))
class TrafficControllerLoopIsolationTests(unittest.TestCase):
def test_get_traffic_controller_does_not_reuse_asyncio_primitives(self):
async def capture_controller_state():
controller = get_traffic_controller()
return {
"loop": asyncio.get_running_loop(),
"controller": controller,
"send_lock": controller.send_queue._state_lock,
"send_wake": controller.send_queue._wake,
"background": controller._background,
"browser": controller._browser,
}
# IsolatedAsyncioTestCase uses a fresh asyncio.Runner per test. Keep
# both closed loop objects alive so the WeakKeyDictionary is exercised
# with two distinct keys rather than relying on garbage collection.
with asyncio.Runner() as first_runner:
first = first_runner.run(capture_controller_state())
with asyncio.Runner() as second_runner:
second = second_runner.run(capture_controller_state())
for key in (
"loop",
"controller",
"send_lock",
"send_wake",
"background",
"browser",
):
self.assertIsNot(first[key], second[key], key)
if __name__ == "__main__":
unittest.main()
View File

Some files were not shown because too many files have changed in this diff Show More