更新
This commit is contained in:
Your Name
@@ -0,0 +1,3 @@
|
||||
from .service import DouyinImService
|
||||
|
||||
__all__ = ["DouyinImService"]
|
||||
@@ -0,0 +1,163 @@
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
from .dy_util import (
|
||||
trans_cookies,
|
||||
generate_msToken,
|
||||
generate_a_bogus,
|
||||
splice_url,
|
||||
generate_webid,
|
||||
normalize_client_cert,
|
||||
resolve_proto_device_id,
|
||||
DEFAULT_USER_AGENT,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("douyin_im.auth")
|
||||
|
||||
|
||||
def _parse_storage_json(raw) -> dict | None:
|
||||
"""Parse localStorage JSON (supports nested data wrapper)."""
|
||||
if not raw:
|
||||
return None
|
||||
if isinstance(raw, dict):
|
||||
data = raw
|
||||
else:
|
||||
text = str(raw).strip()
|
||||
data = None
|
||||
for _ in range(4):
|
||||
try:
|
||||
parsed = json.loads(text)
|
||||
except Exception:
|
||||
break
|
||||
if isinstance(parsed, str):
|
||||
text = parsed
|
||||
continue
|
||||
if isinstance(parsed, dict):
|
||||
data = parsed
|
||||
break
|
||||
break
|
||||
if not data:
|
||||
return None
|
||||
|
||||
inner = data.get("data")
|
||||
if isinstance(inner, str):
|
||||
try:
|
||||
inner = json.loads(inner)
|
||||
except Exception:
|
||||
inner = None
|
||||
if isinstance(inner, dict):
|
||||
return inner
|
||||
return data
|
||||
|
||||
|
||||
class DouyinAuth:
|
||||
def __init__(self):
|
||||
self.cookie = None
|
||||
self.cookie_str = None
|
||||
self.private_key = None
|
||||
self.ticket = None
|
||||
self.ts_sign = None
|
||||
self.client_cert = None
|
||||
self.ree_public_key = None
|
||||
self.uid = None
|
||||
self.msToken = None
|
||||
self.web_id = None
|
||||
|
||||
def perepare_auth(self, cookieStr: str, web_protect_: str = "", keys_: str = ""):
|
||||
self.cookie = trans_cookies(cookieStr)
|
||||
self.cookie_str = cookieStr
|
||||
self.msToken = self.cookie["msToken"] if "msToken" in self.cookie else generate_msToken()
|
||||
self.cookie["msToken"] = self.msToken
|
||||
self.cookie_str = "; ".join([f"{k}={v}" for k, v in self.cookie.items()])
|
||||
web_data = _parse_storage_json(web_protect_)
|
||||
if web_data:
|
||||
try:
|
||||
self.ticket = web_data.get("ticket") or ""
|
||||
self.ts_sign = web_data.get("ts_sign") or ""
|
||||
self.client_cert = web_data.get("client_cert") or ""
|
||||
except Exception as e:
|
||||
logger.debug(f"web_protect parse failed: {e}")
|
||||
|
||||
keys_data = _parse_storage_json(keys_)
|
||||
if keys_data:
|
||||
try:
|
||||
self.private_key = keys_data.get("ec_privateKey") or keys_data.get("privateKey") or ""
|
||||
if self.private_key:
|
||||
self.ree_public_key = base64.b64encode(self.private_key.encode()).decode()
|
||||
except Exception as e:
|
||||
logger.debug(f"keys parse failed: {e}")
|
||||
|
||||
def is_sign_ready(self) -> bool:
|
||||
return bool(
|
||||
self.private_key
|
||||
and self.ticket
|
||||
and self.ts_sign
|
||||
and self.client_cert
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_im_session(cls, session) -> "DouyinAuth":
|
||||
"""从 DouyinImSession 构建 HTTP 发送/拉会话用的签名上下文。
|
||||
|
||||
注意：frontier WS 的 sdk_cert/ts_sign 只用于长连接，不能覆盖 web_protect，
|
||||
否则 bd-ticket-guard 与 protobuf 签名会与 ticket 失配 -> 7911。
|
||||
"""
|
||||
auth = cls()
|
||||
auth.perepare_auth(
|
||||
session.cookie_header(),
|
||||
session.web_protect_str,
|
||||
session.keys_str,
|
||||
)
|
||||
auth.web_id = session.web_id or session.device_id or None
|
||||
auth.user_agent = session.user_agent or DEFAULT_USER_AGENT
|
||||
auth.device_id = resolve_proto_device_id(
|
||||
session.device_id, session.web_id, session.my_uid
|
||||
)
|
||||
# web_protect 缺 client_cert 时，才用 frontier 抓包证书兜底（不覆盖 ts_sign）
|
||||
if not auth.client_cert and getattr(session, "sdk_cert", ""):
|
||||
auth.client_cert = normalize_client_cert(session.sdk_cert)
|
||||
elif auth.client_cert:
|
||||
auth.client_cert = normalize_client_cert(auth.client_cert)
|
||||
return auth
|
||||
|
||||
def get_uid(self):
|
||||
if self.uid is None:
|
||||
# 优先从 cookie 尝试提取，否则请求接口
|
||||
for k in ("uid_tt", "uid_tt_ss"):
|
||||
if self.cookie and self.cookie.get(k):
|
||||
try:
|
||||
self.uid = int(self.cookie.get(k))
|
||||
return self.uid
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
self.uid = self.query_my_uid()
|
||||
except Exception:
|
||||
pass
|
||||
return self.uid
|
||||
|
||||
def query_my_uid(self) -> int:
|
||||
url = 'https://www.douyin.com/aweme/v1/web/query/user/'
|
||||
headers = {
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"publish_video_strategy_type": "2",
|
||||
"verifyFp": self.cookie.get('s_v_web_id', ''),
|
||||
"fp": self.cookie.get('s_v_web_id', ''),
|
||||
"webid": generate_webid(self, "https://www.douyin.com/"),
|
||||
"msToken": self.msToken
|
||||
}
|
||||
query = splice_url(params)
|
||||
abogus = generate_a_bogus(query, user_agent=DEFAULT_USER_AGENT)
|
||||
params['a_bogus'] = abogus
|
||||
|
||||
resp = requests.get(url, params=params, headers=headers, cookies=self.cookie, verify=False, timeout=10)
|
||||
resp_json = resp.json()
|
||||
return int(resp_json['user_uid'])
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Conversation id helpers for Douyin IM."""
|
||||
|
||||
|
||||
def parse_conversation_parts(conversation_id: str) -> tuple[int, int] | None:
|
||||
parts = (conversation_id or "").split(":")
|
||||
if len(parts) >= 4 and parts[0] == "0" and parts[1] == "1":
|
||||
try:
|
||||
return int(parts[2]), int(parts[3])
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def resolve_peer_uid(conversation_id: str, my_uid: int) -> int | None:
|
||||
"""Resolve peer user id from conversation id or bare numeric id."""
|
||||
raw = (conversation_id or "").strip()
|
||||
if not raw:
|
||||
return None
|
||||
|
||||
parts = parse_conversation_parts(raw)
|
||||
if parts:
|
||||
uid1, uid2 = parts
|
||||
if my_uid and uid1 == my_uid:
|
||||
return uid2
|
||||
if my_uid and uid2 == my_uid:
|
||||
return uid1
|
||||
# 0:1:{my}:{peer} — 若 my_uid 与首段不一致，仍取末段为对方
|
||||
return uid2
|
||||
|
||||
if raw.isdigit():
|
||||
peer = int(raw)
|
||||
if my_uid and peer == my_uid:
|
||||
return None
|
||||
return peer
|
||||
return None
|
||||
|
||||
|
||||
def build_conversation_id(my_uid: int, peer_uid: int) -> str:
|
||||
return f"0:1:{int(my_uid)}:{int(peer_uid)}"
|
||||
|
||||
|
||||
def normalize_conversation_id(conversation_id: str, my_uid: int) -> str:
|
||||
"""Ensure conversation id uses current account uid as first participant."""
|
||||
peer_uid = resolve_peer_uid(conversation_id, my_uid)
|
||||
if peer_uid and my_uid:
|
||||
return build_conversation_id(my_uid, peer_uid)
|
||||
return (conversation_id or "").strip()
|
||||
@@ -0,0 +1,205 @@
|
||||
import hashlib
|
||||
import re
|
||||
import time
|
||||
import json
|
||||
import random
|
||||
import base64
|
||||
import urllib.parse
|
||||
from os import path
|
||||
import subprocess
|
||||
original_popen = subprocess.Popen
|
||||
def patched_popen(*args, **kwargs):
|
||||
if kwargs.get('universal_newlines') or kwargs.get('text'):
|
||||
if 'encoding' not in kwargs:
|
||||
kwargs['encoding'] = 'utf-8'
|
||||
return original_popen(*args, **kwargs)
|
||||
subprocess.Popen = patched_popen
|
||||
|
||||
import execjs
|
||||
import requests
|
||||
|
||||
basedir = path.dirname(__file__)
|
||||
static_dir = path.join(basedir, 'static')
|
||||
node_modules = path.join(static_dir, 'node_modules')
|
||||
|
||||
# 全局唯一 User-Agent：a_bogus 签名、HTTP 请求头、protobuf body、webid 采集等
|
||||
# 必须全部使用同一个 UA，否则抖音服务端重算 a_bogus 时与请求头 UA 不一致 -> 7911。
|
||||
# 该值需与浏览器登录上下文（playwright new_context user_agent）保持一致。
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
# 动态编译 JS
|
||||
dy_path = path.join(static_dir, 'dy_ab.js')
|
||||
dy_js = execjs.compile(open(dy_path, 'r', encoding='utf-8').read(), cwd=node_modules)
|
||||
|
||||
sign_path = path.join(static_dir, 'dy_live_sign.js')
|
||||
sign_js = execjs.compile(open(sign_path, 'r', encoding='utf-8').read(), cwd=node_modules)
|
||||
|
||||
login_path = path.join(static_dir, 'login.js')
|
||||
login_js = execjs.compile(open(login_path, 'r', encoding='utf-8').read(), cwd=node_modules)
|
||||
|
||||
|
||||
def generateSecretPhoneNum(phone):
|
||||
return login_js.call('generateSecretPhoneNum', phone)
|
||||
|
||||
|
||||
def generateSecretCode(phone, code):
|
||||
return login_js.call('generateSecretCode', phone, code)
|
||||
|
||||
|
||||
def trans_cookies(cookies_str):
|
||||
cookies = {}
|
||||
for i in cookies_str.split("; "):
|
||||
try:
|
||||
parts = i.split('=')
|
||||
key = parts[0].strip()
|
||||
val = '='.join(parts[1:]).strip()
|
||||
# 防御性清洗：过滤掉因为误粘贴 fetch 等包含非法字符或换行的 Cookie 键
|
||||
if not key or any(c in key for c in "()[]{}'\"\n \t\\"):
|
||||
continue
|
||||
cookies[key] = val
|
||||
except:
|
||||
continue
|
||||
return cookies
|
||||
|
||||
|
||||
def generate_req_sign(e, priK):
|
||||
"""私信传 obj，其他的拼接"""
|
||||
return dy_js.call('get_req_sign', e, priK)
|
||||
|
||||
|
||||
def generate_a_bogus(query, data="", user_agent=None):
|
||||
"""query, data 都是拼接字符串。
|
||||
|
||||
user_agent 必须与实际发出请求所用的 User-Agent 完全一致（见 DEFAULT_USER_AGENT），
|
||||
否则抖音服务端用请求头 UA 重算 a_bogus 会对不上，导致 7911 安全校验失败。
|
||||
"""
|
||||
return dy_js.call('get_ab', query, data, user_agent or DEFAULT_USER_AGENT)
|
||||
|
||||
|
||||
def generate_signature(room_id, user_unique_id):
|
||||
raw_string = f"live_id=1,aid=6383,version_code=180800,webcast_sdk_version=1.0.15,room_id={room_id},sub_room_id=,sub_channel_id=,did_rule=3,user_unique_id={user_unique_id},device_platform=web,device_type=,ac=,identity=audience"
|
||||
x_ms_stub = hashlib.md5(raw_string.encode("utf-8")).hexdigest()
|
||||
result = sign_js.call("get_signature", x_ms_stub)
|
||||
return result.get("X-Bogus")
|
||||
|
||||
|
||||
def generate_ree_key(prik):
|
||||
"""传递私钥"""
|
||||
return dy_js.call('get_ree_key', prik)
|
||||
|
||||
|
||||
def generate_bd_ticket_client_data(api, ticket, ts_sign, priK):
|
||||
"""传递 query, ticket, ts_sign, priK"""
|
||||
timestamp = int(time.time())
|
||||
res_sign = f"ticket={ticket}&path={api}×tamp={timestamp}"
|
||||
p = {
|
||||
'ts_sign': ts_sign,
|
||||
'req_content': 'ticket,path,timestamp',
|
||||
'req_sign': generate_req_sign(res_sign, priK),
|
||||
'timestamp': timestamp,
|
||||
}
|
||||
p = json.dumps(p, ensure_ascii=False, separators=(',', ':'))
|
||||
return base64.urlsafe_b64encode(p.encode('utf-8')).decode('utf-8')
|
||||
|
||||
|
||||
def generate_msToken(randomlength=107):
|
||||
random_str = ''
|
||||
base_str = 'ABCDEFGHIGKLMNOPQRSTUVWXYZabcdefghigklmnopqrstuvwxyz0123456789='
|
||||
length = len(base_str) - 1
|
||||
for _ in range(randomlength):
|
||||
random_str += base_str[random.randint(0, length)]
|
||||
return random_str
|
||||
|
||||
|
||||
def generate_fake_webid(random_length=19):
|
||||
random_str = ''
|
||||
base_str = '0123456789'
|
||||
length = len(base_str) - 1
|
||||
for _ in range(random_length):
|
||||
random_str += base_str[random.randint(0, length)]
|
||||
return random_str
|
||||
|
||||
|
||||
def generate_webid(auth=None, url=""):
|
||||
# 优先用已采集到的 web_id（避免每次发送都发起一次阻塞的 HTTP 请求，导致事件循环卡顿）
|
||||
cached = getattr(auth, "web_id", None) if auth is not None else None
|
||||
if cached:
|
||||
return str(cached)
|
||||
if url == "":
|
||||
url = "https://www.douyin.com/discover?modal_id=7376449060384935209"
|
||||
try:
|
||||
from .auth import DouyinAuth
|
||||
headers = {
|
||||
"User-Agent": DEFAULT_USER_AGENT,
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"upgrade-insecure-requests": "1"
|
||||
}
|
||||
if auth and auth.cookie_str:
|
||||
headers['cookie'] = auth.cookie_str
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
proxies = requests_proxies()
|
||||
except Exception:
|
||||
proxies = None
|
||||
response = requests.get(
|
||||
url, headers=headers, verify=False, timeout=10, proxies=proxies
|
||||
)
|
||||
res_text = response.text
|
||||
user_unique_id = re.findall(r'\\"user_unique_id\\":\\"(.*?)\\"', res_text)[0]
|
||||
# 把发现的 web_id 回写到 auth 上，避免同一次同步里反复发起阻塞的 HTTP 请求
|
||||
if auth is not None and user_unique_id:
|
||||
try:
|
||||
auth.web_id = user_unique_id
|
||||
except Exception:
|
||||
pass
|
||||
return user_unique_id
|
||||
except Exception:
|
||||
# 失败时同样缓存一个伪 web_id，避免后续调用重复走 10s 超时的网络请求
|
||||
fake = generate_fake_webid()
|
||||
if auth is not None:
|
||||
try:
|
||||
if not getattr(auth, "web_id", None):
|
||||
auth.web_id = fake
|
||||
except Exception:
|
||||
pass
|
||||
return fake
|
||||
|
||||
|
||||
def generate_millisecond():
|
||||
return int(round(time.time() * 1000))
|
||||
|
||||
|
||||
def normalize_client_cert(cert: str) -> str:
|
||||
"""统一 client_cert / sdk_cert 格式为「base64 证书体」。
|
||||
|
||||
web_protect.client_cert 与 frontier WS 的 sdk_cert 通常已是 base64(PEM)；
|
||||
若误把 PEM 原文或 frontier 证书二次 base64，会导致 7911。
|
||||
"""
|
||||
cert = (cert or "").strip()
|
||||
if not cert:
|
||||
return ""
|
||||
if cert.startswith("-----BEGIN"):
|
||||
return base64.b64encode(cert.encode("utf-8")).decode("utf-8")
|
||||
return cert
|
||||
|
||||
|
||||
def resolve_proto_device_id(device_id: str = "", web_id: str = "", my_uid: int = 0) -> str:
|
||||
"""protobuf / frontier 更倾向使用数字 device_id（通常等于 my_uid）。"""
|
||||
for candidate in (device_id, web_id, str(my_uid or "")):
|
||||
c = str(candidate or "").strip()
|
||||
if c.isdigit():
|
||||
return c
|
||||
return str(device_id or web_id or "0")
|
||||
|
||||
|
||||
def splice_url(params):
|
||||
splice_url_str = ''
|
||||
for key, value in params.items():
|
||||
if value is None:
|
||||
value = ''
|
||||
splice_url_str += key + '=' + urllib.parse.quote(str(value)) + '&'
|
||||
return splice_url_str[:-1]
|
||||
@@ -0,0 +1,136 @@
|
||||
"""抖音标准表情（评论区 emoji）名称 -> 图片 URL 映射。
|
||||
|
||||
抖音文字表情如 [酷拽]/[微笑] 通过 WS 以 message_type=7 的纯文本下发，
|
||||
content 形如 {"text":"[酷拽]","aweType":700}，不带图片地址；
|
||||
浏览器端靠本地表情表把 [name] 渲染成小图。这里拉取官方表情列表接口，
|
||||
建立 name->url 映射，收到文字表情时补成可显示的贴纸。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger("douyin_im.emoji")
|
||||
|
||||
_EMOJI_MAP: dict[str, str] = {}
|
||||
_FETCHED_AT: float = 0.0
|
||||
_TTL = 6 * 3600 # 6 小时刷新一次
|
||||
_lock = threading.Lock()
|
||||
_fetch_lock = threading.Lock()
|
||||
_LAST_FETCH_ATTEMPT: float = 0.0
|
||||
_FAILURE_RETRY_SECONDS = 60.0
|
||||
|
||||
_BRACKET_RE = re.compile(r"^\[[^\[\]]{1,24}\]$")
|
||||
|
||||
|
||||
def has_emoji_map() -> bool:
|
||||
return bool(_EMOJI_MAP)
|
||||
|
||||
|
||||
def is_fresh() -> bool:
|
||||
return bool(_EMOJI_MAP) and (time.time() - _FETCHED_AT) < _TTL
|
||||
|
||||
|
||||
def set_emoji_map(mapping: dict[str, str]) -> None:
|
||||
global _EMOJI_MAP, _FETCHED_AT
|
||||
if mapping:
|
||||
with _lock:
|
||||
_EMOJI_MAP = dict(mapping)
|
||||
_FETCHED_AT = time.time()
|
||||
|
||||
|
||||
def lookup_emoji_url(name: str) -> str:
|
||||
"""name 可带或不带中括号，返回标准表情图片 URL（无则空串）。"""
|
||||
if not name:
|
||||
return ""
|
||||
key = name.strip()
|
||||
if not key:
|
||||
return ""
|
||||
if not key.startswith("["):
|
||||
key = f"[{key}]"
|
||||
return _EMOJI_MAP.get(key, "")
|
||||
|
||||
|
||||
def looks_like_emoji_token(text: str) -> bool:
|
||||
return bool(_BRACKET_RE.match((text or "").strip()))
|
||||
|
||||
|
||||
def fetch_emoji_map(session) -> dict[str, str]:
|
||||
"""用账号会话拉取官方表情列表，返回 {display_name: url}。失败返回 {}。"""
|
||||
try:
|
||||
import requests
|
||||
|
||||
from .auth import DouyinAuth
|
||||
from .dy_util import generate_a_bogus, generate_msToken, splice_url
|
||||
|
||||
auth = DouyinAuth.from_im_session(session)
|
||||
ua = session.user_agent
|
||||
s_v_web_id = session.cookies.get("s_v_web_id", "") if session.cookies else ""
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"pc_client_type": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"cookie_enabled": "true",
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Mozilla",
|
||||
"browser_online": "true",
|
||||
"verifyFp": s_v_web_id,
|
||||
"fp": s_v_web_id,
|
||||
"webid": session.web_id or session.device_id or "",
|
||||
"msToken": generate_msToken(),
|
||||
}
|
||||
params["a_bogus"] = generate_a_bogus(splice_url(params), user_agent=ua)
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Cookie": session.cookie_header(),
|
||||
}
|
||||
resp = requests.get(
|
||||
"https://www.douyin.com/aweme/v1/web/emoji/list",
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie if getattr(auth, "cookie", None) else None,
|
||||
verify=False,
|
||||
timeout=20,
|
||||
)
|
||||
data = resp.json()
|
||||
mapping: dict[str, str] = {}
|
||||
for item in data.get("emoji_list") or []:
|
||||
name = item.get("display_name")
|
||||
urls = (item.get("emoji_url") or {}).get("url_list") or []
|
||||
if name and urls:
|
||||
mapping[name] = urls[0]
|
||||
logger.info("Fetched %d douyin emoji", len(mapping))
|
||||
return mapping
|
||||
except Exception as e:
|
||||
logger.warning("fetch_emoji_map failed: %s", e)
|
||||
return {}
|
||||
|
||||
|
||||
def ensure_emoji_map(session) -> None:
|
||||
"""若缓存为空/过期则拉取（同步阻塞，调用方建议放线程）。"""
|
||||
global _LAST_FETCH_ATTEMPT
|
||||
if is_fresh():
|
||||
return
|
||||
# Batch-started accounts used to all observe an empty cache and fetch the
|
||||
# same emoji list concurrently. Keep the network request itself inside a
|
||||
# separate single-flight lock (set_emoji_map uses _lock).
|
||||
with _fetch_lock:
|
||||
if is_fresh():
|
||||
return
|
||||
now = time.time()
|
||||
if now - _LAST_FETCH_ATTEMPT < _FAILURE_RETRY_SECONDS:
|
||||
return
|
||||
_LAST_FETCH_ATTEMPT = now
|
||||
mapping = fetch_emoji_map(session)
|
||||
if mapping:
|
||||
set_emoji_map(mapping)
|
||||
@@ -0,0 +1,183 @@
|
||||
"""抖音网页版「粉丝列表」拉取，用于检测新粉丝（关注欢迎语功能）。
|
||||
|
||||
复用与 peer_profile / account_profile 相同的 a_bogus + msToken + cookie 签名方式，
|
||||
调用 https://www.douyin.com/aweme/v1/web/user/follower/list/ 拉取本账号最近的粉丝。
|
||||
|
||||
返回的每个粉丝含：uid / sec_uid / nickname / follow_status / follower_status。
|
||||
其中 follow_status 表示「我」与对方的关系：0=未关注 1=我已关注 2=互相关注（互关）。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from .dy_util import (
|
||||
DEFAULT_USER_AGENT,
|
||||
generate_a_bogus,
|
||||
generate_msToken,
|
||||
generate_webid,
|
||||
splice_url,
|
||||
)
|
||||
from .auth import DouyinAuth
|
||||
|
||||
logger = logging.getLogger("douyin_im.follower_poll")
|
||||
|
||||
FOLLOWER_LIST_URL = "https://www.douyin.com/aweme/v1/web/user/follower/list/"
|
||||
|
||||
|
||||
def _requests_proxies() -> dict | None:
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
|
||||
return requests_proxies()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _to_int(value: Any) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _extract_followers(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
raw = data.get("followers")
|
||||
if not isinstance(raw, list):
|
||||
return []
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
uid = str(item.get("uid") or item.get("user_id") or "").strip()
|
||||
if not uid:
|
||||
continue
|
||||
out.append(
|
||||
{
|
||||
"uid": uid,
|
||||
"sec_uid": str(item.get("sec_uid") or item.get("sec_user_id") or "").strip(),
|
||||
"nickname": str(item.get("nickname") or item.get("nick_name") or "").strip(),
|
||||
# follow_status：我对对方的关系（2=互关）；follower_status：对方对我的关系
|
||||
"follow_status": _to_int(item.get("follow_status")),
|
||||
"follower_status": _to_int(item.get("follower_status")),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def fetch_recent_followers_sync(
|
||||
session,
|
||||
sec_user_id: str,
|
||||
count: int = 20,
|
||||
max_time: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""同步拉取最近粉丝（第一页）。失败返回 []，并在日志里写明原因。"""
|
||||
import requests
|
||||
|
||||
sec_user_id = (sec_user_id or "").strip()
|
||||
if not sec_user_id:
|
||||
logger.warning("fetch followers skipped: 缺少本账号 sec_user_id")
|
||||
return []
|
||||
|
||||
try:
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||||
except Exception as exc:
|
||||
logger.warning("fetch followers: build auth failed: %s", exc)
|
||||
return []
|
||||
|
||||
ua = session.user_agent or DEFAULT_USER_AGENT
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"sec_user_id": sec_user_id,
|
||||
"count": str(count),
|
||||
"max_time": str(max_time),
|
||||
"min_time": "0",
|
||||
"offset": "0",
|
||||
"source_type": "1",
|
||||
"gps_access": "0",
|
||||
"address_book_access": "0",
|
||||
"is_top": "1",
|
||||
"update_version_code": "170400",
|
||||
"pc_client_type": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"cookie_enabled": "true",
|
||||
"screen_width": "1536",
|
||||
"screen_height": "960",
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Chrome",
|
||||
"browser_version": "120.0.0.0",
|
||||
"browser_online": "true",
|
||||
"os_name": "Windows",
|
||||
"os_version": "10",
|
||||
"platform": "PC",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
}
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
try:
|
||||
resp = requests.get(
|
||||
FOLLOWER_LIST_URL,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
timeout=15,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
try:
|
||||
data = resp.json()
|
||||
except Exception:
|
||||
snippet = (resp.text or "")[:200].replace("\n", " ")
|
||||
logger.warning(
|
||||
"fetch followers: 非 JSON 响应 (HTTP %s): %s", resp.status_code, snippet
|
||||
)
|
||||
return []
|
||||
if not isinstance(data, dict):
|
||||
logger.warning("fetch followers: 响应不是 JSON 对象")
|
||||
return []
|
||||
status_code = data.get("status_code")
|
||||
if status_code not in (None, 0):
|
||||
logger.warning(
|
||||
"fetch followers: status_code=%s msg=%s",
|
||||
status_code,
|
||||
data.get("status_msg") or data.get("message") or "",
|
||||
)
|
||||
return []
|
||||
followers = _extract_followers(data)
|
||||
logger.info(
|
||||
"fetch followers ok: 拿到 %s 个粉丝 (has_more=%s total=%s)",
|
||||
len(followers),
|
||||
data.get("has_more"),
|
||||
data.get("total"),
|
||||
)
|
||||
return followers
|
||||
except Exception as exc:
|
||||
logger.warning("fetch followers failed: %s", exc)
|
||||
return []
|
||||
|
||||
|
||||
async def fetch_recent_followers(
|
||||
session,
|
||||
sec_user_id: str,
|
||||
count: int = 20,
|
||||
max_time: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
return await asyncio.to_thread(
|
||||
fetch_recent_followers_sync, session, sec_user_id, count, max_time
|
||||
)
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Build frontier-im WebSocket URL (DouYin_Spider douyin_recv_msg logic)."""
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from typing import Optional
|
||||
from urllib.parse import unquote
|
||||
|
||||
import requests
|
||||
|
||||
from .auth import DouyinAuth
|
||||
from .dy_util import generate_a_bogus, generate_msToken, generate_webid, splice_url
|
||||
from .session import DouyinImSession, is_frontier_ws_url
|
||||
|
||||
logger = logging.getLogger("douyin_im.frontier")
|
||||
|
||||
APP_KEY = "e1bd35ec9db7b8d846de66ed140b1ad9"
|
||||
FP_ID = "9"
|
||||
|
||||
|
||||
def build_frontier_ws_url(session: DouyinImSession, device_id: str) -> Optional[str]:
|
||||
token = session.cookies.get("sessionid") or session.cookies.get("sessionid_ss") or ""
|
||||
if not token or not device_id:
|
||||
return None
|
||||
access_key_raw = f"{FP_ID}{APP_KEY}{device_id}f8a69f1719916z"
|
||||
access_key = hashlib.md5(access_key_raw.encode("utf-8")).hexdigest()
|
||||
params = {
|
||||
"aid": "6383",
|
||||
"device_platform": "douyin_pc",
|
||||
"fpid": FP_ID,
|
||||
"device_id": device_id,
|
||||
"token": token,
|
||||
"access_key": access_key,
|
||||
}
|
||||
query = "&".join(f"{k}={v}" for k, v in params.items())
|
||||
return f"wss://frontier-im.douyin.com/ws/v2?{query}"
|
||||
|
||||
|
||||
def fetch_device_id(session: DouyinImSession) -> str:
|
||||
"""Call Douyin user query API to obtain device/web id."""
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(
|
||||
session.cookie_header(),
|
||||
session.web_protect_str,
|
||||
session.keys_str,
|
||||
)
|
||||
url = "https://www.douyin.com/aweme/v1/web/query/user"
|
||||
headers = {
|
||||
"User-Agent": session.user_agent,
|
||||
"Referer": "https://www.douyin.com/discover",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Cookie": session.cookie_header(),
|
||||
}
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"publish_video_strategy_type": "2",
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/discover"),
|
||||
"msToken": generate_msToken(),
|
||||
}
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=session.user_agent)
|
||||
try:
|
||||
resp = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
verify=False,
|
||||
timeout=15,
|
||||
)
|
||||
data = resp.json()
|
||||
device_id = str(data.get("id") or data.get("device_id") or "")
|
||||
if device_id.isdigit():
|
||||
logger.info(f"Fetched device_id: {device_id[:20]}...")
|
||||
return device_id
|
||||
logger.warning(f"query/user returned non-numeric id: {device_id[:32]!r}")
|
||||
except Exception as e:
|
||||
logger.warning(f"fetch_device_id failed: {e}")
|
||||
return ""
|
||||
|
||||
|
||||
def resolve_frontier_device_id(session: DouyinImSession) -> str:
|
||||
"""Frontier WS requires numeric device_id from Douyin query/user API."""
|
||||
current = str(session.device_id or session.web_id or "")
|
||||
if current.isdigit():
|
||||
return current
|
||||
|
||||
fetched = fetch_device_id(session)
|
||||
if fetched and str(fetched).isdigit():
|
||||
session.device_id = str(fetched)
|
||||
logger.info(f"Using numeric device_id for frontier WS: {fetched[:16]}...")
|
||||
return str(fetched)
|
||||
|
||||
logger.warning(
|
||||
f"Invalid frontier device_id={current[:24]!r}; "
|
||||
"expected numeric id from query/user API"
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def _ws_device_id(url: str) -> str:
|
||||
m = re.search(r"[?&]device_id=([^&\s]+)", url or "")
|
||||
return unquote(m.group(1)) if m else ""
|
||||
|
||||
|
||||
def _ws_device_matches_session(session: DouyinImSession, url: str) -> bool:
|
||||
ws_dev = _ws_device_id(url)
|
||||
if not ws_dev or not ws_dev.isdigit():
|
||||
return True
|
||||
for candidate in (session.my_uid, session.web_id, session.device_id):
|
||||
if candidate and str(candidate) == ws_dev:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _ws_token_looks_encoded(url: str) -> bool:
|
||||
m = re.search(r"[?&]token=([^&\s]+)", url or "")
|
||||
if not m:
|
||||
return False
|
||||
token = unquote(m.group(1))
|
||||
return len(token) >= 40 or not token.replace("_", "").replace("-", "").isalnum()
|
||||
|
||||
|
||||
def ensure_frontier_ws(session: DouyinImSession) -> Optional[str]:
|
||||
"""Ensure session has a usable frontier WebSocket URL."""
|
||||
session.sanitize_ws_urls()
|
||||
|
||||
for url in session.ws_urls:
|
||||
if is_frontier_ws_url(url) and "sdk_cert=" in url and _ws_token_looks_encoded(url):
|
||||
session.ws_urls = [url]
|
||||
logger.info("Using captured real frontier WS URL (with sdk_cert)")
|
||||
return url
|
||||
|
||||
for url in session.ws_urls:
|
||||
if is_frontier_ws_url(url) and _ws_token_looks_encoded(url):
|
||||
session.ws_urls = [url]
|
||||
logger.info("Using captured frontier WS URL")
|
||||
return url
|
||||
|
||||
device_id = resolve_frontier_device_id(session)
|
||||
if not device_id:
|
||||
session.ws_urls = []
|
||||
return None
|
||||
|
||||
built = build_frontier_ws_url(session, device_id)
|
||||
if built:
|
||||
session.ws_urls = [built]
|
||||
logger.info("Built frontier WS URL from cookie session")
|
||||
return built
|
||||
return None
|
||||
@@ -0,0 +1,38 @@
|
||||
"""本系统当前托管中的账号 UID 注册表。
|
||||
|
||||
用途：避免两个都在本系统托管的账号互相自动回复，形成无限回环——
|
||||
这种高频来回发送是触发抖音风控（7911）/业务拒绝（8004）的常见根因。
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger("douyin_im.hosted_registry")
|
||||
|
||||
_HOSTED_UIDS: set[int] = set()
|
||||
|
||||
|
||||
def register(uid) -> None:
|
||||
try:
|
||||
u = int(uid)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
if u:
|
||||
_HOSTED_UIDS.add(u)
|
||||
logger.info(f"Registered hosted uid {u} (total={len(_HOSTED_UIDS)})")
|
||||
|
||||
|
||||
def unregister(uid) -> None:
|
||||
try:
|
||||
u = int(uid)
|
||||
except (TypeError, ValueError):
|
||||
return
|
||||
_HOSTED_UIDS.discard(u)
|
||||
logger.info(f"Unregistered hosted uid {u} (total={len(_HOSTED_UIDS)})")
|
||||
|
||||
|
||||
def is_hosted(uid) -> bool:
|
||||
try:
|
||||
u = int(uid)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return u in _HOSTED_UIDS
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,32 @@
|
||||
"""Parse Douyin IM Response protobuf for conversation metadata."""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
def extract_conv_meta_from_response_bytes(raw: bytes) -> dict[str, dict]:
|
||||
"""Return {conversation_id: {conversation_short_id, ticket}} from IM API protobuf body."""
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
from .static import Response_pb2 as ResponseProto
|
||||
|
||||
response_proto = ResponseProto.Response()
|
||||
response_proto.ParseFromString(raw)
|
||||
body = response_proto.body
|
||||
out: dict[str, dict] = {}
|
||||
for field in (
|
||||
"create_conversation_v2_body",
|
||||
"get_conversation_info_list_v2_response_body",
|
||||
):
|
||||
if not body.HasField(field):
|
||||
continue
|
||||
conv_body = getattr(body, field)
|
||||
for conv in conv_body.conversation_info_list:
|
||||
if not conv.conversation_id:
|
||||
continue
|
||||
out[conv.conversation_id] = {
|
||||
"conversation_short_id": str(conv.conversation_short_id),
|
||||
"ticket": conv.ticket,
|
||||
}
|
||||
return out
|
||||
except Exception:
|
||||
return {}
|
||||
@@ -0,0 +1,720 @@
|
||||
"""抖音 IM 私信图片上传。
|
||||
|
||||
复刻抖音 PC 网页版私信发图的真实链路（与"创作者中心 ImageX 图文图床"是两套，
|
||||
IM 发送只认这条链路产出的 tos-cn-o-* 资源）：
|
||||
|
||||
1. GET www.douyin.com/aweme/v1/web/im/upload/config/v2 （a_bogus 签名）
|
||||
→ 返回内含 STS2 凭证（AccessKeyID + SignedSecretAccessKey）与 SpaceName(=zhenzhen)
|
||||
2. GET vod.bytedanceapi.com/?Action=ApplyUploadInner&SpaceName=zhenzhen&FileType=image
|
||||
（AWS4-HMAC-SHA256，service=vod，带 x-amz-security-token=STS2…）
|
||||
→ 返回 UploadHost / StoreUri(tos-cn-o-*) / Auth(SpaceKey JWT) / SessionKey
|
||||
3. POST https://{UploadHost}/upload/v1/{StoreUri}
|
||||
（Authorization: SpaceKey/zhenzhen/…JWT，Content-CRC32，X-Storage-U=my_uid）
|
||||
4. POST vod.bytedanceapi.com/?Action=CommitUploadInner&SpaceName=zhenzhen （AWS4 签名）
|
||||
→ 确认上传，最终 uri 即 StoreUri（tos-cn-o-*）
|
||||
|
||||
之后用该 tos-cn-o-* uri 构造 type=27 消息体即可被 IM 后端校验通过。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import string
|
||||
import zlib
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
|
||||
logger = logging.getLogger("douyin_im.image_upload")
|
||||
|
||||
_LOCAL_URL_RE = re.compile(
|
||||
r"^(/api/media/messages/|https?://(?:localhost|127\.0\.0\.1)(?::\d+)?/api/media/messages/)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
# 抖音 web 私信图片上传专用通道（VOD/ImageX inner 接口 + IM 上传配置）
|
||||
VOD_HOST = "https://vod.bytedanceapi.com/"
|
||||
VOD_REGION = "cn-north-1"
|
||||
VOD_SERVICE = "vod"
|
||||
IM_UPLOAD_CONFIG_URL = "https://www.douyin.com/aweme/v1/web/im/upload/config/v2"
|
||||
DEFAULT_SPACE_NAME = "zhenzhen"
|
||||
|
||||
|
||||
def is_local_media_url(url: str) -> bool:
|
||||
raw = (url or "").strip()
|
||||
if not raw:
|
||||
return False
|
||||
if raw.startswith("/api/media/messages/"):
|
||||
return True
|
||||
if raw.startswith("/api/media/link-cards/"):
|
||||
return True
|
||||
return bool(_LOCAL_URL_RE.match(raw))
|
||||
|
||||
|
||||
def is_douyin_cdn_url(url: str) -> bool:
|
||||
raw = (url or "").strip().lower()
|
||||
if not raw.startswith("http"):
|
||||
return False
|
||||
return any(
|
||||
host in raw
|
||||
for host in (
|
||||
"douyinpic.com",
|
||||
"byteimg.com",
|
||||
"ibyteimg.com",
|
||||
"douyinstatic.com",
|
||||
"snssdk.com",
|
||||
"vodupload.com",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def parse_local_message_media_path(url: str) -> tuple[int, str] | None:
|
||||
"""从 /api/media/messages/{account_id}/{filename} 解析 account_id 与文件名。"""
|
||||
raw = (url or "").strip()
|
||||
m = re.search(r"/api/media/messages/(\d+)/([^/?#]+)", raw)
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)), m.group(2)
|
||||
|
||||
|
||||
def parse_link_card_media_path(url: str) -> tuple[int, str] | None:
|
||||
"""从 /api/media/link-cards/{owner_id}/{filename} 解析 owner_id 与文件名。"""
|
||||
raw = (url or "").strip()
|
||||
m = re.search(r"/api/media/link-cards/(\d+)/([^/?#]+)", raw)
|
||||
if not m:
|
||||
return None
|
||||
return int(m.group(1)), m.group(2)
|
||||
|
||||
|
||||
|
||||
|
||||
def _requests_proxies() -> dict | None:
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
|
||||
return requests_proxies()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _random_s() -> str:
|
||||
chars = string.digits + string.ascii_lowercase
|
||||
return "".join(random.choice(chars) for _ in range(11))
|
||||
|
||||
|
||||
def _signing_key(secret: str, date_stamp: str, region: str, service: str) -> bytes:
|
||||
k_date = hmac.new(("AWS4" + secret).encode(), date_stamp.encode(), hashlib.sha256).digest()
|
||||
k_region = hmac.new(k_date, region.encode(), hashlib.sha256).digest()
|
||||
k_service = hmac.new(k_region, service.encode(), hashlib.sha256).digest()
|
||||
return hmac.new(k_service, b"aws4_request", hashlib.sha256).digest()
|
||||
|
||||
|
||||
def _aws4_authorization(
|
||||
*,
|
||||
method: str,
|
||||
canonical_querystring: str,
|
||||
amz_date: str,
|
||||
date_stamp: str,
|
||||
session_token: str,
|
||||
access_key_id: str,
|
||||
secret_access_key: str,
|
||||
payload_hash: str | None = None,
|
||||
signed_headers: list[str] | None = None,
|
||||
service: str = VOD_SERVICE,
|
||||
region: str = VOD_REGION,
|
||||
) -> str:
|
||||
if signed_headers is None:
|
||||
signed_headers = ["x-amz-date", "x-amz-security-token"]
|
||||
if payload_hash is None:
|
||||
payload_hash = hashlib.sha256(b"").hexdigest()
|
||||
|
||||
header_lines = []
|
||||
for name in signed_headers:
|
||||
if name == "x-amz-date":
|
||||
header_lines.append(f"x-amz-date:{amz_date}\n")
|
||||
elif name == "x-amz-security-token":
|
||||
header_lines.append(f"x-amz-security-token:{session_token}\n")
|
||||
elif name == "x-amz-content-sha256":
|
||||
header_lines.append(f"x-amz-content-sha256:{payload_hash}\n")
|
||||
canonical_headers = "".join(header_lines)
|
||||
signed = ";".join(signed_headers)
|
||||
canonical_request = (
|
||||
f"{method}\n/\n{canonical_querystring}\n{canonical_headers}\n{signed}\n{payload_hash}"
|
||||
)
|
||||
credential_scope = f"{date_stamp}/{region}/{service}/aws4_request"
|
||||
string_to_sign = (
|
||||
"AWS4-HMAC-SHA256\n"
|
||||
f"{amz_date}\n"
|
||||
f"{credential_scope}\n"
|
||||
f"{hashlib.sha256(canonical_request.encode()).hexdigest()}"
|
||||
)
|
||||
signature = hmac.new(
|
||||
_signing_key(secret_access_key, date_stamp, region, service),
|
||||
string_to_sign.encode(),
|
||||
hashlib.sha256,
|
||||
).hexdigest()
|
||||
return (
|
||||
f"AWS4-HMAC-SHA256 Credential={access_key_id}/{credential_scope}, "
|
||||
f"SignedHeaders={signed}, Signature={signature}"
|
||||
)
|
||||
|
||||
|
||||
def _safe_json(resp) -> dict[str, Any]:
|
||||
text = (getattr(resp, "text", None) or "").strip()
|
||||
if not text:
|
||||
return {"error": f"空响应 (HTTP {getattr(resp, 'status_code', '?')})"}
|
||||
try:
|
||||
data = resp.json()
|
||||
return data if isinstance(data, dict) else {"error": "响应不是 JSON 对象"}
|
||||
except Exception:
|
||||
snippet = text[:200].replace("\n", " ")
|
||||
return {"error": f"非 JSON 响应 (HTTP {getattr(resp, 'status_code', '?')}): {snippet}"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 1 步：拉取 IM 上传配置，提取 STS 凭证 + SpaceName
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _sanitize_for_log(obj: Any, depth: int = 0) -> Any:
|
||||
"""结构化脱敏：保留字段名与层级，长字符串只留前 12 位 + 长度，便于排查凭证字段。"""
|
||||
if depth > 6:
|
||||
return "…"
|
||||
if isinstance(obj, dict):
|
||||
return {str(k): _sanitize_for_log(v, depth + 1) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [_sanitize_for_log(v, depth + 1) for v in obj[:3]]
|
||||
if isinstance(obj, str):
|
||||
return f"{obj[:12]}…(len={len(obj)})" if len(obj) > 24 else obj
|
||||
return obj
|
||||
|
||||
|
||||
def _find_auth_object(obj: Any, depth: int = 0) -> dict | None:
|
||||
"""找到「直接含有 STS2 会话凭证字符串」的那个 dict（即临时凭证对象）。
|
||||
|
||||
该对象里通常同时含有 AccessKeyId / SecretAccessKey / SessionToken(=STS2…)。
|
||||
关键：签名要用同级的 SecretAccessKey 字段，而**不是** STS2 令牌内部解出的
|
||||
SignedSecretAccessKey（那是服务端校验用的，拿来当签名密钥会 SignatureDoesNotMatch）。
|
||||
"""
|
||||
if depth > 8 or not isinstance(obj, (dict, list)):
|
||||
return None
|
||||
if isinstance(obj, dict):
|
||||
for value in obj.values():
|
||||
if isinstance(value, str) and value.startswith("STS2"):
|
||||
return obj
|
||||
for value in obj.values():
|
||||
found = _find_auth_object(value, depth + 1)
|
||||
if found is not None:
|
||||
return found
|
||||
else:
|
||||
for value in obj:
|
||||
found = _find_auth_object(value, depth + 1)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _find_sts_token(obj: Any, depth: int = 0) -> str:
|
||||
"""递归在响应里找到形如 'STS2...' 的会话凭证字符串。"""
|
||||
if depth > 8:
|
||||
return ""
|
||||
if isinstance(obj, str):
|
||||
return obj if obj.startswith("STS2") else ""
|
||||
if isinstance(obj, dict):
|
||||
for value in obj.values():
|
||||
found = _find_sts_token(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
found = _find_sts_token(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
return ""
|
||||
|
||||
|
||||
def _find_space_name(obj: Any, depth: int = 0) -> str:
|
||||
if depth > 8:
|
||||
return ""
|
||||
if isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
if str(key).lower() in ("space_name", "spacename") and isinstance(value, str) and value:
|
||||
return value
|
||||
found = _find_space_name(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
elif isinstance(obj, list):
|
||||
for value in obj:
|
||||
found = _find_space_name(value, depth + 1)
|
||||
if found:
|
||||
return found
|
||||
return ""
|
||||
|
||||
|
||||
def _decode_sts(sts_token: str) -> tuple[str, str]:
|
||||
"""STS2<base64(JSON)>，解出 AccessKeyID 与 SignedSecretAccessKey。"""
|
||||
try:
|
||||
b64 = sts_token[4:] if sts_token.startswith("STS2") else sts_token
|
||||
b64 += "=" * (-len(b64) % 4)
|
||||
data = json.loads(base64.b64decode(b64).decode("utf-8", "ignore"))
|
||||
ak = data.get("AccessKeyID") or data.get("AccessKeyId") or ""
|
||||
sk = data.get("SignedSecretAccessKey") or data.get("SecretAccessKey") or ""
|
||||
return ak, sk
|
||||
except Exception:
|
||||
return "", ""
|
||||
|
||||
|
||||
def _fetch_im_upload_sts(session) -> tuple[str, str, str, str]:
|
||||
"""返回 (access_key_id, secret_access_key, sts_token, space_name)。"""
|
||||
import requests
|
||||
|
||||
from .auth import DouyinAuth
|
||||
from .dy_util import (
|
||||
DEFAULT_USER_AGENT,
|
||||
generate_a_bogus,
|
||||
generate_msToken,
|
||||
generate_webid,
|
||||
splice_url,
|
||||
)
|
||||
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||||
ua = session.user_agent or DEFAULT_USER_AGENT
|
||||
|
||||
params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"update_version_code": "170400",
|
||||
"pc_client_type": "1",
|
||||
"pc_libra_divert": "Windows",
|
||||
"support_h265": "1",
|
||||
"support_dash": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"cookie_enabled": "true",
|
||||
"screen_width": "1536",
|
||||
"screen_height": "960",
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Chrome",
|
||||
"browser_version": "120.0.0.0",
|
||||
"browser_online": "true",
|
||||
"engine_name": "Blink",
|
||||
"engine_version": "120.0.0.0",
|
||||
"os_name": "Windows",
|
||||
"os_version": "10",
|
||||
"cpu_core_num": "8",
|
||||
"device_memory": "8",
|
||||
"platform": "PC",
|
||||
"downlink": "10",
|
||||
"effective_type": "4g",
|
||||
"round_trip_time": "50",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
}
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
resp = requests.get(
|
||||
IM_UPLOAD_CONFIG_URL,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
timeout=20,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"获取 IM 上传配置失败：{data['error']}")
|
||||
if data.get("status_code") not in (None, 0):
|
||||
raise RuntimeError(
|
||||
f"获取 IM 上传配置失败：status_code={data.get('status_code')} "
|
||||
f"{data.get('status_msg') or ''}"
|
||||
)
|
||||
|
||||
# 诊断：把 config/v2 响应结构（字段名保留、长字符串脱敏）打到日志，便于核对凭证字段。
|
||||
try:
|
||||
logger.info(
|
||||
"im/upload/config/v2 结构: %s",
|
||||
json.dumps(_sanitize_for_log(data), ensure_ascii=False),
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 凭证块选择（决定 8003 是否发生的关键）：
|
||||
# - inner_image_config / public_image_config / public_file_config 共用同一套
|
||||
# 可用于 VOD 上传的 STS 令牌，仅 space 不同；
|
||||
# - public_image_config_v2 的 token 不同，不兼容 VOD 内部上传（会报
|
||||
# "session token sequence is broken"），必须避开。
|
||||
# 空间含义：maya_review = 审核空间，图片上传后处于待审核态，作为消息发送会被拒(8003)；
|
||||
# zhenzhen = 公开可发送空间，IM 图片消息应使用它。
|
||||
# 因此优先 public_image_config（zhenzhen），其凭证同样能完成 VOD 上传。
|
||||
def _block_has_sts(block: Any) -> bool:
|
||||
return isinstance(block, dict) and any(
|
||||
isinstance(v, str) and v.startswith("STS2") for v in block.values()
|
||||
)
|
||||
|
||||
auth_obj = None
|
||||
for _key in ("public_image_config", "inner_image_config"):
|
||||
if isinstance(data, dict) and _block_has_sts(data.get(_key)):
|
||||
auth_obj = data[_key]
|
||||
break
|
||||
if auth_obj is None:
|
||||
auth_obj = _find_auth_object(data)
|
||||
if not auth_obj:
|
||||
raise RuntimeError(
|
||||
"IM 上传配置响应里未找到 STS 凭证（STS2 token）；可能 cookie/签名失效，请用浏览器模式重新登录"
|
||||
)
|
||||
|
||||
sts = next(
|
||||
(v for v in auth_obj.values() if isinstance(v, str) and v.startswith("STS2")),
|
||||
"",
|
||||
)
|
||||
# 优先用凭证对象同级的 AccessKeyId / SecretAccessKey（用于 SigV4 签名的真实密钥）。
|
||||
ak = (
|
||||
auth_obj.get("AccessKeyID")
|
||||
or auth_obj.get("AccessKeyId")
|
||||
or auth_obj.get("access_key_id")
|
||||
or ""
|
||||
)
|
||||
sk = (
|
||||
auth_obj.get("SecretAccessKey")
|
||||
or auth_obj.get("SecretAccesskey")
|
||||
or auth_obj.get("secret_access_key")
|
||||
or ""
|
||||
)
|
||||
# 兜底：若响应没给独立的 ak/sk，再尝试从 STS2 令牌解码（SignedSecretAccessKey 一般不可用，仅最后兜底）。
|
||||
if not ak or not sk:
|
||||
dec_ak, dec_sk = _decode_sts(sts)
|
||||
ak = ak or dec_ak
|
||||
sk = sk or dec_sk
|
||||
if not ak or not sk:
|
||||
raise RuntimeError("解析 STS 凭证失败（缺少 AccessKeyId / SecretAccessKey）")
|
||||
# space 必须取自与凭证同一块，避免凭证用 inner_image_config 而 space 误取到别处。
|
||||
space = (
|
||||
auth_obj.get("space_name")
|
||||
or auth_obj.get("SpaceName")
|
||||
or auth_obj.get("spaceName")
|
||||
or _find_space_name(data)
|
||||
or DEFAULT_SPACE_NAME
|
||||
)
|
||||
return ak, sk, sts, space
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 2 步：ApplyUploadInner（VOD），申请上传地址
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _extract_apply_inner(data: dict[str, Any]) -> tuple[str, str, str, str]:
|
||||
"""返回 (upload_host, store_uri, jwt_auth, session_key)。"""
|
||||
result = data.get("Result") or {}
|
||||
addr = result.get("InnerUploadAddress") or result.get("UploadAddress") or {}
|
||||
|
||||
nodes = addr.get("UploadNodes") or []
|
||||
if nodes:
|
||||
node = nodes[0]
|
||||
stores = node.get("StoreInfos") or []
|
||||
store = stores[0] if stores else {}
|
||||
host = node.get("UploadHost") or ""
|
||||
if not host:
|
||||
hosts = node.get("UploadHosts") or addr.get("UploadHosts") or []
|
||||
host = hosts[0] if hosts else ""
|
||||
return (
|
||||
str(host or ""),
|
||||
str(store.get("StoreUri") or ""),
|
||||
str(store.get("Auth") or ""),
|
||||
str(node.get("SessionKey") or addr.get("SessionKey") or ""),
|
||||
)
|
||||
|
||||
hosts = addr.get("UploadHosts") or []
|
||||
host = hosts[0] if hosts else ""
|
||||
stores = addr.get("StoreInfos") or []
|
||||
store = stores[0] if stores else {}
|
||||
return (
|
||||
str(host or ""),
|
||||
str(store.get("StoreUri") or ""),
|
||||
str(store.get("Auth") or ""),
|
||||
str(addr.get("SessionKey") or result.get("SessionKey") or ""),
|
||||
)
|
||||
|
||||
|
||||
def _vod_apply_upload_inner(
|
||||
ak: str, sk: str, token: str, space: str, file_size: int
|
||||
) -> tuple[str, str, str, str]:
|
||||
import requests
|
||||
|
||||
from .dy_util import DEFAULT_USER_AGENT
|
||||
|
||||
now = datetime.datetime.utcnow()
|
||||
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
|
||||
date_stamp = now.strftime("%Y%m%d")
|
||||
params = {
|
||||
"Action": "ApplyUploadInner",
|
||||
"Version": "2020-11-19",
|
||||
"SpaceName": space,
|
||||
"FileType": "image",
|
||||
"IsInner": "1",
|
||||
"NeedFallback": "true",
|
||||
"FileSize": str(file_size),
|
||||
"s": _random_s(),
|
||||
}
|
||||
qs = urlencode(sorted(params.items()))
|
||||
authorization = _aws4_authorization(
|
||||
method="GET",
|
||||
canonical_querystring=qs,
|
||||
amz_date=amz_date,
|
||||
date_stamp=date_stamp,
|
||||
session_token=token,
|
||||
access_key_id=ak,
|
||||
secret_access_key=sk,
|
||||
service=VOD_SERVICE,
|
||||
)
|
||||
resp = requests.get(
|
||||
f"{VOD_HOST}?{qs}",
|
||||
headers={
|
||||
"accept": "*/*",
|
||||
"authorization": authorization,
|
||||
"user-agent": DEFAULT_USER_AGENT,
|
||||
"x-amz-date": amz_date,
|
||||
"x-amz-security-token": token,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
},
|
||||
timeout=30,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"申请上传地址失败：{data['error']}")
|
||||
meta = data.get("ResponseMetadata") or {}
|
||||
err = meta.get("Error")
|
||||
if err:
|
||||
raise RuntimeError(f"申请上传地址失败：{err.get('Message') or err.get('Code')}")
|
||||
return _extract_apply_inner(data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 3 步：上传二进制（SpaceKey JWT 鉴权）
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _vod_upload_binary(
|
||||
host: str, store_uri: str, jwt_auth: str, user_id: str, raw: bytes, session=None
|
||||
) -> None:
|
||||
import requests
|
||||
|
||||
from .dy_util import DEFAULT_USER_AGENT
|
||||
|
||||
crc32 = format(zlib.crc32(raw) & 0xFFFFFFFF, "08x")
|
||||
url = f"https://{host}/upload/v1/{store_uri}"
|
||||
ua = (getattr(session, "user_agent", None) or DEFAULT_USER_AGENT)
|
||||
headers = {
|
||||
"Authorization": jwt_auth,
|
||||
"Content-CRC32": crc32,
|
||||
"Content-Type": "application/octet-stream",
|
||||
"Content-Disposition": 'attachment; filename="undefined"',
|
||||
"User-Agent": ua,
|
||||
}
|
||||
if user_id:
|
||||
headers["X-Storage-U"] = str(user_id)
|
||||
resp = requests.post(
|
||||
url,
|
||||
headers=headers,
|
||||
data=raw,
|
||||
timeout=60,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"上传图片数据失败：{data['error']}")
|
||||
if data.get("code") not in (2000, 0, None):
|
||||
raise RuntimeError(f"上传图片数据失败：{data.get('message') or data}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 第 4 步：CommitUploadInner（VOD），确认上传
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _vod_commit_upload_inner(
|
||||
ak: str, sk: str, token: str, space: str, session_key: str
|
||||
) -> dict[str, Any]:
|
||||
import requests
|
||||
|
||||
from .dy_util import DEFAULT_USER_AGENT
|
||||
|
||||
now = datetime.datetime.utcnow()
|
||||
amz_date = now.strftime("%Y%m%dT%H%M%SZ")
|
||||
date_stamp = now.strftime("%Y%m%d")
|
||||
params = {
|
||||
"Action": "CommitUploadInner",
|
||||
"Version": "2020-11-19",
|
||||
"SpaceName": space,
|
||||
}
|
||||
qs = urlencode(sorted(params.items()))
|
||||
body = json.dumps({"SessionKey": session_key, "Functions": []}, separators=(",", ":"))
|
||||
payload_hash = hashlib.sha256(body.encode()).hexdigest()
|
||||
signed_headers = ["x-amz-content-sha256", "x-amz-date", "x-amz-security-token"]
|
||||
authorization = _aws4_authorization(
|
||||
method="POST",
|
||||
canonical_querystring=qs,
|
||||
amz_date=amz_date,
|
||||
date_stamp=date_stamp,
|
||||
session_token=token,
|
||||
access_key_id=ak,
|
||||
secret_access_key=sk,
|
||||
payload_hash=payload_hash,
|
||||
signed_headers=signed_headers,
|
||||
service=VOD_SERVICE,
|
||||
)
|
||||
resp = requests.post(
|
||||
f"{VOD_HOST}?{qs}",
|
||||
data=body,
|
||||
headers={
|
||||
"accept": "*/*",
|
||||
"authorization": authorization,
|
||||
"content-type": "application/json",
|
||||
"user-agent": DEFAULT_USER_AGENT,
|
||||
"x-amz-content-sha256": payload_hash,
|
||||
"x-amz-date": amz_date,
|
||||
"x-amz-security-token": token,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
},
|
||||
timeout=30,
|
||||
verify=False,
|
||||
proxies=_requests_proxies(),
|
||||
)
|
||||
data = _safe_json(resp)
|
||||
if data.get("error"):
|
||||
raise RuntimeError(f"确认上传失败：{data['error']}")
|
||||
meta = data.get("ResponseMetadata") or {}
|
||||
err = meta.get("Error")
|
||||
if err:
|
||||
raise RuntimeError(f"确认上传失败：{err.get('Message') or err.get('Code')}")
|
||||
return data.get("Result") or {}
|
||||
|
||||
|
||||
def upload_im_image(
|
||||
session,
|
||||
raw: bytes,
|
||||
*,
|
||||
filename: str = "image.jpg",
|
||||
content_type: str = "image/jpeg",
|
||||
) -> dict[str, Any]:
|
||||
"""上传图片到抖音 IM 私信图床（VOD/zhenzhen 空间）。
|
||||
|
||||
成功返回 {uri(tos-cn-o-*), url, url_list, md5}；失败返回 {"error": "..."}。
|
||||
"""
|
||||
del filename, content_type # VOD 按二进制上传，文件名仅用于本地存储
|
||||
if not raw:
|
||||
return {"error": "图片为空"}
|
||||
try:
|
||||
ak, sk, token, space = _fetch_im_upload_sts(session)
|
||||
host, store_uri, jwt_auth, session_key = _vod_apply_upload_inner(
|
||||
ak, sk, token, space, len(raw)
|
||||
)
|
||||
if not host or not store_uri or not jwt_auth:
|
||||
return {"error": "申请上传地址失败：缺少 UploadHost/StoreUri/Auth"}
|
||||
|
||||
user_id = str(getattr(session, "my_uid", "") or "")
|
||||
_vod_upload_binary(host, store_uri, jwt_auth, user_id, raw, session)
|
||||
_vod_commit_upload_inner(ak, sk, token, space, session_key)
|
||||
|
||||
uri = store_uri.lstrip("/")
|
||||
out: dict[str, Any] = {"uri": uri, "md5": hashlib.md5(raw).hexdigest()}
|
||||
|
||||
from .message_content import uri_to_cdn_urls
|
||||
|
||||
urls = uri_to_cdn_urls(uri)
|
||||
if urls:
|
||||
out["url_list"] = urls
|
||||
out["url"] = urls[0]
|
||||
|
||||
logger.info("Uploaded IM image via VOD uri=%s host=%s space=%s", uri, host, space)
|
||||
return out
|
||||
except Exception as exc:
|
||||
logger.warning("upload_im_image (VOD) failed: %s", exc)
|
||||
return {"error": str(exc)}
|
||||
|
||||
|
||||
def prepare_image_reply_spec(spec: dict[str, Any], session, upload_dir: str) -> tuple[dict[str, Any], str]:
|
||||
"""若图片仍是本地地址，则上传到抖音 CDN 并补全 uri。返回 (spec, error)。"""
|
||||
if spec.get("type") != "image":
|
||||
return spec, ""
|
||||
|
||||
uri = str(spec.get("uri") or "").strip()
|
||||
url = str(spec.get("url") or "").strip()
|
||||
|
||||
# 已经持有抖音 CDN 的 uri，说明图片早已上传完成（且经历过风控/转码），直接复用即可。
|
||||
# 关键修复：不要因为 url 仍是本机预览地址(/api/media/...)而再次上传——
|
||||
# 重复上传会拿到一个“刚提交、尚未完成风控/转码”的新 uri，发送时常被抖音以
|
||||
# raw_check_code=1 / status_code=8003 拒绝（与是否互关无关）。
|
||||
# 发送链路(build_msg_payload)只用 uri / url_list，从不使用这个本机 url，故本机 url 无害。
|
||||
if uri and not is_local_media_url(uri):
|
||||
return spec, ""
|
||||
|
||||
if url and is_douyin_cdn_url(url) and not uri:
|
||||
return spec, ""
|
||||
|
||||
raw: bytes | None = None
|
||||
filename = "image.jpg"
|
||||
content_type = "image/jpeg"
|
||||
|
||||
if is_local_media_url(url):
|
||||
import os
|
||||
|
||||
card_parsed = parse_link_card_media_path(url)
|
||||
if card_parsed:
|
||||
# 卡片封面在 uploads/link-cards/{owner}/ 下，与消息图片目录(uploads/messages)不同。
|
||||
owner_id, fname = card_parsed
|
||||
link_cards_dir = os.path.join(os.path.dirname(upload_dir), "link-cards")
|
||||
path = os.path.join(link_cards_dir, str(owner_id), fname)
|
||||
else:
|
||||
parsed = parse_local_message_media_path(url)
|
||||
if not parsed:
|
||||
return spec, "无法解析本地图片路径"
|
||||
_, fname = parsed
|
||||
path = os.path.join(upload_dir, str(parsed[0]), fname)
|
||||
if not os.path.isfile(path):
|
||||
return spec, f"本地图片不存在：{fname}"
|
||||
with open(path, "rb") as f:
|
||||
raw = f.read()
|
||||
filename = fname
|
||||
if fname.lower().endswith(".png"):
|
||||
content_type = "image/png"
|
||||
elif fname.lower().endswith(".webp"):
|
||||
content_type = "image/webp"
|
||||
elif fname.lower().endswith(".gif"):
|
||||
content_type = "image/gif"
|
||||
|
||||
if raw is None:
|
||||
if url.startswith("http") and not is_douyin_cdn_url(url):
|
||||
return spec, "图片地址必须是抖音 CDN 或本地上传后的地址，外部 URL 无法用于 IM 发送"
|
||||
return spec, "缺少可上传的图片数据"
|
||||
|
||||
uploaded = upload_im_image(session, raw, filename=filename, content_type=content_type)
|
||||
if uploaded.get("error"):
|
||||
return spec, uploaded["error"]
|
||||
if not uploaded.get("uri"):
|
||||
return spec, "抖音图片上传失败：未返回 uri"
|
||||
|
||||
merged = {
|
||||
**spec,
|
||||
**uploaded,
|
||||
"type": "image",
|
||||
"text": spec.get("text") or "[图片]",
|
||||
}
|
||||
return merged, ""
|
||||
@@ -0,0 +1,746 @@
|
||||
"""私信消息内容解析、存储与展示（文本 / 图片 / 表情 / 语音 / 视频）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
MSG_TYPE_TEXT = 7
|
||||
MSG_TYPE_STICKER = 5
|
||||
MSG_TYPE_VOICE = 17
|
||||
MSG_TYPE_IMAGE = 27
|
||||
MSG_TYPE_VIDEO = 8
|
||||
MSG_TYPE_LINK_CARD = 70
|
||||
|
||||
_TYPE_LABELS = {
|
||||
"text": "文本",
|
||||
"image": "图片",
|
||||
"sticker": "表情",
|
||||
"voice": "语音",
|
||||
"video": "视频",
|
||||
"link": "链接",
|
||||
"link_card": "链接卡片",
|
||||
}
|
||||
|
||||
_PLACEHOLDER_MARKERS = {
|
||||
"[表情包]",
|
||||
"[语音]",
|
||||
"[图片]",
|
||||
"[视频]",
|
||||
"[未读消息]",
|
||||
}
|
||||
|
||||
_URI_HINT_RE = re.compile(
|
||||
r"(tos-cn|aweme-|voice/|ies-music|\.mp3|\.m4a|\.aac|\.mpeg|\.webp|\.jpeg|\.jpg|\.png|\.gif)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
_AUDIO_URL_RE = re.compile(
|
||||
r"(douyin-user-audio|/audio/|sc=audio|voice/|ies-music|\.mp3|\.m4a|\.aac|\.mpeg|\.wav|\.ogg)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
_VIDEO_URL_RE = re.compile(
|
||||
r"(sc=video|/video/|\.mp4|\.mov|\.webm|\.m3u8)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
_MEDIA_STRING_KEYS = (
|
||||
"url",
|
||||
"uri",
|
||||
"main_url",
|
||||
"download_url",
|
||||
"remote_url",
|
||||
"encrypt_url",
|
||||
"play_url",
|
||||
"secret_url",
|
||||
"audio_url",
|
||||
"video_url",
|
||||
"cover_url",
|
||||
"local_path",
|
||||
)
|
||||
|
||||
_MEDIA_NESTED_KEYS = (
|
||||
"resource_url",
|
||||
"static_url",
|
||||
"animate_url",
|
||||
"cover_url",
|
||||
"thumb_url",
|
||||
"origin_url",
|
||||
"image",
|
||||
"picture",
|
||||
"pic",
|
||||
"sticker",
|
||||
"emoji",
|
||||
"audio",
|
||||
"voice",
|
||||
"video",
|
||||
"media",
|
||||
"large_url",
|
||||
"medium_url",
|
||||
"thumb",
|
||||
"avatar_thumb",
|
||||
"play_url",
|
||||
)
|
||||
|
||||
|
||||
def _normalize_uri_path(raw: str) -> str:
|
||||
path = (raw or "").strip().lstrip("/")
|
||||
if path.startswith("obj/"):
|
||||
path = path[4:]
|
||||
return path
|
||||
|
||||
|
||||
def uri_to_cdn_urls(uri: str, *, prefer_voice: bool = False) -> list[str]:
|
||||
"""将抖音 IM 中的 uri / tos 路径转为可访问的 CDN URL 候选列表。"""
|
||||
raw = (uri or "").strip()
|
||||
if not raw:
|
||||
return []
|
||||
if raw.startswith("//"):
|
||||
return [f"https:{raw}"]
|
||||
if raw.startswith("http://") or raw.startswith("https://"):
|
||||
return [raw]
|
||||
|
||||
path = _normalize_uri_path(raw)
|
||||
if not path:
|
||||
return []
|
||||
|
||||
candidates: list[str] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
def add(url: str) -> None:
|
||||
url = (url or "").strip()
|
||||
if url and url not in seen:
|
||||
seen.add(url)
|
||||
candidates.append(url)
|
||||
|
||||
lower = path.lower()
|
||||
is_voice = prefer_voice or lower.startswith("voice/") or lower.endswith((".mp3", ".m4a", ".aac"))
|
||||
is_image = (
|
||||
not is_voice
|
||||
and (
|
||||
"tos-cn-i" in lower
|
||||
or "aweme-" in lower
|
||||
or lower.endswith((".jpeg", ".jpg", ".png", ".webp", ".gif"))
|
||||
)
|
||||
)
|
||||
|
||||
if is_voice:
|
||||
for host in (
|
||||
"sf6-cdn-tos.douyinstatic.com",
|
||||
"sf3-cdn-tos.douyinstatic.com",
|
||||
"lf3-static.bytednsdoc.com",
|
||||
):
|
||||
add(f"https://{host}/obj/{path}")
|
||||
add(f"https://p3.douyinpic.com/obj/{path}")
|
||||
|
||||
if is_image or "tos-cn" in lower or "aweme" in lower:
|
||||
add(f"https://p3.douyinpic.com/obj/{path}")
|
||||
for size in ("720x720", "480x480", "300x300", "200x200", "100x100"):
|
||||
add(f"https://p3.douyinpic.com/aweme/{size}/{path}")
|
||||
add(f"https://p9-dy.byteimg.com/img/{path}")
|
||||
add(f"https://p6-dy.byteimg.com/img/{path}")
|
||||
|
||||
add(f"https://p3.douyinpic.com/obj/{path}")
|
||||
add(f"https://p3-sign.douyinpic.com/obj/{path}".replace("-sign", ""))
|
||||
return candidates
|
||||
|
||||
|
||||
def resolve_media_uri(uri: str, *, prefer_voice: bool = False) -> str:
|
||||
urls = uri_to_cdn_urls(uri, prefer_voice=prefer_voice)
|
||||
return urls[0] if urls else ""
|
||||
|
||||
|
||||
def _looks_like_audio_url(value: str) -> bool:
|
||||
raw = (value or "").strip()
|
||||
return bool(raw and _AUDIO_URL_RE.search(raw))
|
||||
|
||||
|
||||
def _looks_like_video_url(value: str) -> bool:
|
||||
raw = (value or "").strip()
|
||||
return bool(raw and _VIDEO_URL_RE.search(raw))
|
||||
|
||||
|
||||
def _infer_media_type_from_url(url: str) -> str:
|
||||
"""根据 URL 特征推断媒体类型（语音/视频/图片）。"""
|
||||
if _looks_like_audio_url(url):
|
||||
return "voice"
|
||||
if _looks_like_video_url(url):
|
||||
return "video"
|
||||
return "image"
|
||||
|
||||
|
||||
def _looks_like_media_uri(value: str) -> bool:
|
||||
raw = (value or "").strip()
|
||||
if not raw or raw.startswith("{"):
|
||||
return False
|
||||
if raw.startswith("http://") or raw.startswith("https://") or raw.startswith("//"):
|
||||
return True
|
||||
return bool(_URI_HINT_RE.search(raw))
|
||||
|
||||
|
||||
def _resolve_string_media(value: str, *, prefer_voice: bool = False) -> tuple[str, str]:
|
||||
raw = (value or "").strip()
|
||||
if not raw:
|
||||
return "", ""
|
||||
if raw.startswith("//"):
|
||||
return f"https:{raw}", raw
|
||||
if raw.startswith("http://") or raw.startswith("https://"):
|
||||
return raw, ""
|
||||
if _looks_like_media_uri(raw):
|
||||
return resolve_media_uri(raw, prefer_voice=prefer_voice), raw
|
||||
return "", ""
|
||||
|
||||
|
||||
def _valid_sticker_id(value: Any) -> str:
|
||||
if value is None or value == "":
|
||||
return ""
|
||||
try:
|
||||
if int(value) == 0:
|
||||
return ""
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
sticker_id = str(value).strip()
|
||||
return "" if sticker_id in ("0", "null", "None") else sticker_id
|
||||
|
||||
|
||||
def _collect_media_candidates(value: Any, out: list[str], *, prefer_voice: bool = False, depth: int = 0) -> None:
|
||||
if depth > 12:
|
||||
return
|
||||
if isinstance(value, str):
|
||||
raw = value.strip()
|
||||
if raw.startswith("http://") or raw.startswith("https://") or raw.startswith("//"):
|
||||
url, _ = _resolve_string_media(raw, prefer_voice=prefer_voice)
|
||||
if url:
|
||||
out.append(url)
|
||||
return
|
||||
if isinstance(value, dict):
|
||||
for list_key in ("url_list", "urls", "urlList"):
|
||||
urls = value.get(list_key)
|
||||
if isinstance(urls, list):
|
||||
for item in urls:
|
||||
_collect_media_candidates(item, out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
for key in _MEDIA_STRING_KEYS:
|
||||
direct = value.get(key)
|
||||
if isinstance(direct, str):
|
||||
url, _ = _resolve_string_media(direct, prefer_voice=prefer_voice)
|
||||
if url:
|
||||
out.append(url)
|
||||
for nested_key in _MEDIA_NESTED_KEYS:
|
||||
_collect_media_candidates(value.get(nested_key), out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
for nested in value.values():
|
||||
if isinstance(nested, (dict, list)):
|
||||
_collect_media_candidates(nested, out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
_collect_media_candidates(item, out, prefer_voice=prefer_voice, depth=depth + 1)
|
||||
|
||||
|
||||
def _pick_http_url(value: Any, *, prefer_voice: bool = False) -> str:
|
||||
candidates: list[str] = []
|
||||
_collect_media_candidates(value, candidates, prefer_voice=prefer_voice)
|
||||
return candidates[0] if candidates else ""
|
||||
|
||||
|
||||
def _pick_media_uri(value: Any) -> str:
|
||||
if isinstance(value, str) and _looks_like_media_uri(value) and not value.strip().startswith("http"):
|
||||
return _normalize_uri_path(value)
|
||||
if isinstance(value, dict):
|
||||
for key in ("uri", "local_path", "remote_url"):
|
||||
direct = value.get(key)
|
||||
if isinstance(direct, str) and _looks_like_media_uri(direct) and not direct.strip().startswith("http"):
|
||||
return _normalize_uri_path(direct)
|
||||
for nested_key in _MEDIA_NESTED_KEYS:
|
||||
uri = _pick_media_uri(value.get(nested_key))
|
||||
if uri:
|
||||
return uri
|
||||
for nested in value.values():
|
||||
if isinstance(nested, (dict, list)):
|
||||
uri = _pick_media_uri(nested)
|
||||
if uri:
|
||||
return uri
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
uri = _pick_media_uri(item)
|
||||
if uri:
|
||||
return uri
|
||||
return ""
|
||||
|
||||
|
||||
def _coerce_message_type(value: Any, default: int = MSG_TYPE_TEXT) -> int:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return default
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _parse_content_json(content_raw: str | dict | None) -> dict[str, Any]:
|
||||
if isinstance(content_raw, dict):
|
||||
data = content_raw
|
||||
else:
|
||||
raw = str(content_raw or "").strip()
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(raw)
|
||||
if isinstance(parsed, dict):
|
||||
data = parsed
|
||||
else:
|
||||
return {}
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
else:
|
||||
return {}
|
||||
|
||||
for nested_key in ("ext", "ai_ext", "extra", "payload", "data"):
|
||||
nested = data.get(nested_key)
|
||||
if isinstance(nested, str) and nested.strip().startswith("{"):
|
||||
try:
|
||||
nested_data = json.loads(nested)
|
||||
if isinstance(nested_data, dict):
|
||||
merged = {**nested_data, **data}
|
||||
data = merged
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _safe_int(value: Any) -> int | None:
|
||||
try:
|
||||
if value is None or value == "":
|
||||
return None
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def format_im_message(content_raw: str | dict | None, message_type: int = MSG_TYPE_TEXT) -> dict[str, Any]:
|
||||
"""将 IM 原始 content 解析为统一结构 {type, text, url, ...}。"""
|
||||
content_json = _parse_content_json(content_raw)
|
||||
embedded_type = _coerce_message_type(
|
||||
content_json.get("message_type")
|
||||
or content_json.get("messageType")
|
||||
or content_json.get("msg_type"),
|
||||
message_type,
|
||||
)
|
||||
if embedded_type != MSG_TYPE_TEXT:
|
||||
message_type = embedded_type
|
||||
|
||||
_EMPTY_MEDIA_DEFAULTS = {
|
||||
MSG_TYPE_IMAGE: ("image", "[图片]"),
|
||||
MSG_TYPE_STICKER: ("sticker", "[表情包]"),
|
||||
MSG_TYPE_VOICE: ("voice", "[语音]"),
|
||||
MSG_TYPE_VIDEO: ("video", "[视频]"),
|
||||
MSG_TYPE_LINK_CARD: ("link_card", "[链接卡片]"),
|
||||
}
|
||||
|
||||
if not content_json:
|
||||
raw = str(content_raw or "").strip()
|
||||
if not raw:
|
||||
# 抖音相册图片(type 27)/部分语音等会以「空 content」推送，URL 不随推送下发。
|
||||
# 不能直接丢弃，否则消息「收不到」；这里按类型返回占位，URL 留空待后续补取。
|
||||
if message_type in _EMPTY_MEDIA_DEFAULTS:
|
||||
t, txt = _EMPTY_MEDIA_DEFAULTS[message_type]
|
||||
return {"type": t, "text": txt}
|
||||
return {"type": "text", "text": ""}
|
||||
if raw in _PLACEHOLDER_MARKERS:
|
||||
mapping = {
|
||||
"[图片]": "image",
|
||||
"[表情包]": "sticker",
|
||||
"[语音]": "voice",
|
||||
"[视频]": "video",
|
||||
}
|
||||
return {"type": mapping.get(raw, "text"), "text": raw}
|
||||
if message_type == MSG_TYPE_TEXT:
|
||||
return {"type": "text", "text": raw}
|
||||
content_json = {"text": raw}
|
||||
|
||||
prefer_voice = message_type == MSG_TYPE_VOICE or bool(content_json.get("audio") or content_json.get("voice"))
|
||||
url = _pick_http_url(content_json, prefer_voice=prefer_voice)
|
||||
media_uri = _pick_media_uri(content_json)
|
||||
if not url and media_uri:
|
||||
url = resolve_media_uri(media_uri, prefer_voice=prefer_voice)
|
||||
duration = _safe_int(
|
||||
content_json.get("duration")
|
||||
or content_json.get("audio_duration")
|
||||
or content_json.get("video_duration")
|
||||
)
|
||||
width = _safe_int(content_json.get("width") or content_json.get("w"))
|
||||
height = _safe_int(content_json.get("height") or content_json.get("h"))
|
||||
|
||||
def _media_payload(msg_type: str, text: str, **extra: Any) -> dict[str, Any]:
|
||||
payload: dict[str, Any] = {"type": msg_type, "text": text}
|
||||
if url:
|
||||
payload["url"] = url
|
||||
elif media_uri:
|
||||
payload["uri"] = media_uri
|
||||
payload.update({k: v for k, v in extra.items() if v not in (None, "", 0)})
|
||||
return payload
|
||||
|
||||
# 语音/视频也会带 resource_url，不能仅凭该字段判为图片；优先按类型与 URL 特征识别。
|
||||
is_voice = (
|
||||
message_type == MSG_TYPE_VOICE
|
||||
or bool(content_json.get("audio") or content_json.get("voice"))
|
||||
or (url and _looks_like_audio_url(url))
|
||||
)
|
||||
is_video = (
|
||||
message_type == MSG_TYPE_VIDEO
|
||||
or bool(content_json.get("video"))
|
||||
or (url and _looks_like_video_url(url))
|
||||
)
|
||||
if is_voice and not is_video:
|
||||
return _media_payload("voice", "[语音]", duration=duration)
|
||||
if is_video:
|
||||
return _media_payload("video", "[视频]", duration=duration, width=width, height=height)
|
||||
|
||||
has_image_hint = (
|
||||
message_type == MSG_TYPE_IMAGE
|
||||
or content_json.get("image")
|
||||
or content_json.get("inline_pic")
|
||||
or (
|
||||
content_json.get("resource_url")
|
||||
and not (url and (_looks_like_audio_url(url) or _looks_like_video_url(url)))
|
||||
)
|
||||
)
|
||||
if has_image_hint:
|
||||
# 抖音相册私图(biz_tag=aweme_im)的大图 URL 是加密内容，浏览器无法直接渲染；
|
||||
# 但 content 内嵌 inline_pic（base64 WEBP 缩略图），直接转 data URI 即可显示。
|
||||
inline = content_json.get("inline_pic")
|
||||
if isinstance(inline, str) and inline.strip():
|
||||
b64 = re.sub(r"\s+", "", inline)
|
||||
data_uri = f"data:image/webp;base64,{b64}"
|
||||
payload = {"type": "image", "text": "[图片]", "url": data_uri}
|
||||
if width:
|
||||
payload["width"] = width
|
||||
if height:
|
||||
payload["height"] = height
|
||||
return payload
|
||||
return _media_payload("image", "[图片]", width=width, height=height)
|
||||
if message_type == MSG_TYPE_STICKER or content_json.get("static_url") or content_json.get("animate_url") or _valid_sticker_id(
|
||||
content_json.get("sticker_id") or content_json.get("id")
|
||||
):
|
||||
sticker_id = _valid_sticker_id(content_json.get("sticker_id") or content_json.get("id"))
|
||||
return _media_payload(
|
||||
"sticker",
|
||||
"[表情包]",
|
||||
sticker_id=sticker_id,
|
||||
name=str(content_json.get("display_name") or content_json.get("name") or ""),
|
||||
)
|
||||
|
||||
link_card = _parse_link_card_payload(content_json, message_type)
|
||||
if link_card:
|
||||
return link_card
|
||||
|
||||
rich_link = _parse_rich_text_link(content_json)
|
||||
if rich_link:
|
||||
return rich_link
|
||||
|
||||
text = (
|
||||
str(content_json.get("text") or content_json.get("content") or content_json.get("message") or "")
|
||||
).strip()
|
||||
if not text and url:
|
||||
inferred = _infer_media_type_from_url(url)
|
||||
if inferred == "voice":
|
||||
return {"type": "voice", "text": "[语音]", "url": url, "duration": duration}
|
||||
if inferred == "video":
|
||||
return {"type": "video", "text": "[视频]", "url": url, "duration": duration}
|
||||
if message_type == MSG_TYPE_IMAGE:
|
||||
return {"type": "image", "text": "[图片]", "url": url, "width": width, "height": height}
|
||||
if message_type == MSG_TYPE_STICKER:
|
||||
return {"type": "sticker", "text": "[表情包]", "url": url}
|
||||
if message_type == MSG_TYPE_VOICE:
|
||||
return {"type": "voice", "text": "[语音]", "url": url, "duration": duration}
|
||||
if message_type == MSG_TYPE_VIDEO:
|
||||
return {"type": "video", "text": "[视频]", "url": url, "duration": duration}
|
||||
|
||||
final_text = text or str(content_raw or "").strip()
|
||||
emoji = _resolve_text_emoji(final_text)
|
||||
if emoji:
|
||||
return emoji
|
||||
return {"type": "text", "text": final_text}
|
||||
|
||||
|
||||
def _parse_link_card_payload(content_json: dict[str, Any], message_type: int) -> dict[str, Any] | None:
|
||||
link_info = content_json.get("link_info")
|
||||
if not isinstance(link_info, dict):
|
||||
link_info = {}
|
||||
has_link = (
|
||||
message_type == MSG_TYPE_LINK_CARD
|
||||
or link_info
|
||||
or content_json.get("link_url")
|
||||
or content_json.get("cover_url")
|
||||
)
|
||||
if not has_link:
|
||||
return None
|
||||
title = str(content_json.get("title") or link_info.get("title") or "").strip()
|
||||
desc = str(
|
||||
content_json.get("desc")
|
||||
or content_json.get("description")
|
||||
or link_info.get("desc")
|
||||
or link_info.get("description")
|
||||
or ""
|
||||
).strip()
|
||||
url = str(
|
||||
content_json.get("link_url")
|
||||
or content_json.get("url")
|
||||
or link_info.get("url")
|
||||
or link_info.get("link_url")
|
||||
or ""
|
||||
).strip()
|
||||
cover = str(content_json.get("cover_url") or link_info.get("cover_url") or "").strip()
|
||||
text = title or desc or url or "[链接卡片]"
|
||||
payload: dict[str, Any] = {
|
||||
"type": "link_card",
|
||||
"text": text,
|
||||
"title": title,
|
||||
"desc": desc,
|
||||
}
|
||||
if url:
|
||||
payload["url"] = url
|
||||
if cover:
|
||||
payload["cover_url"] = cover
|
||||
return payload
|
||||
|
||||
|
||||
def _parse_rich_text_link(content_json: dict[str, Any]) -> dict[str, Any] | None:
|
||||
rich = content_json.get("richTextInfos")
|
||||
if not isinstance(rich, list):
|
||||
return None
|
||||
for item in rich:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
link = str(item.get("link") or item.get("url") or "").strip()
|
||||
if not link:
|
||||
continue
|
||||
text = str(item.get("text") or item.get("display_text") or link).strip()
|
||||
return {"type": "link", "text": text or link, "url": link}
|
||||
return None
|
||||
|
||||
|
||||
def _resolve_text_emoji(text: str) -> dict[str, Any] | None:
|
||||
"""文字表情 [酷拽] 等：查标准表情表，命中则转成可显示的贴纸。"""
|
||||
stripped = (text or "").strip()
|
||||
if not stripped or not (stripped.startswith("[") and stripped.endswith("]")):
|
||||
return None
|
||||
try:
|
||||
from .emoji_pack import looks_like_emoji_token, lookup_emoji_url
|
||||
|
||||
if not looks_like_emoji_token(stripped):
|
||||
return None
|
||||
url = lookup_emoji_url(stripped)
|
||||
if url:
|
||||
return {
|
||||
"type": "sticker",
|
||||
"text": stripped,
|
||||
"url": url,
|
||||
"name": stripped.strip("[]"),
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def parse_incoming_message(data: dict[str, Any]) -> str:
|
||||
"""从 IM API / WebSocket 消息 dict 提取并序列化展示内容。"""
|
||||
if not isinstance(data, dict):
|
||||
return str(data or "").strip()
|
||||
|
||||
msg_type = _coerce_message_type(
|
||||
data.get("message_type") or data.get("messageType") or data.get("msg_type"),
|
||||
MSG_TYPE_TEXT,
|
||||
)
|
||||
content_raw = (
|
||||
data.get("content")
|
||||
or data.get("message")
|
||||
or data.get("msg")
|
||||
or data.get("lastMessage")
|
||||
or data.get("last_msg")
|
||||
or data.get("preview")
|
||||
or data.get("brief")
|
||||
or ""
|
||||
)
|
||||
if isinstance(content_raw, dict):
|
||||
if not msg_type or msg_type == MSG_TYPE_TEXT:
|
||||
msg_type = _coerce_message_type(
|
||||
content_raw.get("message_type")
|
||||
or content_raw.get("messageType")
|
||||
or content_raw.get("msg_type"),
|
||||
msg_type,
|
||||
)
|
||||
parsed = format_im_message(content_raw, msg_type)
|
||||
return serialize_message_content(parsed)
|
||||
|
||||
if isinstance(content_raw, str):
|
||||
raw = content_raw.strip()
|
||||
if raw.startswith("{"):
|
||||
parsed = format_im_message(raw, msg_type)
|
||||
if parsed.get("type") != "text" or parsed.get("url") or msg_type != MSG_TYPE_TEXT:
|
||||
return serialize_message_content(parsed)
|
||||
if raw in _PLACEHOLDER_MARKERS and msg_type != MSG_TYPE_TEXT:
|
||||
parsed = format_im_message(raw, msg_type)
|
||||
return serialize_message_content(parsed)
|
||||
if msg_type != MSG_TYPE_TEXT:
|
||||
parsed = format_im_message(raw, msg_type)
|
||||
return serialize_message_content(parsed)
|
||||
return raw
|
||||
|
||||
return ""
|
||||
|
||||
|
||||
def serialize_message_content(msg: dict[str, Any]) -> str:
|
||||
"""序列化写入 message_logs / reply_content。"""
|
||||
msg_type = (msg.get("type") or "text").strip()
|
||||
if msg_type == "text":
|
||||
text = str(msg.get("text") or "").strip()
|
||||
return text
|
||||
cleaned = {k: v for k, v in msg.items() if v not in (None, "", [], {})}
|
||||
return json.dumps(cleaned, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def parse_stored_content(raw: str | None) -> dict[str, Any]:
|
||||
"""解析数据库中的 message_content / reply_content。"""
|
||||
text = (raw or "").strip()
|
||||
if not text:
|
||||
return {"type": "text", "text": ""}
|
||||
if text.startswith("{"):
|
||||
try:
|
||||
data = json.loads(text)
|
||||
if isinstance(data, dict) and data.get("type"):
|
||||
if not data.get("url") and data.get("uri"):
|
||||
prefer_voice = data.get("type") == "voice"
|
||||
resolved = resolve_media_uri(str(data["uri"]), prefer_voice=prefer_voice)
|
||||
if resolved:
|
||||
data = {**data, "url": resolved}
|
||||
url = str(data.get("url") or "").strip()
|
||||
if url:
|
||||
inferred = _infer_media_type_from_url(url)
|
||||
current = data.get("type")
|
||||
if current == "image" and inferred in ("voice", "video"):
|
||||
data = {
|
||||
**data,
|
||||
"type": inferred,
|
||||
"text": "[语音]" if inferred == "voice" else "[视频]",
|
||||
}
|
||||
elif current not in ("voice", "video", "image", "sticker") and inferred:
|
||||
data = {**data, "type": inferred}
|
||||
return data
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if text in _PLACEHOLDER_MARKERS:
|
||||
mapping = {
|
||||
"[图片]": "image",
|
||||
"[表情包]": "sticker",
|
||||
"[语音]": "voice",
|
||||
"[视频]": "video",
|
||||
}
|
||||
return {"type": mapping.get(text, "text"), "text": text}
|
||||
return {"type": "text", "text": text}
|
||||
|
||||
|
||||
def message_preview(raw: str | None) -> str:
|
||||
"""会话列表/日志摘要。"""
|
||||
msg = parse_stored_content(raw)
|
||||
msg_type = msg.get("type") or "text"
|
||||
if msg_type == "text":
|
||||
return str(msg.get("text") or "")
|
||||
label = _TYPE_LABELS.get(msg_type, msg.get("text") or "[消息]")
|
||||
extra = str(msg.get("name") or "").strip()
|
||||
if extra and msg_type == "sticker":
|
||||
return f"[表情] {extra}"
|
||||
return str(msg.get("text") or label)
|
||||
|
||||
|
||||
def normalize_outgoing_content(
|
||||
content: str = "",
|
||||
message_type: str | None = None,
|
||||
media_url: str | None = None,
|
||||
sticker_url: str | None = None,
|
||||
width: int | None = None,
|
||||
height: int | None = None,
|
||||
sticker_id: str | None = None,
|
||||
) -> str:
|
||||
"""构造可发送/可落库的 content 字符串。"""
|
||||
raw = (content or "").strip()
|
||||
parsed_json: dict[str, Any] | None = None
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and data.get("type"):
|
||||
parsed_json = data
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if parsed_json:
|
||||
merged = dict(parsed_json)
|
||||
if width and not merged.get("width"):
|
||||
merged["width"] = width
|
||||
if height and not merged.get("height"):
|
||||
merged["height"] = height
|
||||
if sticker_id and not merged.get("sticker_id"):
|
||||
merged["sticker_id"] = sticker_id
|
||||
url = (media_url or sticker_url or "").strip()
|
||||
if url and not merged.get("url"):
|
||||
merged["url"] = url
|
||||
return serialize_message_content(merged)
|
||||
|
||||
explicit_type = (message_type or "").strip().lower()
|
||||
if explicit_type in ("image", "sticker", "voice", "video", "text"):
|
||||
if explicit_type == "text":
|
||||
return (content or "").strip()
|
||||
payload: dict[str, Any] = {"type": explicit_type}
|
||||
url = (media_url or sticker_url or "").strip()
|
||||
if url:
|
||||
payload["url"] = url
|
||||
if explicit_type == "sticker" and sticker_id:
|
||||
payload["sticker_id"] = sticker_id
|
||||
if width:
|
||||
payload["width"] = width
|
||||
if height:
|
||||
payload["height"] = height
|
||||
text = (content or "").strip()
|
||||
if text:
|
||||
payload["text"] = text
|
||||
elif explicit_type == "image":
|
||||
payload["text"] = "[图片]"
|
||||
elif explicit_type == "sticker":
|
||||
payload["text"] = "[表情包]"
|
||||
return serialize_message_content(payload)
|
||||
|
||||
raw = (content or "").strip()
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and data.get("type"):
|
||||
return serialize_message_content(data)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return raw
|
||||
|
||||
|
||||
def is_media_message(raw: str | None) -> bool:
|
||||
return parse_stored_content(raw).get("type") not in (None, "text")
|
||||
|
||||
|
||||
def format_system_log_message(raw: str | None) -> str:
|
||||
"""系统诊断日志中的消息摘要。"""
|
||||
msg = parse_stored_content(raw)
|
||||
msg_type = msg.get("type") or "text"
|
||||
if msg_type == "text":
|
||||
return str(msg.get("text") or "")
|
||||
parts = [message_preview(raw)]
|
||||
url = str(msg.get("url") or "").strip()
|
||||
if url:
|
||||
parts.append(f"URL: {url}")
|
||||
duration = msg.get("duration")
|
||||
if duration:
|
||||
parts.append(f"时长: {duration}s")
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def extract_urls_from_detail(detail: str | None) -> list[str]:
|
||||
if not detail:
|
||||
return []
|
||||
return re.findall(r"https?://[^\s\]|）)\"']+", detail)
|
||||
@@ -0,0 +1,258 @@
|
||||
"""极简 protobuf wire 解码器（无第三方依赖）。
|
||||
|
||||
用于解析抖音 IM 发送私信的响应：官方 Response.proto 只建模了
|
||||
create/get_info/new_message_notify 三种 body，没有“发送消息响应”，
|
||||
导致仅凭 error_desc 为空就误判为发送成功。这里直接按 wire 格式解码，
|
||||
读取真实的 status_code / server_message_id，判定是否真的投递成功。
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
def _format_status_json(status_json: dict) -> str:
|
||||
"""把内嵌 status JSON 格式化为更可读的失败原因。"""
|
||||
code = status_json.get("status_code")
|
||||
raw_check = status_json.get("raw_check_code")
|
||||
decision = status_json.get("decision_type")
|
||||
parts = [f"status_code={code}"]
|
||||
if raw_check is not None:
|
||||
parts.append(f"raw_check_code={raw_check}")
|
||||
if decision:
|
||||
parts.append(f"decision_type={decision}")
|
||||
return "；".join(parts)
|
||||
|
||||
|
||||
def _read_varint(buf: bytes, i: int) -> tuple[int, int]:
|
||||
shift = 0
|
||||
result = 0
|
||||
n = len(buf)
|
||||
while i < n:
|
||||
b = buf[i]
|
||||
i += 1
|
||||
result |= (b & 0x7F) << shift
|
||||
if not (b & 0x80):
|
||||
return result, i
|
||||
shift += 7
|
||||
if shift > 70:
|
||||
break
|
||||
raise ValueError("truncated varint")
|
||||
|
||||
|
||||
def decode_fields(buf: bytes) -> list[tuple[int, int, Any]]:
|
||||
"""返回 [(field_num, wire_type, value), ...]。
|
||||
|
||||
wire_type: 0=varint(int), 1=64bit(int), 2=length-delimited(bytes), 5=32bit(int)
|
||||
"""
|
||||
out: list[tuple[int, int, Any]] = []
|
||||
i = 0
|
||||
n = len(buf)
|
||||
while i < n:
|
||||
key, i = _read_varint(buf, i)
|
||||
field = key >> 3
|
||||
wt = key & 7
|
||||
if wt == 0:
|
||||
val, i = _read_varint(buf, i)
|
||||
out.append((field, wt, val))
|
||||
elif wt == 2:
|
||||
ln, i = _read_varint(buf, i)
|
||||
val = buf[i:i + ln]
|
||||
i += ln
|
||||
out.append((field, wt, val))
|
||||
elif wt == 5:
|
||||
val = int.from_bytes(buf[i:i + 4], "little")
|
||||
i += 4
|
||||
out.append((field, wt, val))
|
||||
elif wt == 1:
|
||||
val = int.from_bytes(buf[i:i + 8], "little")
|
||||
i += 8
|
||||
out.append((field, wt, val))
|
||||
else:
|
||||
raise ValueError(f"unsupported wire type {wt}")
|
||||
return out
|
||||
|
||||
|
||||
def _collect_big_varints(buf: bytes, acc: list[int], depth: int = 0) -> None:
|
||||
"""递归收集疑似 ID 的大整数（server_message_id / short_id 等都是大数）。"""
|
||||
if depth > 6:
|
||||
return
|
||||
try:
|
||||
fields = decode_fields(buf)
|
||||
except Exception:
|
||||
return
|
||||
for _field, wt, val in fields:
|
||||
if wt == 0 and isinstance(val, int) and val > 10 ** 12:
|
||||
acc.append(val)
|
||||
elif wt == 2 and isinstance(val, (bytes, bytearray)) and val:
|
||||
_collect_big_varints(bytes(val), acc, depth + 1)
|
||||
|
||||
|
||||
def _extract_status_json(raw: bytes) -> Optional[dict]:
|
||||
"""抖音发送响应的 body 内嵌一段 JSON：{"status_code":x,"tips":"...","status_msg":{...}}。
|
||||
|
||||
这才是“消息是否真正投递”的权威结论（status_code=0 才是真成功）。
|
||||
顶层 message=OK 只是接口层面的“已受理”，不代表已投递。
|
||||
"""
|
||||
marker = b'"status_code"'
|
||||
idx = raw.find(marker)
|
||||
if idx < 0:
|
||||
return None
|
||||
start = raw.rfind(b"{", 0, idx)
|
||||
if start < 0:
|
||||
return None
|
||||
depth = 0
|
||||
in_str = False
|
||||
esc = False
|
||||
end = -1
|
||||
for i in range(start, len(raw)):
|
||||
c = raw[i]
|
||||
if in_str:
|
||||
if esc:
|
||||
esc = False
|
||||
elif c == 0x5C: # backslash
|
||||
esc = True
|
||||
elif c == 0x22: # quote
|
||||
in_str = False
|
||||
continue
|
||||
if c == 0x22:
|
||||
in_str = True
|
||||
elif c == 0x7B: # {
|
||||
depth += 1
|
||||
elif c == 0x7D: # }
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
end = i + 1
|
||||
break
|
||||
if end < 0:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw[start:end].decode("utf-8", "ignore"))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def analyze_send_response(raw: bytes) -> dict:
|
||||
"""分析“发送私信”的 protobuf 响应，判定是否真正投递成功。
|
||||
|
||||
顶层 Response 字段（见 Response.proto）：
|
||||
1=cmd, 2=sequence_id, 3=error_desc(string), 4=message(string),
|
||||
5=inbox_type, 6=body(ResponseBody)。
|
||||
|
||||
注意：顶层没有 status_code 字段。真正“消息已写入服务端”的标志是
|
||||
body(field 6) 里带有服务端分配的 server_message_id（大整数）。
|
||||
sequence_id(field 2) 也是大整数，因此只在 body 内部查找 message_id，
|
||||
避免把 sequence_id 误当成投递成功标志。
|
||||
|
||||
返回:
|
||||
ok: 是否真正发送成功（body 内带服务端 message_id，或 message=OK 且有 body）
|
||||
cmd: 顶层 cmd
|
||||
message: 顶层 message 文本（field 4）
|
||||
error_desc: 顶层 error_desc 文本（field 3）
|
||||
server_message_id: body 内服务端消息 ID（投递成功的强信号）
|
||||
has_body: 是否带 body
|
||||
summary: 顶层字段概览 + hex 片段，便于排查
|
||||
"""
|
||||
info = {
|
||||
"ok": False,
|
||||
"cmd": None,
|
||||
"status": None,
|
||||
"status_code": None,
|
||||
"raw_check_code": None,
|
||||
"delivered_with_notice": False,
|
||||
"status_reason": "",
|
||||
"message": "",
|
||||
"error_desc": "",
|
||||
"server_message_id": None,
|
||||
"has_body": False,
|
||||
"summary": "",
|
||||
}
|
||||
if not raw:
|
||||
info["summary"] = "空响应"
|
||||
return info
|
||||
try:
|
||||
fields = decode_fields(raw)
|
||||
except Exception as e:
|
||||
info["summary"] = f"解码失败: {e}; hex={raw[:120].hex()}"
|
||||
return info
|
||||
|
||||
body = None
|
||||
parts = []
|
||||
for field, wt, val in fields:
|
||||
if field == 1 and wt == 0:
|
||||
info["cmd"] = val
|
||||
elif field == 3 and wt == 2:
|
||||
try:
|
||||
info["error_desc"] = bytes(val).decode("utf-8", "ignore")
|
||||
except Exception:
|
||||
pass
|
||||
elif field == 4 and wt == 2:
|
||||
try:
|
||||
info["message"] = bytes(val).decode("utf-8", "ignore")
|
||||
except Exception:
|
||||
pass
|
||||
elif field == 6 and wt == 2:
|
||||
body = bytes(val)
|
||||
info["has_body"] = len(body) > 0
|
||||
|
||||
if wt == 0:
|
||||
parts.append(f"{field}=int:{val}")
|
||||
elif wt == 2:
|
||||
parts.append(f"{field}=bytes[{len(val)}]")
|
||||
else:
|
||||
parts.append(f"{field}={val}")
|
||||
info["summary"] = " ".join(parts) + f" | hex={raw[:120].hex()}"
|
||||
|
||||
# 只在 body 内部查找服务端 message_id（避免误用顶层 sequence_id）
|
||||
if body:
|
||||
ids: list[int] = []
|
||||
_collect_big_varints(body, ids)
|
||||
if ids:
|
||||
info["server_message_id"] = max(ids)
|
||||
|
||||
# 权威结论：body 内嵌 JSON 的 status_code（0 才是真成功）
|
||||
status_json = _extract_status_json(raw)
|
||||
if status_json is not None:
|
||||
info["status_code"] = status_json.get("status_code")
|
||||
info["raw_check_code"] = status_json.get("raw_check_code")
|
||||
tips = (status_json.get("tips") or "").strip()
|
||||
status_msg = status_json.get("status_msg")
|
||||
msg_text = ""
|
||||
if isinstance(status_msg, dict):
|
||||
# 抖音把人类可读提示放在 status_msg.msg_content.tips
|
||||
mc = status_msg.get("msg_content")
|
||||
if isinstance(mc, dict):
|
||||
msg_text = (mc.get("tips") or mc.get("content") or "").strip()
|
||||
if not msg_text:
|
||||
msg_text = (
|
||||
status_msg.get("toast")
|
||||
or status_msg.get("content")
|
||||
or status_msg.get("msg")
|
||||
or ""
|
||||
)
|
||||
elif isinstance(status_msg, str):
|
||||
msg_text = status_msg
|
||||
info["status_reason"] = tips or msg_text or _format_status_json(status_json)
|
||||
|
||||
msg_ok = info["message"].strip().upper() == "OK"
|
||||
|
||||
# 优先用 status_code 判定：明确给了 status_code 就以它为准（0=成功，非0=另判）
|
||||
if info["status_code"] is not None:
|
||||
if info["status_code"] == 0 and not info["error_desc"]:
|
||||
info["ok"] = True
|
||||
elif info["raw_check_code"] == 0 and msg_ok and not info["error_desc"]:
|
||||
# raw_check_code=0 表示已通过抖音风控/安全校验；配合 message=OK，
|
||||
# 说明消息已实际投递。此时非零 status_code 只是“业务侧提示”
|
||||
# （如营销/陌生人限制提醒），对方仍能收到，不应判为发送失败。
|
||||
info["ok"] = True
|
||||
info["delivered_with_notice"] = True
|
||||
else:
|
||||
# raw_check_code=1（被风控拦截）或缺少 OK 标志：判为未送达
|
||||
info["ok"] = False
|
||||
else:
|
||||
# 没有内嵌 status_code 时，退回“message=OK 且 body 内有服务端 message_id”
|
||||
info["ok"] = bool(
|
||||
not info["error_desc"]
|
||||
and msg_ok
|
||||
and info["server_message_id"] is not None
|
||||
)
|
||||
return info
|
||||
@@ -0,0 +1,225 @@
|
||||
"""私信对方用户资料抓取（昵称 / 头像 / UID）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
|
||||
from rpa_engine.device_profiles import resolve_user_agent
|
||||
from .auth import DouyinAuth
|
||||
from .conv_util import resolve_peer_uid
|
||||
from .dy_util import (
|
||||
DEFAULT_USER_AGENT,
|
||||
generate_a_bogus,
|
||||
generate_msToken,
|
||||
generate_webid,
|
||||
splice_url,
|
||||
)
|
||||
from .protocol import _pick_avatar_url
|
||||
from .session import DouyinImSession
|
||||
|
||||
logger = logging.getLogger("douyin_im.peer_profile")
|
||||
|
||||
_profile_cache: dict[str, dict[str, str]] = {}
|
||||
_profile_cache_at: dict[str, float] = {}
|
||||
_PROFILE_SUCCESS_TTL = 6 * 3600
|
||||
_PROFILE_FAILURE_TTL = 5 * 60
|
||||
|
||||
|
||||
def _cache_key(account_id: int, peer_uid: str) -> str:
|
||||
return f"{account_id}:{peer_uid}"
|
||||
|
||||
|
||||
def _pick_str(data: dict, *keys: str) -> str:
|
||||
for key in keys:
|
||||
value = data.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_profile_from_payload(data: Any) -> dict[str, str]:
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
nodes = [data, data.get("user"), data.get("user_info"), data.get("data")]
|
||||
for node in nodes:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
uid = _pick_str(node, "uid", "user_id", "user_uid", "id")
|
||||
nickname = _pick_str(
|
||||
node,
|
||||
"nickname",
|
||||
"nick_name",
|
||||
"unique_id",
|
||||
"display_name",
|
||||
"name",
|
||||
)
|
||||
avatar = _pick_avatar_url(node)
|
||||
if uid or nickname or avatar:
|
||||
return {"uid": uid, "nickname": nickname, "avatar_url": avatar}
|
||||
return {}
|
||||
|
||||
|
||||
def _requests_proxies() -> dict | None:
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
|
||||
return requests_proxies()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _build_auth(session: DouyinImSession) -> tuple[DouyinAuth, str]:
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||||
ua = resolve_user_agent(session.user_agent or DEFAULT_USER_AGENT)
|
||||
return auth, ua
|
||||
|
||||
|
||||
def is_generic_peer_name(name: str, peer_uid: str = "") -> bool:
|
||||
value = (name or "").strip()
|
||||
if not value:
|
||||
return True
|
||||
if peer_uid and value == peer_uid:
|
||||
return True
|
||||
if value.isdigit():
|
||||
return True
|
||||
if value.startswith("用户") and value[2:].isdigit():
|
||||
return True
|
||||
if value.startswith("会话") and len(value) <= 16:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def fetch_peer_profile_sync(
|
||||
session: DouyinImSession,
|
||||
peer_uid: int | str,
|
||||
account_id: int = 0,
|
||||
) -> dict[str, str]:
|
||||
uid = str(peer_uid or "").strip()
|
||||
if not uid.isdigit():
|
||||
return {}
|
||||
|
||||
cache_key = _cache_key(account_id, uid)
|
||||
cached = _profile_cache.get(cache_key)
|
||||
cached_at = _profile_cache_at.get(cache_key, 0.0)
|
||||
if cached:
|
||||
ttl = (
|
||||
_PROFILE_SUCCESS_TTL
|
||||
if cached.get("nickname") or cached.get("avatar_url")
|
||||
else _PROFILE_FAILURE_TTL
|
||||
)
|
||||
if time.time() - cached_at < ttl:
|
||||
return dict(cached)
|
||||
|
||||
result = {"uid": uid, "nickname": "", "avatar_url": ""}
|
||||
try:
|
||||
auth, ua = _build_auth(session)
|
||||
except Exception as exc:
|
||||
logger.warning(f"build auth for peer profile failed: {exc}")
|
||||
_profile_cache[cache_key] = dict(result)
|
||||
_profile_cache_at[cache_key] = time.time()
|
||||
return result
|
||||
|
||||
try:
|
||||
web_id = session.web_id or generate_webid(auth, "https://www.douyin.com/")
|
||||
if web_id and not session.web_id:
|
||||
# Reuse the homepage-derived ID for every peer on this account.
|
||||
session.web_id = str(web_id)
|
||||
except Exception as exc:
|
||||
logger.debug(f"generate webid for peer profile failed: {exc}")
|
||||
web_id = session.web_id or ""
|
||||
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
base_params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"publish_video_strategy_type": "2",
|
||||
"user_id": uid,
|
||||
"sec_user_id": "",
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"webid": web_id,
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
}
|
||||
endpoints = [
|
||||
"https://www.douyin.com/aweme/v1/web/user/profile/other/",
|
||||
"https://www.douyin.com/aweme/v1/web/im/user/info/",
|
||||
]
|
||||
|
||||
proxies = _requests_proxies()
|
||||
for url in endpoints:
|
||||
try:
|
||||
params = dict(base_params)
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
resp = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
verify=False,
|
||||
timeout=12,
|
||||
proxies=proxies,
|
||||
)
|
||||
data = resp.json()
|
||||
extracted = _extract_profile_from_payload(data)
|
||||
if extracted.get("uid") and not result["uid"]:
|
||||
result["uid"] = extracted["uid"]
|
||||
if extracted.get("nickname") and not result["nickname"]:
|
||||
result["nickname"] = extracted["nickname"]
|
||||
if extracted.get("avatar_url") and not result["avatar_url"]:
|
||||
result["avatar_url"] = extracted["avatar_url"]
|
||||
if result["nickname"] and result["avatar_url"]:
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.debug(f"peer profile fetch failed for {url}: {exc}")
|
||||
|
||||
# Cache both success and failure. Without a short negative TTL, missing or
|
||||
# rate-limited profiles were fetched again for every account poll.
|
||||
_profile_cache[cache_key] = dict(result)
|
||||
_profile_cache_at[cache_key] = time.time()
|
||||
return result
|
||||
|
||||
|
||||
async def fetch_peer_profile(
|
||||
session: DouyinImSession,
|
||||
peer_uid: int | str,
|
||||
account_id: int = 0,
|
||||
) -> dict[str, str]:
|
||||
from .traffic_control import get_traffic_controller
|
||||
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(account_id, "peer profile"):
|
||||
return await asyncio.to_thread(
|
||||
fetch_peer_profile_sync,
|
||||
session,
|
||||
peer_uid,
|
||||
account_id,
|
||||
)
|
||||
|
||||
|
||||
def enrich_conversation_item(conv: dict, my_uid: int = 0) -> dict:
|
||||
"""补全会话项中的 peer_uid / sender_id。"""
|
||||
item = dict(conv or {})
|
||||
conv_id = str(item.get("conversation_id") or "").strip()
|
||||
peer_uid = str(item.get("peer_uid") or item.get("sender_id") or "").strip()
|
||||
if (not peer_uid or not peer_uid.isdigit()) and conv_id and my_uid:
|
||||
resolved = resolve_peer_uid(conv_id, int(my_uid))
|
||||
if resolved:
|
||||
peer_uid = str(resolved)
|
||||
if peer_uid:
|
||||
item["peer_uid"] = peer_uid
|
||||
item["sender_id"] = peer_uid
|
||||
elif conv_id:
|
||||
item["sender_id"] = conv_id
|
||||
return item
|
||||
@@ -0,0 +1,120 @@
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
|
||||
from .static import Request_pb2 as RequestProto
|
||||
from .dy_util import (
|
||||
generate_webid,
|
||||
generate_req_sign,
|
||||
generate_millisecond,
|
||||
normalize_client_cert,
|
||||
DEFAULT_USER_AGENT,
|
||||
)
|
||||
|
||||
|
||||
def _ua_headers(auth) -> tuple[str, str]:
|
||||
ua = getattr(auth, "user_agent", None) or DEFAULT_USER_AGENT
|
||||
browser_version = ua.split("Mozilla/", 1)[-1] if "Mozilla/" in ua else ua
|
||||
return ua, browser_version
|
||||
|
||||
|
||||
class ProtoBuilder:
|
||||
@staticmethod
|
||||
def build_normal_request(auth, cmd):
|
||||
ua, browser_version = _ua_headers(auth)
|
||||
request = RequestProto.Request()
|
||||
request.cmd = cmd
|
||||
request.sequence_id = random.randint(10000, 11000)
|
||||
request.sdk_version = "1.1.3"
|
||||
request.token = auth.ticket if auth.ticket else ""
|
||||
request.refer = 3
|
||||
request.inbox_type = 0
|
||||
request.build_number = "5fa6ff1:Detached: 5fa6ff1111fd53aafc4c753505d3c93daad74d27"
|
||||
did = str(getattr(auth, "device_id", "") or "0")
|
||||
request.device_id = did
|
||||
request.device_platform = 'douyin_pc'
|
||||
request.headers['session_aid'] = '6383'
|
||||
request.headers['session_did'] = did
|
||||
request.headers['app_name'] = 'douyin_pc'
|
||||
request.headers['priority_region'] = 'cn'
|
||||
request.headers['user_agent'] = ua
|
||||
request.headers['cookie_enabled'] = 'true'
|
||||
request.headers['browser_language'] = 'zh-CN'
|
||||
request.headers['browser_platform'] = 'Win32'
|
||||
request.headers['browser_name'] = 'Mozilla'
|
||||
request.headers['browser_version'] = browser_version
|
||||
request.headers['browser_online'] = 'true'
|
||||
request.headers['screen_width'] = '1707'
|
||||
request.headers['screen_height'] = '960'
|
||||
request.headers['referer'] = ''
|
||||
request.headers['timezone_name'] = 'Etc/GMT-8'
|
||||
request.headers['deviceId'] = did
|
||||
request.headers['webid'] = generate_webid(auth)
|
||||
request.headers['fp'] = auth.cookie.get('s_v_web_id', '') if auth.cookie else ''
|
||||
request.headers['is-retry'] = '0'
|
||||
request.auth_type = 4
|
||||
request.biz = 'douyin_web'
|
||||
request.access = 'web_sdk'
|
||||
request.ts_sign = auth.ts_sign if auth.ts_sign else ""
|
||||
request.sdk_cert = normalize_client_cert(auth.client_cert or "")
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def build_create_conversation_request(auth, toId, myId):
|
||||
request = ProtoBuilder.build_normal_request(auth, 609)
|
||||
request.body.create_conversation_v2_body.conversation_type = 1
|
||||
request.body.create_conversation_v2_body.participants.extend([int(toId), int(myId)])
|
||||
reuqest_sign = generate_req_sign({
|
||||
"sign_data": f"avatar_url=&idempotent_id=&name=&participants={toId},{myId}",
|
||||
"certType": "cookie",
|
||||
"scene": "web_protect"
|
||||
}, auth.private_key)
|
||||
request.reuqest_sign = reuqest_sign
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def build_get_conversation_list_info_request(auth, toId, myId, conversation_short_id):
|
||||
request = ProtoBuilder.build_normal_request(auth, 610)
|
||||
request.body.get_conversation_info_list_v2_body.data.conversation_id = f"0:1:{myId}:{toId}"
|
||||
request.body.get_conversation_info_list_v2_body.data.conversation_short_id = int(conversation_short_id)
|
||||
request.body.get_conversation_info_list_v2_body.data.conversation_type = 1
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
def build_send_message_request(
|
||||
auth,
|
||||
conversation_id,
|
||||
conversation_short_id,
|
||||
ticket,
|
||||
msg_content,
|
||||
message_type: int = 7,
|
||||
):
|
||||
client_message_id = str(uuid.uuid4())
|
||||
request = ProtoBuilder.build_normal_request(auth, 100)
|
||||
request.body.send_message_body.conversation_id = conversation_id
|
||||
request.body.send_message_body.conversation_type = 1
|
||||
request.body.send_message_body.conversation_short_id = int(conversation_short_id)
|
||||
request.body.send_message_body.content = json.dumps(msg_content, ensure_ascii=False,
|
||||
separators=(',', ':'))
|
||||
request.body.send_message_body.ext.append(
|
||||
RequestProto.ExtValue(key='s:client_message_id', value=client_message_id)
|
||||
)
|
||||
request.body.send_message_body.ext.append(
|
||||
RequestProto.ExtValue(key='s:stime', value=str(generate_millisecond()))
|
||||
)
|
||||
request.body.send_message_body.ext.append(
|
||||
RequestProto.ExtValue(key='s:mentioned_users', value='')
|
||||
)
|
||||
request.body.send_message_body.message_type = int(message_type)
|
||||
request.body.send_message_body.ticket = ticket
|
||||
request.body.send_message_body.client_message_id = client_message_id
|
||||
|
||||
# 签名数据计算
|
||||
sign_data_str = f'content={json.dumps(msg_content, separators=(",", ":"), ensure_ascii=False)}' + f'&conversation_id={conversation_id}&conversation_short_id={conversation_short_id}'
|
||||
req_sign = generate_req_sign({
|
||||
"sign_data": sign_data_str,
|
||||
"certType": "cookie",
|
||||
"scene": "web_protect"
|
||||
}, auth.private_key)
|
||||
request.reuqest_sign = req_sign
|
||||
return request
|
||||
@@ -0,0 +1,518 @@
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Optional
|
||||
|
||||
from .message_content import (
|
||||
MSG_TYPE_IMAGE,
|
||||
MSG_TYPE_LINK_CARD,
|
||||
MSG_TYPE_STICKER,
|
||||
MSG_TYPE_TEXT,
|
||||
MSG_TYPE_VIDEO,
|
||||
MSG_TYPE_VOICE,
|
||||
_coerce_message_type,
|
||||
format_im_message,
|
||||
message_preview,
|
||||
parse_incoming_message,
|
||||
parse_stored_content,
|
||||
serialize_message_content,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("douyin_im.protocol")
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def _is_control_payload(content_json: Any, msg_type: int = 0) -> bool:
|
||||
"""判断是否为「会话控制/状态更新」等非聊天内容帧。
|
||||
|
||||
例如 command_type=6 的连续互动统计(consecutive_chat_data)、ext_data 元数据更新、
|
||||
message_type>=50000 的系统通知等——这些不是用户发的消息，不应记录/展示成聊天气泡，
|
||||
更不应触发自动回复。
|
||||
"""
|
||||
try:
|
||||
if msg_type and int(msg_type) >= 50000:
|
||||
return True
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if isinstance(content_json, dict) and ("command_type" in content_json or "ext_data" in content_json):
|
||||
return True
|
||||
return False
|
||||
|
||||
_WS_DEBUG_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "ws_media_debug.log")
|
||||
|
||||
|
||||
def _should_emit_ws_message(
|
||||
conversation_id: str,
|
||||
msg_type: int,
|
||||
) -> bool:
|
||||
"""判断 WS 帧是否为用户聊天消息（控制帧已在 _is_control_payload 过滤）。"""
|
||||
if not conversation_id:
|
||||
return False
|
||||
try:
|
||||
if int(msg_type) >= 50000:
|
||||
return False
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def _dump_ws_message(msg_type: int, conversation_id: str, content_str: str, msg: Any = None) -> None:
|
||||
"""把每条 WS 消息的全部字段落到调试文件，便于排查媒体字段结构。
|
||||
|
||||
默认关闭，仅当设置环境变量 KEFU_WS_DEBUG=1 时写盘，避免生产环境无界增长 / 泄露聊天内容。
|
||||
content 为空时（如 type=26 瘦推送）会额外打印 protobuf 其余字段，确保「接收到的全部信息」可见。
|
||||
"""
|
||||
if os.getenv("KEFU_WS_DEBUG", "") not in ("1", "true", "True"):
|
||||
return
|
||||
try:
|
||||
extra = ""
|
||||
if msg is not None:
|
||||
fields = {}
|
||||
try:
|
||||
for f, v in msg.ListFields():
|
||||
if f.name == "content":
|
||||
continue
|
||||
fields[f.name] = v
|
||||
except Exception:
|
||||
pass
|
||||
if fields:
|
||||
extra = " | fields=" + json.dumps(fields, ensure_ascii=False, default=str)
|
||||
line = (
|
||||
f"{datetime.now().isoformat()} type={msg_type} "
|
||||
f"conv={conversation_id} content={content_str}{extra}\n"
|
||||
)
|
||||
with open(_WS_DEBUG_PATH, "a", encoding="utf-8") as fh:
|
||||
fh.write(line)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _walk_strings(data: Any, depth: int = 0, max_depth: int = 10):
|
||||
if depth > max_depth:
|
||||
return
|
||||
if isinstance(data, dict):
|
||||
for v in data.values():
|
||||
yield from _walk_strings(v, depth + 1, max_depth)
|
||||
elif isinstance(data, list):
|
||||
for item in data:
|
||||
yield from _walk_strings(item, depth + 1, max_depth)
|
||||
elif isinstance(data, str) and data.strip():
|
||||
yield data.strip()
|
||||
|
||||
|
||||
def extract_json_objects(raw: bytes | str) -> list[dict]:
|
||||
"""从二进制帧中尽量提取 JSON 对象"""
|
||||
if isinstance(raw, bytes):
|
||||
for codec in ("utf-8", "latin-1"):
|
||||
try:
|
||||
text = raw.decode(codec, errors="ignore")
|
||||
break
|
||||
except Exception:
|
||||
text = ""
|
||||
else:
|
||||
text = ""
|
||||
else:
|
||||
text = raw
|
||||
|
||||
results = []
|
||||
for match in re.finditer(r"\{[^{}]{0,2000}\}", text):
|
||||
chunk = match.group(0)
|
||||
try:
|
||||
obj = json.loads(chunk)
|
||||
if isinstance(obj, dict):
|
||||
results.append(obj)
|
||||
except Exception:
|
||||
continue
|
||||
return results
|
||||
|
||||
|
||||
def parse_ws_payload(raw: bytes | str) -> list[dict]:
|
||||
"""解析 WebSocket 二进制帧，返回标准化消息 dict 列表"""
|
||||
messages = []
|
||||
|
||||
# 尝试 Protobuf 解包
|
||||
if isinstance(raw, bytes):
|
||||
try:
|
||||
from .static import Live_pb2, Response_pb2
|
||||
frame = Live_pb2.PushFrame()
|
||||
frame.ParseFromString(raw)
|
||||
if frame.payloadType == 'pb':
|
||||
response = Response_pb2.Response()
|
||||
response.ParseFromString(frame.payload)
|
||||
body = response.body
|
||||
if body.HasField("new_message_notify"):
|
||||
notify = body.new_message_notify
|
||||
if notify.HasField("message"):
|
||||
msg = notify.message
|
||||
sender = str(msg.sender)
|
||||
msg_type = msg.message_type
|
||||
conversation_id = msg.conversation_id
|
||||
content_str = msg.content
|
||||
server_message_id = str(getattr(msg, "server_message_id", "") or "")
|
||||
|
||||
_dump_ws_message(msg_type, conversation_id, content_str, msg)
|
||||
|
||||
text_content = ""
|
||||
media_msg: dict = {}
|
||||
content_json: dict = {}
|
||||
try:
|
||||
content_json = json.loads(content_str) if content_str else {}
|
||||
if not isinstance(content_json, dict):
|
||||
content_json = {}
|
||||
media_msg = format_im_message(content_json, msg_type)
|
||||
text_content = media_msg.get("text") or ""
|
||||
except Exception:
|
||||
media_msg = format_im_message(content_str or "", msg_type)
|
||||
text_content = media_msg.get("text") or content_str or ""
|
||||
|
||||
# 跳过会话控制/状态更新等非聊天内容帧（不记录、不展示、不触发自动回复）
|
||||
if _is_control_payload(content_json, msg_type):
|
||||
logger.debug(
|
||||
"Skip control WS frame: type=%s conv=%s", msg_type, conversation_id
|
||||
)
|
||||
return messages
|
||||
|
||||
if _should_emit_ws_message(conversation_id, msg_type):
|
||||
sender_uid = str(msg.sender)
|
||||
if media_msg and (
|
||||
media_msg.get("text")
|
||||
or media_msg.get("type") not in (None, "text", "")
|
||||
):
|
||||
display_content = serialize_message_content(media_msg)
|
||||
else:
|
||||
display_content = text_content or content_str
|
||||
payload = {
|
||||
"sender_name": sender_uid,
|
||||
"sender_uid": sender_uid,
|
||||
"content": display_content,
|
||||
"raw_content": content_str,
|
||||
"conversation_id": conversation_id,
|
||||
"unread_count": 1,
|
||||
"server_message_id": server_message_id,
|
||||
"message_type": msg_type,
|
||||
}
|
||||
if msg_type in (
|
||||
MSG_TYPE_IMAGE,
|
||||
MSG_TYPE_STICKER,
|
||||
MSG_TYPE_VOICE,
|
||||
MSG_TYPE_VIDEO,
|
||||
MSG_TYPE_LINK_CARD,
|
||||
):
|
||||
if not media_msg.get("url") and not media_msg.get("uri"):
|
||||
logger.warning(
|
||||
"Media WS message missing url: type=%s content=%s",
|
||||
msg_type,
|
||||
(content_str or "")[:800],
|
||||
)
|
||||
elif not media_msg.get("url"):
|
||||
logger.info(
|
||||
"Media WS message resolved via uri: type=%s uri=%s",
|
||||
msg_type,
|
||||
media_msg.get("uri"),
|
||||
)
|
||||
messages.append(payload)
|
||||
logger.info(
|
||||
"Protobuf WS message parsed: sender=%s type=%s content=%s conv=%s",
|
||||
sender,
|
||||
msg_type,
|
||||
text_content,
|
||||
conversation_id,
|
||||
)
|
||||
return messages
|
||||
except Exception as e:
|
||||
logger.debug(f"Protobuf WS parse failed: {e}")
|
||||
|
||||
if isinstance(raw, str):
|
||||
payloads = [raw.encode("utf-8", errors="ignore")]
|
||||
else:
|
||||
payloads = [raw]
|
||||
# 尝试 gzip 解压（frontier 常见）
|
||||
try:
|
||||
payloads.append(gzip.decompress(raw))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for payload in payloads:
|
||||
# 1) 直接 JSON
|
||||
if isinstance(payload, bytes):
|
||||
text = payload.decode("utf-8", errors="ignore").strip()
|
||||
else:
|
||||
text = str(payload).strip()
|
||||
if text.startswith("{") or text.startswith("["):
|
||||
try:
|
||||
data = json.loads(text)
|
||||
messages.extend(normalize_im_payload(data))
|
||||
continue
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2) 嵌入 JSON
|
||||
for obj in extract_json_objects(payload):
|
||||
messages.extend(normalize_im_payload(obj))
|
||||
|
||||
# 3) 纯文本兜底
|
||||
if isinstance(payload, bytes):
|
||||
text = payload.decode("utf-8", errors="ignore")
|
||||
plain = _extract_plain_text(text)
|
||||
if plain:
|
||||
messages.append({"content": plain, "sender_name": "", "raw_content": plain, "raw": True})
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def normalize_im_payload_from_bytes(raw: bytes) -> list[dict]:
|
||||
"""Try to extract conversation/message payloads from binary IM API responses."""
|
||||
results: list[dict] = []
|
||||
for obj in extract_json_objects(raw):
|
||||
results.extend(normalize_im_payload(obj))
|
||||
if results:
|
||||
return results
|
||||
|
||||
try:
|
||||
from .static import Response_pb2
|
||||
response = Response_pb2.Response()
|
||||
response.ParseFromString(raw)
|
||||
body = response.body
|
||||
for field in (
|
||||
"get_conversation_info_list_v2_response_body",
|
||||
"create_conversation_v2_body",
|
||||
):
|
||||
if body.HasField(field):
|
||||
conv_body = getattr(body, field)
|
||||
for conv in conv_body.conversation_info_list:
|
||||
conv_id = conv.conversation_id
|
||||
peer_uid = ""
|
||||
parts = conv_id.split(":")
|
||||
if len(parts) >= 4:
|
||||
peer_uid = parts[-1]
|
||||
label = f"用户{peer_uid[-6:]}" if peer_uid else conv_id
|
||||
results.append({
|
||||
"conversation_id": conv_id,
|
||||
"sender_name": label,
|
||||
"content": "",
|
||||
"unread_count": 0,
|
||||
"peer_uid": peer_uid,
|
||||
"conversation_short_id": str(conv.conversation_short_id),
|
||||
"ticket": conv.ticket,
|
||||
})
|
||||
except Exception:
|
||||
pass
|
||||
return results
|
||||
|
||||
|
||||
def normalize_im_payload(data: Any, depth: int = 0) -> list[dict]:
|
||||
"""递归标准化 IM JSON 为 {sender_name, content, conversation_id, unread_count}"""
|
||||
if depth > 12:
|
||||
return []
|
||||
results = []
|
||||
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
results.extend(normalize_im_payload(item, depth + 1))
|
||||
return results
|
||||
|
||||
if not isinstance(data, dict):
|
||||
return results
|
||||
|
||||
sender = (
|
||||
_pick_str(data, "sender_name", "senderName", "nickname", "nick_name", "userName", "peerName")
|
||||
or _pick_nested(data, ("core_info", "user_info", "peer_info"), "nick_name", "nickname", "name")
|
||||
)
|
||||
sender_avatar = _pick_avatar_url(data)
|
||||
content = _pick_message_text(data)
|
||||
msg_type = _coerce_message_type(
|
||||
data.get("message_type") or data.get("messageType") or data.get("msg_type"),
|
||||
MSG_TYPE_TEXT,
|
||||
)
|
||||
# 会话控制/状态更新帧（command_type / ext_data / 系统通知）直接忽略，不当作聊天消息
|
||||
if _is_control_payload(data, msg_type) or _is_control_payload(data.get("content"), msg_type):
|
||||
return results
|
||||
if msg_type != MSG_TYPE_TEXT or (isinstance(data.get("content"), dict)):
|
||||
parsed = parse_incoming_message(data)
|
||||
if parsed:
|
||||
content = parsed
|
||||
elif content and content.startswith("{"):
|
||||
try:
|
||||
parsed = format_im_message(json.loads(content), msg_type)
|
||||
content = serialize_message_content(parsed)
|
||||
except Exception:
|
||||
pass
|
||||
elif content in _NON_TEXT_MESSAGE_MARKERS:
|
||||
parsed = parse_stored_content(content)
|
||||
content = serialize_message_content(parsed)
|
||||
conv_id = _pick_str(
|
||||
data,
|
||||
"conversation_id",
|
||||
"conversationId",
|
||||
"conv_id",
|
||||
"cid",
|
||||
)
|
||||
unread = data.get("unread_count") or data.get("unreadCount") or data.get("unread_cnt") or 0
|
||||
try:
|
||||
unread = int(unread or 0)
|
||||
except (TypeError, ValueError):
|
||||
unread = 0
|
||||
|
||||
if content and len(content) < 500:
|
||||
from_self = data.get("is_self") or data.get("isSelf") or data.get("fromSelf") or data.get("self")
|
||||
if not from_self:
|
||||
raw_content = _extract_raw_content(data) or content
|
||||
results.append({
|
||||
"sender_name": sender or "未知用户",
|
||||
"sender_avatar": sender_avatar or None,
|
||||
"content": content,
|
||||
"raw_content": raw_content,
|
||||
"conversation_id": conv_id or "",
|
||||
"unread_count": unread,
|
||||
"message_type": msg_type,
|
||||
})
|
||||
|
||||
if sender and unread > 0 and not content:
|
||||
results.append({
|
||||
"sender_name": sender,
|
||||
"sender_avatar": sender_avatar or None,
|
||||
"content": "[未读消息]",
|
||||
"raw_content": "[未读消息]",
|
||||
"conversation_id": conv_id or "",
|
||||
"unread_count": unread,
|
||||
"message_type": msg_type,
|
||||
})
|
||||
|
||||
for key in ("conversations", "conversation_list", "data", "messages", "messagesList", "body"):
|
||||
nested = data.get(key)
|
||||
if nested is not None:
|
||||
results.extend(normalize_im_payload(nested, depth + 1))
|
||||
|
||||
for value in data.values():
|
||||
if isinstance(value, (dict, list)):
|
||||
results.extend(normalize_im_payload(value, depth + 1))
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _extract_raw_content(data: dict) -> str:
|
||||
for key in ("content", "message", "msg", "lastMessage", "last_msg", "preview", "brief"):
|
||||
val = data.get(key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
return val.strip()
|
||||
if isinstance(val, dict):
|
||||
return json.dumps(val, ensure_ascii=False, separators=(",", ":"))
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_str(data: dict, *keys: str) -> str:
|
||||
for key in keys:
|
||||
val = data.get(key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
return val.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_nested(data: dict, parent_keys: tuple, *child_keys: str) -> str:
|
||||
for pk in parent_keys:
|
||||
nested = data.get(pk)
|
||||
if isinstance(nested, dict):
|
||||
val = _pick_str(nested, *child_keys)
|
||||
if val:
|
||||
return val
|
||||
return ""
|
||||
|
||||
|
||||
def _avatar_from_value(val: Any) -> str:
|
||||
if isinstance(val, str) and val.strip().startswith("http"):
|
||||
return val.strip()
|
||||
if isinstance(val, dict):
|
||||
direct = val.get("url")
|
||||
if isinstance(direct, str) and direct.startswith("http"):
|
||||
return direct.strip()
|
||||
for list_key in ("url_list", "urls"):
|
||||
urls = val.get(list_key)
|
||||
if isinstance(urls, list):
|
||||
for item in urls:
|
||||
if isinstance(item, str) and item.startswith("http"):
|
||||
return item.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_avatar_url(data: dict) -> str:
|
||||
for key in ("avatar_url", "avatarUrl", "head_url", "headUrl", "avatar"):
|
||||
url = _avatar_from_value(data.get(key))
|
||||
if url:
|
||||
return url
|
||||
for thumb_key in ("avatar_thumb", "avatar_medium", "avatar_larger", "avatarThumb"):
|
||||
url = _avatar_from_value(data.get(thumb_key))
|
||||
if url:
|
||||
return url
|
||||
for parent_key in ("core_info", "user_info", "peer_info", "target_user", "conversation_core_info"):
|
||||
nested = data.get(parent_key)
|
||||
if isinstance(nested, dict):
|
||||
url = _pick_avatar_url(nested)
|
||||
if url:
|
||||
return url
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_message_text(data: dict) -> str:
|
||||
for key in (
|
||||
"text",
|
||||
"content",
|
||||
"message",
|
||||
"msg",
|
||||
"lastMessage",
|
||||
"last_msg",
|
||||
"preview",
|
||||
"brief",
|
||||
):
|
||||
val = data.get(key)
|
||||
if isinstance(val, str) and val.strip():
|
||||
return val.strip()
|
||||
if isinstance(val, dict):
|
||||
inner = _pick_str(val, "text", "content", "message")
|
||||
if inner:
|
||||
return inner
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_plain_text(text: str) -> Optional[str]:
|
||||
text = (text or "").strip()
|
||||
if not text or len(text) > 200:
|
||||
return None
|
||||
if text.startswith("{") or text.startswith("["):
|
||||
return None
|
||||
# 过滤明显二进制垃圾
|
||||
printable = sum(1 for c in text if c.isprintable() or c in "\n\r\t")
|
||||
if printable / max(len(text), 1) < 0.8:
|
||||
return None
|
||||
return text
|
||||
|
||||
|
||||
_NON_TEXT_MESSAGE_MARKERS = {
|
||||
"[表情包]",
|
||||
"[语音]",
|
||||
"[图片]",
|
||||
"[视频]",
|
||||
"[未读消息]",
|
||||
}
|
||||
|
||||
|
||||
def should_skip_auto_reply(content: str) -> tuple[bool, str]:
|
||||
"""判断收到的内容是否不适合触发自动回复（如纯点赞/表情互动）。"""
|
||||
from .message_content import is_media_message, message_preview
|
||||
|
||||
text = (content or "").strip()
|
||||
if not text:
|
||||
return True, "空消息"
|
||||
if is_media_message(text):
|
||||
return True, f"非文本消息({message_preview(text)})"
|
||||
if text in _NON_TEXT_MESSAGE_MARKERS:
|
||||
return True, f"非文本消息({text})"
|
||||
if re.fullmatch(r"(\[赞\])+", text):
|
||||
return True, "表情互动消息(点赞)，抖音通常不允许对此类消息自动回复"
|
||||
if re.fullmatch(r"\[[^\]]+\](\[[^\]]+\])*", text) and "http" not in text:
|
||||
inner = re.sub(r"[\[\]]", "", text)
|
||||
if len(inner) <= 20 and not any(ch.isalnum() for ch in inner):
|
||||
return True, f"非文本互动消息({text})"
|
||||
return False, ""
|
||||
@@ -0,0 +1,330 @@
|
||||
"""自动回复内容解析与 IM 消息体构造（文本 / 网址 / 卡片）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Tuple
|
||||
|
||||
from .message_content import (
|
||||
MSG_TYPE_IMAGE,
|
||||
MSG_TYPE_STICKER,
|
||||
MSG_TYPE_TEXT,
|
||||
message_preview,
|
||||
parse_stored_content,
|
||||
serialize_message_content,
|
||||
)
|
||||
|
||||
|
||||
def _normalize_reply_spec(data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
reply_type = data.get("type")
|
||||
if reply_type in ("text", "link", "card", "image", "sticker"):
|
||||
return data
|
||||
return None
|
||||
|
||||
|
||||
def parse_reply_messages(raw: str) -> list[dict[str, Any]]:
|
||||
"""解析规则中的 reply_content，支持单条或多条回复。"""
|
||||
raw = (raw or "").strip()
|
||||
if not raw:
|
||||
return [{"type": "text", "text": ""}]
|
||||
if raw.startswith("{") or raw.startswith("["):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict) and isinstance(data.get("messages"), list):
|
||||
specs = [_normalize_reply_spec(item) for item in data["messages"]]
|
||||
specs = [item for item in specs if item]
|
||||
if specs:
|
||||
return specs
|
||||
if isinstance(data, list):
|
||||
specs = [_normalize_reply_spec(item) for item in data]
|
||||
specs = [item for item in specs if item]
|
||||
if specs:
|
||||
return specs
|
||||
if isinstance(data, dict):
|
||||
spec = _normalize_reply_spec(data)
|
||||
if spec:
|
||||
return [spec]
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return [{"type": "text", "text": raw}]
|
||||
|
||||
|
||||
def parse_reply_content(raw: str) -> dict[str, Any]:
|
||||
"""解析发送/规则中的 content。纯字符串视为文本，JSON 为结构化消息。"""
|
||||
raw = (raw or "").strip()
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
if isinstance(data, dict):
|
||||
spec = _normalize_reply_spec(data)
|
||||
if spec:
|
||||
return spec
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
parsed = parse_stored_content(raw)
|
||||
if parsed.get("type") != "text":
|
||||
return parsed
|
||||
return parse_reply_messages(raw)[0]
|
||||
|
||||
|
||||
def _format_reply_spec(spec: dict[str, Any]) -> str:
|
||||
reply_type = spec.get("type", "text")
|
||||
if reply_type in ("image", "sticker", "voice", "video"):
|
||||
return message_preview(serialize_message_content(spec))
|
||||
if reply_type == "text":
|
||||
return (spec.get("text") or "").strip()
|
||||
if reply_type == "link":
|
||||
text = (spec.get("text") or spec.get("title") or "").strip()
|
||||
url = (spec.get("url") or "").strip()
|
||||
if text and url:
|
||||
return f"{text} → {url}"
|
||||
return text or url
|
||||
if reply_type == "card":
|
||||
title = (spec.get("title") or "").strip()
|
||||
desc = (spec.get("desc") or spec.get("description") or "").strip()
|
||||
page_url = (spec.get("url") or "").strip()
|
||||
if title and page_url:
|
||||
return f"[卡片] {title} → {page_url}"
|
||||
return title or desc or page_url or "[卡片]"
|
||||
return message_preview(serialize_message_content(spec))
|
||||
|
||||
|
||||
def format_reply_display(raw: str) -> str:
|
||||
"""将 reply_content 格式化为日志/列表中的可读摘要。"""
|
||||
preview = message_preview(raw)
|
||||
if preview:
|
||||
return preview
|
||||
specs = parse_reply_messages(raw)
|
||||
parts = [_format_reply_spec(spec) for spec in specs]
|
||||
parts = [part for part in parts if part]
|
||||
if not parts:
|
||||
return (raw or "").strip()
|
||||
if len(parts) == 1:
|
||||
return parts[0]
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def serialize_reply_messages(specs: list[dict[str, Any]]) -> str:
|
||||
cleaned = [spec for spec in specs if _normalize_reply_spec(spec)]
|
||||
if not cleaned:
|
||||
cleaned = [{"type": "text", "text": ""}]
|
||||
if len(cleaned) == 1:
|
||||
return serialize_reply_content(cleaned[0])
|
||||
return json.dumps({"messages": cleaned}, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def _extract_link_card_media_path(url: str) -> str:
|
||||
value = (url or "").strip()
|
||||
idx = value.find("/api/media/link-cards/")
|
||||
return value[idx:] if idx >= 0 else ""
|
||||
|
||||
|
||||
def expand_card_spec(spec: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"""卡片专属发送规则：展开为「封面图片 + 标题/内容/可点击链接文字」两条消息。
|
||||
|
||||
抖音 Web 协议无法发原生合并卡片(type=70 带图→8004/不带图→空白)，因此卡片以
|
||||
「图片消息(横幅) + 文本(标题/内容/链接)」组合呈现：图片提供视觉、文本提供可点击跳转。
|
||||
图片消息仅互关用户可收（陌生人会被 8003 拦截，但文本仍可送达，不影响链接触达）。
|
||||
"""
|
||||
title = (spec.get("title") or "").strip()
|
||||
desc = (spec.get("desc") or spec.get("description") or "").strip()
|
||||
target = (
|
||||
spec.get("target_url")
|
||||
or spec.get("url")
|
||||
or spec.get("link_url")
|
||||
or ""
|
||||
).strip()
|
||||
if "localhost" in target or "127.0.0.1" in target:
|
||||
target = (spec.get("target_url") or spec.get("link_url") or "").strip() or target
|
||||
|
||||
cover = (spec.get("image_path") or "").strip()
|
||||
if not cover:
|
||||
cover = _extract_link_card_media_path(spec.get("cover_url") or "")
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
if cover:
|
||||
out.append({"type": "image", "url": cover, "text": "[图片]"})
|
||||
lines = [x for x in (title, desc, target) if x]
|
||||
text = "\n".join(lines) if lines else target
|
||||
if text:
|
||||
out.append({"type": "text", "text": text})
|
||||
if not out:
|
||||
out.append({"type": "text", "text": target})
|
||||
return out
|
||||
|
||||
|
||||
def split_reply_payloads(raw: str) -> list[str]:
|
||||
"""将规则 reply_content 拆成可逐条发送的 payload 列表。卡片单独展开为图片+文本。"""
|
||||
payloads: list[str] = []
|
||||
for spec in parse_reply_messages(raw):
|
||||
if spec.get("type") == "card":
|
||||
payloads.extend(serialize_reply_content(s) for s in expand_card_spec(spec))
|
||||
else:
|
||||
payloads.append(serialize_reply_content(spec))
|
||||
return payloads
|
||||
|
||||
|
||||
def serialize_reply_log(payloads: list[str]) -> str:
|
||||
"""把实际逐条发送的 payload（JSON 字符串）合并为结构化的日志内容。
|
||||
|
||||
单条直接返回该 payload；多条用 {"messages":[...]} 包裹，便于前端逐条渲染
|
||||
（图片/表情正常显示为媒体，而不是被压扁成 "图片" 这样的占位文本）。
|
||||
"""
|
||||
specs: list[dict[str, Any]] = []
|
||||
for payload in payloads:
|
||||
raw = (payload or "").strip()
|
||||
if not raw:
|
||||
continue
|
||||
spec: dict[str, Any] | None = None
|
||||
if raw.startswith("{"):
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
spec = _normalize_reply_spec(data)
|
||||
except json.JSONDecodeError:
|
||||
spec = None
|
||||
if spec is None:
|
||||
spec = {"type": "text", "text": raw}
|
||||
specs.append(spec)
|
||||
return serialize_reply_messages(specs)
|
||||
|
||||
|
||||
def serialize_reply_content(spec: dict[str, Any]) -> str:
|
||||
return json.dumps(spec, ensure_ascii=False, separators=(",", ":"))
|
||||
|
||||
|
||||
def build_msg_payload(spec: dict[str, Any]) -> Tuple[dict[str, Any], int]:
|
||||
"""根据回复规格构造 IM msg_content 与 message_type。"""
|
||||
reply_type = spec.get("type", "text")
|
||||
|
||||
if reply_type == "text":
|
||||
text = (spec.get("text") or "").strip()
|
||||
return (
|
||||
{
|
||||
"mention_users": [],
|
||||
"aweType": 700,
|
||||
"richTextInfos": [],
|
||||
"text": text,
|
||||
},
|
||||
7,
|
||||
)
|
||||
|
||||
if reply_type == "link":
|
||||
display = (spec.get("text") or spec.get("title") or "").strip()
|
||||
url = (spec.get("url") or "").strip()
|
||||
if not display:
|
||||
display = url
|
||||
msg_content: dict[str, Any] = {
|
||||
"mention_users": [],
|
||||
"aweType": 700,
|
||||
"richTextInfos": [],
|
||||
"text": display,
|
||||
}
|
||||
if url and display:
|
||||
msg_content["richTextInfos"] = [
|
||||
{
|
||||
"start": 0,
|
||||
"end": len(display),
|
||||
"type": 2,
|
||||
"link": url,
|
||||
"text": display,
|
||||
}
|
||||
]
|
||||
return msg_content, 7
|
||||
|
||||
if reply_type == "card":
|
||||
# 卡片在 split_reply_payloads 阶段已展开为「图片 + 文本」两条，正常不会走到这里。
|
||||
# 兜底：万一收到未展开的卡片 spec，退化为「标题/内容/链接」文本，确保可送达。
|
||||
title = (spec.get("title") or "").strip()
|
||||
desc = (spec.get("desc") or spec.get("description") or "").strip()
|
||||
target = (
|
||||
spec.get("target_url")
|
||||
or spec.get("url")
|
||||
or spec.get("link_url")
|
||||
or ""
|
||||
).strip()
|
||||
lines = [x for x in (title, desc, target) if x]
|
||||
text = "\n".join(lines) if lines else target
|
||||
return (
|
||||
{"mention_users": [], "aweType": 700, "richTextInfos": [], "text": text},
|
||||
MSG_TYPE_TEXT,
|
||||
)
|
||||
|
||||
if reply_type == "image":
|
||||
uri = (spec.get("uri") or "").strip().lstrip("/")
|
||||
url = (spec.get("url") or "").strip()
|
||||
width = spec.get("width")
|
||||
height = spec.get("height")
|
||||
md5 = (spec.get("md5") or "").strip()
|
||||
url_list = spec.get("url_list")
|
||||
if not isinstance(url_list, list):
|
||||
url_list = [url] if url.startswith("http") else []
|
||||
|
||||
clean_urls = [str(u).strip() for u in url_list if str(u).strip().startswith("http")]
|
||||
|
||||
resource_url: dict[str, Any] = {}
|
||||
if uri:
|
||||
resource_url["uri"] = uri
|
||||
if clean_urls:
|
||||
resource_url["url_list"] = clean_urls
|
||||
if md5:
|
||||
resource_url["md5"] = md5
|
||||
if width:
|
||||
try:
|
||||
resource_url["width"] = int(width)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if height:
|
||||
try:
|
||||
resource_url["height"] = int(height)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
msg_content: dict[str, Any] = {
|
||||
"aweType": 2702,
|
||||
"from_gallery": 1,
|
||||
"create_type": 0,
|
||||
}
|
||||
if resource_url:
|
||||
msg_content["resource_url"] = resource_url
|
||||
if uri:
|
||||
msg_content["local_path"] = uri
|
||||
if md5:
|
||||
msg_content["md5"] = md5
|
||||
if width:
|
||||
try:
|
||||
msg_content["cover_width"] = int(width)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if height:
|
||||
try:
|
||||
msg_content["cover_height"] = int(height)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return msg_content, MSG_TYPE_IMAGE
|
||||
|
||||
if reply_type == "sticker":
|
||||
url = (spec.get("url") or "").strip()
|
||||
sticker_id = spec.get("sticker_id") or spec.get("id")
|
||||
msg_content = {
|
||||
"display_name": (spec.get("name") or spec.get("text") or "[表情包]").strip(),
|
||||
}
|
||||
if sticker_id:
|
||||
msg_content["id"] = sticker_id
|
||||
msg_content["sticker_id"] = sticker_id
|
||||
if url:
|
||||
msg_content["static_url"] = {"url_list": [url]}
|
||||
msg_content["animate_url"] = {"url_list": [url]}
|
||||
return msg_content, MSG_TYPE_STICKER
|
||||
|
||||
text = format_reply_display(serialize_reply_content(spec))
|
||||
return (
|
||||
{
|
||||
"mention_users": [],
|
||||
"aweType": 700,
|
||||
"richTextInfos": [],
|
||||
"text": text,
|
||||
},
|
||||
7,
|
||||
)
|
||||
@@ -0,0 +1,352 @@
|
||||
"""Observable per-account serial queue for delayed automatic replies."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Awaitable, Callable, Iterable, Optional
|
||||
|
||||
|
||||
logger = logging.getLogger("douyin_im.reply_queue")
|
||||
|
||||
ReplyCallback = Callable[[], Awaitable[Any]]
|
||||
ErrorCallback = Callable[[str, BaseException], None]
|
||||
DetailsMerger = Callable[[dict[str, Any]], dict[str, Any]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class _QueueItem:
|
||||
job_id: str
|
||||
due_at: float
|
||||
slot_seconds: float
|
||||
callback: ReplyCallback
|
||||
description: str
|
||||
queued_at: float
|
||||
merge_keys: frozenset[str] = field(default_factory=frozenset)
|
||||
details: dict[str, Any] = field(default_factory=dict)
|
||||
expedited: bool = False
|
||||
|
||||
|
||||
class AccountReplyQueue:
|
||||
"""Run and expose delayed reply jobs for one hosted account.
|
||||
|
||||
A single consumer is the only code path allowed to invoke callbacks. Jobs
|
||||
selected for immediate delivery are moved to an urgent FIFO, so they can
|
||||
never overlap an already active send. Removing a scheduled job also moves
|
||||
every job behind it forward by the removed job's reserved slot.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
account_id: int,
|
||||
on_error: Optional[ErrorCallback] = None,
|
||||
) -> None:
|
||||
self.account_id = account_id
|
||||
self._on_error = on_error
|
||||
self._waiting: list[_QueueItem] = []
|
||||
self._urgent: deque[_QueueItem] = deque()
|
||||
self._active_item: Optional[_QueueItem] = None
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._running = False
|
||||
self._state_lock = asyncio.Lock()
|
||||
self._wake = asyncio.Event()
|
||||
self._tail_due_at = 0.0
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
"""Approximate active + urgent + waiting count for lightweight badges."""
|
||||
return (
|
||||
len(self._waiting)
|
||||
+ len(self._urgent)
|
||||
+ (1 if self._active_item is not None else 0)
|
||||
)
|
||||
|
||||
async def start(self) -> None:
|
||||
async with self._state_lock:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._running = True
|
||||
self._tail_due_at = 0.0
|
||||
self._wake.clear()
|
||||
self._task = asyncio.create_task(
|
||||
self._run(),
|
||||
name=f"account-reply-queue-{self.account_id}",
|
||||
)
|
||||
|
||||
async def enqueue(
|
||||
self,
|
||||
delay_seconds: float,
|
||||
callback: ReplyCallback,
|
||||
description: str = "",
|
||||
details: Optional[dict[str, Any]] = None,
|
||||
merge_key: str = "",
|
||||
merge_keys: Optional[Iterable[str]] = None,
|
||||
) -> int:
|
||||
"""Append one reply job and return its current 1-based queue position."""
|
||||
interval = max(0.0, float(delay_seconds or 0))
|
||||
loop = asyncio.get_running_loop()
|
||||
async with self._state_lock:
|
||||
if not self._running or not self._task or self._task.done():
|
||||
raise RuntimeError("reply queue is not running")
|
||||
due_at = max(loop.time(), self._tail_due_at) + interval
|
||||
self._tail_due_at = due_at
|
||||
self._waiting.append(
|
||||
_QueueItem(
|
||||
job_id=uuid.uuid4().hex,
|
||||
due_at=due_at,
|
||||
slot_seconds=interval,
|
||||
callback=callback,
|
||||
description=description,
|
||||
queued_at=time.time(),
|
||||
merge_keys=self._normalize_merge_keys(
|
||||
merge_keys if merge_keys is not None else merge_key
|
||||
),
|
||||
details=deepcopy(details or {}),
|
||||
)
|
||||
)
|
||||
position = self.pending_count
|
||||
self._wake.set()
|
||||
return position
|
||||
|
||||
@staticmethod
|
||||
def _normalize_merge_keys(value: str | Iterable[str]) -> frozenset[str]:
|
||||
values = [value] if isinstance(value, str) else list(value or [])
|
||||
return frozenset(str(item or "").strip() for item in values if str(item or "").strip())
|
||||
|
||||
@staticmethod
|
||||
def _merge_keys_match(
|
||||
existing: frozenset[str],
|
||||
incoming: frozenset[str],
|
||||
) -> bool:
|
||||
existing_conversations = {key for key in existing if key.startswith("conv:")}
|
||||
incoming_conversations = {key for key in incoming if key.startswith("conv:")}
|
||||
if existing_conversations & incoming_conversations:
|
||||
return True
|
||||
# Two explicit, different conversation IDs must never merge just because
|
||||
# their partial source data happens to expose the same peer identifier.
|
||||
if existing_conversations and incoming_conversations:
|
||||
return False
|
||||
existing_peers = {key for key in existing if key.startswith("peer:")}
|
||||
incoming_peers = {key for key in incoming if key.startswith("peer:")}
|
||||
return bool(existing_peers & incoming_peers)
|
||||
|
||||
async def merge_pending(
|
||||
self,
|
||||
merge_key: str | Iterable[str],
|
||||
details_merger: DetailsMerger,
|
||||
) -> dict[str, Any]:
|
||||
"""Merge details into one queued conversation without changing its slot.
|
||||
|
||||
Only waiting and urgent jobs are mutable. Once the consumer marks a job
|
||||
active, its callback is sealed and a later message must follow the normal
|
||||
new-message path.
|
||||
"""
|
||||
normalized_keys = self._normalize_merge_keys(merge_key)
|
||||
if not normalized_keys:
|
||||
return {"status": "not_found"}
|
||||
|
||||
async with self._state_lock:
|
||||
if not self._running or not self._task or self._task.done():
|
||||
return {"status": "not_running"}
|
||||
|
||||
active_offset = 1 if self._active_item is not None else 0
|
||||
matches: list[tuple[_QueueItem, str, int]] = []
|
||||
|
||||
for index, candidate in enumerate(self._urgent):
|
||||
if self._merge_keys_match(candidate.merge_keys, normalized_keys):
|
||||
matches.append((candidate, "ready", active_offset + index + 1))
|
||||
|
||||
waiting_offset = active_offset + len(self._urgent)
|
||||
for index, candidate in enumerate(self._waiting):
|
||||
if self._merge_keys_match(candidate.merge_keys, normalized_keys):
|
||||
matches.append((candidate, "waiting", waiting_offset + index + 1))
|
||||
|
||||
if not matches:
|
||||
return {"status": "not_found"}
|
||||
|
||||
incoming_conversations = {
|
||||
key for key in normalized_keys if key.startswith("conv:")
|
||||
}
|
||||
if not incoming_conversations:
|
||||
matched_conversations = {
|
||||
key
|
||||
for candidate, _, _ in matches
|
||||
for key in candidate.merge_keys
|
||||
if key.startswith("conv:")
|
||||
}
|
||||
if len(matched_conversations) > 1:
|
||||
return {"status": "not_found", "reason": "ambiguous_peer"}
|
||||
|
||||
item, item_status, position = matches[0]
|
||||
|
||||
merged_details = details_merger(deepcopy(item.details))
|
||||
if not isinstance(merged_details, dict):
|
||||
raise TypeError("reply queue details merger must return a dict")
|
||||
item.details = deepcopy(merged_details)
|
||||
item.merge_keys = frozenset(item.merge_keys | normalized_keys)
|
||||
return {
|
||||
"status": "merged",
|
||||
"job_id": item.job_id,
|
||||
"position": position,
|
||||
"queue_status": item_status,
|
||||
"message_count": int(item.details.get("message_count") or 1),
|
||||
}
|
||||
|
||||
async def snapshot(self) -> list[dict[str, Any]]:
|
||||
"""Return a callback-free management snapshot ordered by execution."""
|
||||
loop = asyncio.get_running_loop()
|
||||
now_mono = loop.time()
|
||||
now_epoch = time.time()
|
||||
async with self._state_lock:
|
||||
ordered: list[tuple[_QueueItem, str]] = []
|
||||
if self._active_item is not None:
|
||||
ordered.append((self._active_item, "sending"))
|
||||
ordered.extend((item, "ready") for item in self._urgent)
|
||||
ordered.extend((item, "waiting") for item in self._waiting)
|
||||
|
||||
result = []
|
||||
for position, (item, status) in enumerate(ordered, start=1):
|
||||
remaining = 0.0 if status != "waiting" else max(0.0, item.due_at - now_mono)
|
||||
scheduled_epoch = now_epoch + max(0.0, item.due_at - now_mono)
|
||||
payload = {
|
||||
"job_id": item.job_id,
|
||||
"account_id": self.account_id,
|
||||
"position": position,
|
||||
"status": status,
|
||||
"expedited": bool(item.expedited),
|
||||
"description": item.description,
|
||||
"interval_seconds": int(round(item.slot_seconds)),
|
||||
"enqueued_at": datetime.fromtimestamp(
|
||||
item.queued_at, tz=timezone.utc
|
||||
).isoformat(),
|
||||
"scheduled_at": datetime.fromtimestamp(
|
||||
scheduled_epoch, tz=timezone.utc
|
||||
).isoformat(),
|
||||
"remaining_seconds": int(max(0, round(remaining))),
|
||||
}
|
||||
# Details are controlled by DouyinImService and never contain callbacks/session data.
|
||||
payload.update(deepcopy(item.details))
|
||||
result.append(payload)
|
||||
return result
|
||||
|
||||
async def send_now(self, job_id: str) -> dict[str, Any]:
|
||||
"""Move one waiting job to the urgent FIFO and free its future slot."""
|
||||
job_id = str(job_id or "").strip()
|
||||
async with self._state_lock:
|
||||
if not self._running or not self._task or self._task.done():
|
||||
return {"status": "not_running", "job_id": job_id}
|
||||
if self._active_item and self._active_item.job_id == job_id:
|
||||
return {"status": "already_sending", "job_id": job_id}
|
||||
if any(item.job_id == job_id for item in self._urgent):
|
||||
return {"status": "already_requested", "job_id": job_id}
|
||||
|
||||
selected_index = next(
|
||||
(index for index, item in enumerate(self._waiting) if item.job_id == job_id),
|
||||
None,
|
||||
)
|
||||
if selected_index is None:
|
||||
return {"status": "not_found", "job_id": job_id}
|
||||
|
||||
item = self._waiting.pop(selected_index)
|
||||
shift_seconds = max(0.0, item.slot_seconds)
|
||||
shifted_count = 0
|
||||
for later in self._waiting[selected_index:]:
|
||||
later.due_at -= shift_seconds
|
||||
shifted_count += 1
|
||||
|
||||
item.due_at = asyncio.get_running_loop().time()
|
||||
item.expedited = True
|
||||
self._urgent.append(item)
|
||||
self._recalculate_tail_due_at()
|
||||
self._wake.set()
|
||||
return {
|
||||
"status": "accepted",
|
||||
"job_id": job_id,
|
||||
"shifted_count": shifted_count,
|
||||
}
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Cancel the active wait/send and discard all remaining jobs."""
|
||||
async with self._state_lock:
|
||||
self._running = False
|
||||
self._wake.set()
|
||||
task = self._task
|
||||
self._task = None
|
||||
|
||||
if task:
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async with self._state_lock:
|
||||
self._waiting.clear()
|
||||
self._urgent.clear()
|
||||
self._active_item = None
|
||||
self._tail_due_at = 0.0
|
||||
self._wake.clear()
|
||||
|
||||
def _recalculate_tail_due_at(self) -> None:
|
||||
scheduled = [item.due_at for item in self._waiting]
|
||||
self._tail_due_at = max(scheduled, default=0.0)
|
||||
|
||||
async def _run(self) -> None:
|
||||
while True:
|
||||
item: Optional[_QueueItem] = None
|
||||
wait_seconds: Optional[float] = None
|
||||
async with self._state_lock:
|
||||
if not self._running:
|
||||
return
|
||||
if self._urgent:
|
||||
item = self._urgent.popleft()
|
||||
elif self._waiting:
|
||||
candidate = self._waiting[0]
|
||||
remaining = candidate.due_at - asyncio.get_running_loop().time()
|
||||
if remaining <= 0:
|
||||
item = self._waiting.pop(0)
|
||||
else:
|
||||
wait_seconds = remaining
|
||||
|
||||
if item is not None:
|
||||
self._active_item = item
|
||||
self._wake.clear()
|
||||
|
||||
if item is None:
|
||||
try:
|
||||
if wait_seconds is None:
|
||||
await self._wake.wait()
|
||||
else:
|
||||
await asyncio.wait_for(self._wake.wait(), timeout=wait_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
continue
|
||||
|
||||
try:
|
||||
await item.callback()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"Account %s queued reply failed (%s)",
|
||||
self.account_id,
|
||||
item.description,
|
||||
)
|
||||
if self._on_error:
|
||||
try:
|
||||
self._on_error(item.description, exc)
|
||||
except Exception:
|
||||
logger.debug("Reply queue error callback failed", exc_info=True)
|
||||
finally:
|
||||
async with self._state_lock:
|
||||
if self._active_item is item:
|
||||
self._active_item = None
|
||||
if not self._waiting:
|
||||
self._tail_due_at = 0.0
|
||||
self._wake.set()
|
||||
@@ -0,0 +1,983 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from typing import Awaitable, Callable, Optional
|
||||
|
||||
from utils import system_logger
|
||||
from .frontier import ensure_frontier_ws
|
||||
from .http_client import DouyinImHttpClient, format_session_credential_summary
|
||||
from .session import DouyinImSession
|
||||
from .ws_client import DouyinImWsClient
|
||||
from .reply_queue import AccountReplyQueue
|
||||
from .traffic_control import get_traffic_controller
|
||||
|
||||
from .reply_payload import format_reply_display, serialize_reply_log
|
||||
from .conv_util import resolve_peer_uid
|
||||
from .peer_profile import (
|
||||
enrich_conversation_item,
|
||||
fetch_peer_profile,
|
||||
is_generic_peer_name,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("douyin_im.service")
|
||||
|
||||
MatchReplyFn = Callable[[str], Awaitable[Optional[list[str]]]]
|
||||
LogFn = Callable[..., Awaitable[None]]
|
||||
ReceivedLogFn = Callable[..., Awaitable[None]]
|
||||
|
||||
|
||||
class DouyinImService:
|
||||
"""抖音 IM 直连服务：WebSocket 实时监听 + HTTP 轮询 + 自动回复"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session: DouyinImSession,
|
||||
match_reply: MatchReplyFn,
|
||||
log_fn: LogFn,
|
||||
account_id: int,
|
||||
received_log_fn: Optional[ReceivedLogFn] = None,
|
||||
reply_delay_seconds: int = 0,
|
||||
reply_delay_resolver: Optional[Callable[[], Awaitable[int]]] = None,
|
||||
reply_cooldown_seconds: Optional[int] = None,
|
||||
cooldown_resolver: Optional[Callable[[], Awaitable[int]]] = None,
|
||||
refresh_credentials: Optional[Callable[[], Awaitable[bool]]] = None,
|
||||
follow_tick: Optional[Callable[[], Awaitable[None]]] = None,
|
||||
on_session_invalid: Optional[Callable[[str], Awaitable[None]]] = None,
|
||||
):
|
||||
self.session = session
|
||||
self.match_reply = match_reply
|
||||
self.log_fn = log_fn
|
||||
self.received_log_fn = received_log_fn
|
||||
self.account_id = account_id
|
||||
# 由 worker 注入：周期性检测新粉丝并发送关注欢迎语（约每 60s 触发一次）
|
||||
self.follow_tick = follow_tick
|
||||
# 由 worker 注入：检测到 IM 登录失效（INVALID_REQUEST）时回调，用于自动下线
|
||||
self.on_session_invalid = on_session_invalid
|
||||
self._session_invalid_strikes = 0
|
||||
self._session_invalid_fired = False
|
||||
self.reply_delay_seconds = max(0, int(reply_delay_seconds or 0))
|
||||
# 实时解析账号排队间隔：账号专属优先，否则使用系统默认值。
|
||||
self._reply_delay_resolver = reply_delay_resolver
|
||||
self._reply_queue = AccountReplyQueue(
|
||||
account_id=self.account_id,
|
||||
on_error=self._on_reply_queue_error,
|
||||
)
|
||||
# WS 帧与 HTTP 轮询会并发进入；按到达顺序串行完成预处理/入队，确保 FIFO。
|
||||
self._incoming_lock = asyncio.Lock()
|
||||
# 该账号专属冷却秒数；None 表示继承全局系统设置（仅作为无 resolver 时的兜底）
|
||||
self._cooldown_override = (
|
||||
max(0, int(reply_cooldown_seconds)) if reply_cooldown_seconds is not None else None
|
||||
)
|
||||
# 实时解析冷却秒数的回调（账号专属优先，否则全局）；优先于 _cooldown_override
|
||||
self._cooldown_resolver = cooldown_resolver
|
||||
# 由 worker 注入：触发后台重新采集 web_protect/keys（刷新 ts_sign），返回是否刷新成功
|
||||
self.refresh_credentials = refresh_credentials
|
||||
self._running = False
|
||||
self._replied_keys: set[str] = set()
|
||||
self._logged_keys: set[str] = set()
|
||||
self._received_logged_keys: set[str] = set()
|
||||
# 每个对话/用户最近一次自动回复的时间戳（monotonic 秒），用于冷却窗口去重
|
||||
self._last_reply_at: dict[str, float] = {}
|
||||
self._conv_previews: dict[str, str] = {}
|
||||
self._conv_names: dict[str, str] = {} # uid/conv_id -> nickname
|
||||
self._conv_meta: dict[str, dict] = {} # conversation_id -> meta
|
||||
self._ws_client: Optional[DouyinImWsClient] = None
|
||||
self.last_error: str = ""
|
||||
|
||||
def _reply_key(self, conversation_key: str, content: str) -> str:
|
||||
return f"{conversation_key}::{content}"
|
||||
|
||||
@staticmethod
|
||||
def _reply_queue_merge_keys(
|
||||
conversation_id: str,
|
||||
peer_uid: str,
|
||||
) -> tuple[str, ...]:
|
||||
"""Return every stable identifier currently known for one conversation."""
|
||||
conversation_id = str(conversation_id or "").strip()
|
||||
peer_uid = str(peer_uid or "").strip()
|
||||
aliases: list[str] = []
|
||||
if conversation_id:
|
||||
aliases.append(f"conv:{conversation_id}")
|
||||
if peer_uid:
|
||||
aliases.append(f"peer:{peer_uid}")
|
||||
return tuple(aliases)
|
||||
|
||||
@staticmethod
|
||||
def _merge_reply_queue_details(
|
||||
existing: dict,
|
||||
*,
|
||||
incoming_content: str,
|
||||
sender_name: str,
|
||||
sender_id: str,
|
||||
sender_avatar: Optional[str],
|
||||
conversation_id: str,
|
||||
) -> dict:
|
||||
"""Append one received message while preserving the task's one reply."""
|
||||
merged = dict(existing or {})
|
||||
contents = merged.get("incoming_contents")
|
||||
if isinstance(contents, list):
|
||||
contents = list(contents)
|
||||
else:
|
||||
contents = []
|
||||
if not contents and "incoming_content" in merged:
|
||||
contents.append(str(merged.get("incoming_content") or ""))
|
||||
|
||||
latest_content = str(incoming_content or "")
|
||||
contents.append(latest_content)
|
||||
merged["incoming_content"] = latest_content
|
||||
merged["incoming_contents"] = contents
|
||||
merged["message_count"] = len(contents)
|
||||
|
||||
if sender_name:
|
||||
merged["sender_name"] = sender_name
|
||||
if sender_id:
|
||||
merged["sender_id"] = sender_id
|
||||
if sender_avatar:
|
||||
merged["sender_avatar"] = sender_avatar
|
||||
if conversation_id:
|
||||
merged["conversation_id"] = conversation_id
|
||||
return merged
|
||||
|
||||
def _cooldown_seconds_sync(self) -> int:
|
||||
"""无 resolver 时的兜底：账号专属优先，否则取全局设置；0 表示关闭。"""
|
||||
if self._cooldown_override is not None:
|
||||
return self._cooldown_override
|
||||
try:
|
||||
from auth.system_settings import get_cached_settings
|
||||
|
||||
return max(0, int(get_cached_settings().auto_reply_cooldown_seconds or 0))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
async def _resolve_cooldown_seconds(self) -> int:
|
||||
"""实时解析冷却秒数：优先用 worker 注入的 resolver（账号优先、否则全局），否则兜底。"""
|
||||
if self._cooldown_resolver is not None:
|
||||
try:
|
||||
return max(0, int(await self._cooldown_resolver() or 0))
|
||||
except Exception as e:
|
||||
logger.debug(f"cooldown resolver failed: {e}")
|
||||
return self._cooldown_seconds_sync()
|
||||
|
||||
def _reply_delay_seconds_sync(self) -> int:
|
||||
"""无 resolver 时解析排队间隔；0 表示不启用排队规则。"""
|
||||
if self.reply_delay_seconds > 0:
|
||||
return self.reply_delay_seconds
|
||||
try:
|
||||
from auth.system_settings import get_cached_settings
|
||||
|
||||
return max(0, int(get_cached_settings().auto_reply_delay_seconds or 0))
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
async def _resolve_reply_delay_seconds(self) -> int:
|
||||
"""实时解析账号生效的回复排队间隔。"""
|
||||
if self._reply_delay_resolver is not None:
|
||||
try:
|
||||
return max(0, int(await self._reply_delay_resolver() or 0))
|
||||
except Exception as exc:
|
||||
logger.debug(f"reply delay resolver failed: {exc}")
|
||||
return self._reply_delay_seconds_sync()
|
||||
|
||||
def _on_reply_queue_error(self, description: str, exc: BaseException) -> None:
|
||||
system_logger.record(
|
||||
"账号回复队列执行失败",
|
||||
detail=f"{description or '自动回复任务'}：{exc}",
|
||||
level="error",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
def _peer_in_cooldown(self, peer_key: str, cooldown: int) -> bool:
|
||||
if cooldown <= 0 or not peer_key:
|
||||
return False
|
||||
last = self._last_reply_at.get(peer_key)
|
||||
if last is None:
|
||||
return False
|
||||
return (time.monotonic() - last) < cooldown
|
||||
|
||||
def _resolve_sender_name(self, msg: dict) -> str:
|
||||
sender_uid = str(msg.get("sender_uid") or msg.get("sender_name") or "").strip()
|
||||
conv_id = str(msg.get("conversation_id") or "")
|
||||
name = (msg.get("sender_name") or "").strip()
|
||||
if name and not name.isdigit():
|
||||
return name
|
||||
if sender_uid and self._conv_names.get(sender_uid):
|
||||
return self._conv_names[sender_uid]
|
||||
if conv_id and self._conv_names.get(conv_id):
|
||||
return self._conv_names[conv_id]
|
||||
if sender_uid:
|
||||
return f"用户{sender_uid[-6:]}" if len(sender_uid) > 6 else f"用户{sender_uid}"
|
||||
return "未知用户"
|
||||
|
||||
def _is_self_message(self, msg: dict) -> bool:
|
||||
sender_uid = str(msg.get("sender_uid") or "").strip()
|
||||
if not sender_uid or not self.session.my_uid:
|
||||
return False
|
||||
try:
|
||||
return int(sender_uid) == int(self.session.my_uid)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
|
||||
async def _resolve_peer_profile(
|
||||
self,
|
||||
conv_id: str,
|
||||
sender_uid: str,
|
||||
sender: str,
|
||||
sender_avatar: str,
|
||||
) -> tuple[str, str, str]:
|
||||
my_uid = int(self.session.my_uid or 0)
|
||||
peer_uid = str(sender_uid or "").strip()
|
||||
if (not peer_uid or not peer_uid.isdigit()) and conv_id and my_uid:
|
||||
resolved = resolve_peer_uid(conv_id, my_uid)
|
||||
if resolved:
|
||||
peer_uid = str(resolved)
|
||||
|
||||
meta = self._conv_meta.get(conv_id, {}) if conv_id else {}
|
||||
name = (sender or meta.get("sender_name") or "").strip()
|
||||
avatar = (sender_avatar or meta.get("sender_avatar") or "").strip()
|
||||
|
||||
if peer_uid and self._conv_names.get(peer_uid):
|
||||
cached_name = self._conv_names[peer_uid]
|
||||
if is_generic_peer_name(name, peer_uid):
|
||||
name = cached_name
|
||||
if conv_id and self._conv_names.get(conv_id) and is_generic_peer_name(name, peer_uid):
|
||||
name = self._conv_names[conv_id]
|
||||
|
||||
if peer_uid and (is_generic_peer_name(name, peer_uid) or not avatar):
|
||||
profile = await fetch_peer_profile(self.session, peer_uid, self.account_id)
|
||||
if profile.get("nickname"):
|
||||
name = profile["nickname"]
|
||||
self._conv_names[peer_uid] = name
|
||||
if profile.get("avatar_url"):
|
||||
avatar = profile["avatar_url"]
|
||||
if profile.get("uid"):
|
||||
peer_uid = str(profile["uid"])
|
||||
|
||||
if not name:
|
||||
name = self._resolve_sender_name(
|
||||
{"sender_uid": peer_uid, "conversation_id": conv_id, "sender_name": sender}
|
||||
)
|
||||
return name, avatar, peer_uid
|
||||
|
||||
async def _fetch_message_by_id(self, conv_id: str, server_message_id: str) -> dict | None:
|
||||
"""按 server_message_id 调 get_by_conversation 拉取该条消息的完整数据
|
||||
（含真实 content / message_type / URL）。命中返回原始消息 dict，否则 None。"""
|
||||
if not conv_id or not server_message_id:
|
||||
return None
|
||||
try:
|
||||
from .auth import DouyinAuth
|
||||
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "message detail fetch"):
|
||||
meta = self._conv_meta.get(conv_id, {})
|
||||
short_id = str(meta.get("conversation_short_id") or "")
|
||||
auth = DouyinAuth.from_im_session(self.session)
|
||||
my_uid = int(self.session.my_uid or 0)
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
if not short_id:
|
||||
peer_uid = resolve_peer_uid(conv_id, my_uid)
|
||||
if peer_uid:
|
||||
_, short_id, _ = await http.get_conversation_info(
|
||||
auth, int(peer_uid), my_uid, conv_id, 0
|
||||
)
|
||||
if short_id:
|
||||
self._conv_meta[conv_id] = {
|
||||
**self._conv_meta.get(conv_id, {}),
|
||||
"conversation_short_id": short_id,
|
||||
}
|
||||
messages = await http.get_conversation_messages(
|
||||
auth, conv_id, int(short_id or 0), limit=20
|
||||
)
|
||||
for m in messages:
|
||||
if str(m.get("server_message_id") or "") == server_message_id:
|
||||
return m
|
||||
except Exception as e:
|
||||
logger.debug(f"_fetch_message_by_id failed: {e}")
|
||||
return None
|
||||
|
||||
async def _enrich_media_content(self, conv_id: str, server_message_id: str, content: str) -> str:
|
||||
"""媒体消息（相册图片/语音/视频）WS 推送 content 为空时，按 server_message_id
|
||||
调 get_by_conversation 拉取真实内容并补全 URL。命中失败则原样返回。"""
|
||||
if not conv_id or not server_message_id or not content:
|
||||
return content
|
||||
try:
|
||||
from .message_content import parse_stored_content, format_im_message, serialize_message_content
|
||||
|
||||
parsed = parse_stored_content(content)
|
||||
mtype = parsed.get("type")
|
||||
if mtype not in ("image", "voice", "video"):
|
||||
return content
|
||||
if parsed.get("url"):
|
||||
return content # 已有 URL（如商店表情/带 url 的图）
|
||||
|
||||
m = await self._fetch_message_by_id(conv_id, server_message_id)
|
||||
if m:
|
||||
real = format_im_message(m.get("content") or "", int(m.get("message_type") or 0))
|
||||
if real.get("url"):
|
||||
enriched = serialize_message_content(real)
|
||||
logger.info(
|
||||
"Enriched media via get_by_conversation: smid=%s type=%s",
|
||||
server_message_id, real.get("type"),
|
||||
)
|
||||
return enriched
|
||||
except Exception as e:
|
||||
logger.debug(f"_enrich_media_content failed: {e}")
|
||||
return content
|
||||
|
||||
async def _handle_incoming(self, msg: dict):
|
||||
# asyncio.Lock 按等待顺序唤醒。锁只覆盖解析、去重、规则匹配与入队；
|
||||
# 未启用排队时，真正的网络发送仍在锁外执行，保持原有并发行为。
|
||||
async with self._incoming_lock:
|
||||
immediate_reply = await self._prepare_incoming(msg)
|
||||
if immediate_reply is not None and self._running:
|
||||
await immediate_reply()
|
||||
|
||||
async def _prepare_incoming(
|
||||
self,
|
||||
msg: dict,
|
||||
) -> Optional[Callable[[], Awaitable[None]]]:
|
||||
if self._is_self_message(msg):
|
||||
return
|
||||
|
||||
conv_id = msg.get("conversation_id") or ""
|
||||
sender_uid = str(msg.get("sender_uid") or "")
|
||||
sender = self._resolve_sender_name(msg)
|
||||
sender_avatar = str(msg.get("sender_avatar") or "").strip()
|
||||
sender, sender_avatar, peer_uid = await self._resolve_peer_profile(
|
||||
conv_id, sender_uid, sender, sender_avatar
|
||||
)
|
||||
content = (msg.get("content") or "").strip()
|
||||
has_raw_ws = "raw_content" in msg
|
||||
raw_incoming = msg.get("raw_content") if has_raw_ws else None
|
||||
ws_message_type = msg.get("message_type")
|
||||
# 每条 WS 消息带唯一 server_message_id：用它去重，避免“同一用户重复发送
|
||||
# 相同文字（如多次‘你好’）被按内容去重而整条丢弃”，这是“有时收不到”的根因。
|
||||
# HTTP 轮询的会话预览没有该 ID，则退回按 内容 去重（避免对同一未读重复回复）。
|
||||
server_message_id = str(msg.get("server_message_id") or "")
|
||||
# WS 仅推送瘦消息（如 type=26）content 为空：按 server_message_id 回 HTTP 拉取
|
||||
# 完整消息，补全 content / message_type，确保「接收到的全部信息」都被记录。
|
||||
if not content and server_message_id and not (raw_incoming or "").strip():
|
||||
real = await self._fetch_message_by_id(conv_id, server_message_id)
|
||||
if real:
|
||||
real_content = (real.get("content") or "").strip()
|
||||
if real_content:
|
||||
raw_incoming = real.get("content")
|
||||
has_raw_ws = True
|
||||
real_type = real.get("message_type")
|
||||
if real_type is not None:
|
||||
ws_message_type = real_type
|
||||
try:
|
||||
from .message_content import format_im_message, serialize_message_content
|
||||
|
||||
parsed = format_im_message(real.get("content") or "", int(real_type or 0))
|
||||
content = serialize_message_content(parsed) if parsed else real_content
|
||||
except Exception:
|
||||
content = real_content
|
||||
logger.info(
|
||||
"Enriched empty WS push via get_by_conversation: smid=%s type=%s",
|
||||
server_message_id, real_type,
|
||||
)
|
||||
# 相册图片/语音等 WS 推送 content 为空，按 server_message_id 拉取真实内容补 URL
|
||||
content = await self._enrich_media_content(conv_id, server_message_id, content)
|
||||
unread = int(msg.get("unread_count") or 0)
|
||||
|
||||
if conv_id:
|
||||
self._conv_meta[conv_id] = {
|
||||
**self._conv_meta.get(conv_id, {}),
|
||||
"conversation_id": conv_id,
|
||||
"sender_name": sender,
|
||||
"sender_avatar": sender_avatar or self._conv_meta.get(conv_id, {}).get("sender_avatar"),
|
||||
"content": content or self._conv_meta.get(conv_id, {}).get("content", ""),
|
||||
"unread_count": unread,
|
||||
"peer_uid": peer_uid,
|
||||
}
|
||||
if sender and peer_uid:
|
||||
self._conv_names[peer_uid] = sender
|
||||
|
||||
if not content and unread <= 0 and raw_incoming is None and not server_message_id:
|
||||
return
|
||||
|
||||
if content == "[未读消息]" and sender in self._conv_previews:
|
||||
content = self._conv_previews.get(sender, content)
|
||||
|
||||
if server_message_id:
|
||||
log_key = f"mid:{server_message_id}"
|
||||
key = f"mid:{server_message_id}"
|
||||
else:
|
||||
# HTTP 会话预览通常没有 message_id；必须带 conversation_id/peer_uid，
|
||||
# 否则两个同名用户发送相同内容会被误判成同一条消息。
|
||||
conversation_key = str(conv_id or peer_uid or sender or "unknown")
|
||||
log_key = self._reply_key(conversation_key, content or "[未读]")
|
||||
key = self._reply_key(conversation_key, content)
|
||||
|
||||
log_kwargs = {
|
||||
"sender_name": sender,
|
||||
"sender_id": peer_uid or conv_id or None,
|
||||
"sender_avatar": sender_avatar or self._conv_meta.get(conv_id, {}).get("sender_avatar"),
|
||||
"message": content or (raw_incoming if raw_incoming is not None else ""),
|
||||
}
|
||||
|
||||
# 接收消息原始日志：WS content 原样落库（瘦推送已回 HTTP 补全为真实 content）
|
||||
if self.received_log_fn and has_raw_ws:
|
||||
recv_key = f"recv:mid:{server_message_id}" if server_message_id else f"recv:{log_key}"
|
||||
if recv_key not in self._received_logged_keys:
|
||||
self._received_logged_keys.add(recv_key)
|
||||
message_type = ws_message_type
|
||||
try:
|
||||
message_type = int(message_type) if message_type is not None else None
|
||||
except (TypeError, ValueError):
|
||||
message_type = None
|
||||
await self.received_log_fn(
|
||||
sender_name=sender,
|
||||
sender_id=peer_uid or conv_id or None,
|
||||
sender_avatar=log_kwargs.get("sender_avatar"),
|
||||
raw_content="" if raw_incoming is None else raw_incoming,
|
||||
conversation_id=conv_id or None,
|
||||
message_type=message_type,
|
||||
server_message_id=server_message_id or None,
|
||||
)
|
||||
|
||||
if log_key not in self._logged_keys and content:
|
||||
self._logged_keys.add(log_key)
|
||||
await self.log_fn(
|
||||
**log_kwargs,
|
||||
reply=None,
|
||||
status="received",
|
||||
)
|
||||
try:
|
||||
from .message_content import format_system_log_message, parse_stored_content
|
||||
|
||||
parsed = parse_stored_content(content)
|
||||
msg_type = parsed.get("type") or "text"
|
||||
detail = format_system_log_message(content)
|
||||
if server_message_id:
|
||||
detail = f"{detail} | mid={server_message_id}"
|
||||
system_logger.record(
|
||||
f"收到{'' if msg_type == 'text' else '['+msg_type+']'}消息：{sender}",
|
||||
detail=detail,
|
||||
level="info",
|
||||
category="recv",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.debug(f"record recv system log failed: {exc}")
|
||||
|
||||
if key in self._replied_keys:
|
||||
return
|
||||
# WS 与 HTTP 轮询可能同时发现同一条消息。检查后立即占位（中间不 await），
|
||||
# 防止延迟排队期间被重复加入发送队列。
|
||||
self._replied_keys.add(key)
|
||||
|
||||
# 同账号、同会话只保留一个尚未发送的回复任务。后续来信只追加到
|
||||
# 原任务详情，不改变它的发送时间、位置或已经匹配好的回复。
|
||||
queue_merge_keys = self._reply_queue_merge_keys(conv_id, peer_uid)
|
||||
if queue_merge_keys and self._running:
|
||||
merge_result = await self._reply_queue.merge_pending(
|
||||
queue_merge_keys,
|
||||
lambda existing: self._merge_reply_queue_details(
|
||||
existing,
|
||||
incoming_content=content or "",
|
||||
sender_name=sender,
|
||||
sender_id=peer_uid or conv_id or "",
|
||||
sender_avatar=log_kwargs.get("sender_avatar"),
|
||||
conversation_id=conv_id,
|
||||
),
|
||||
)
|
||||
if merge_result.get("status") == "merged":
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
message_count = int(merge_result.get("message_count") or 1)
|
||||
logger.info(
|
||||
"Merged message into queued reply for %s on account %s: "
|
||||
"job=%s messages=%s position=%s",
|
||||
sender,
|
||||
self.account_id,
|
||||
merge_result.get("job_id"),
|
||||
message_count,
|
||||
merge_result.get("position"),
|
||||
)
|
||||
system_logger.record(
|
||||
"同一会话消息已合并到回复队列",
|
||||
detail=(
|
||||
f"{sender} 的新消息已并入原任务；当前共 {message_count} 条消息，"
|
||||
"发送时间和队列位置保持不变。"
|
||||
),
|
||||
level="info",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
|
||||
# 收到新消息即尝试自动回复，不按消息类型/托管关系/内容形态过滤
|
||||
replies = await self.match_reply(content if content != "[未读消息]" else "")
|
||||
if not replies:
|
||||
replies = await self.match_reply("")
|
||||
if not replies:
|
||||
await self.log_fn(
|
||||
**log_kwargs,
|
||||
reply=None,
|
||||
status="ignored",
|
||||
error="未配置任何自动回复规则，请在「自动回复规则」中添加至少一条启用规则",
|
||||
)
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
return
|
||||
|
||||
# 冷却窗口：同一用户在设定时间内，无论发多少条消息，只自动回复一次（账号设置优先，否则全局）
|
||||
peer_key = (peer_uid or conv_id or sender or "").strip()
|
||||
cooldown = await self._resolve_cooldown_seconds()
|
||||
if self._peer_in_cooldown(peer_key, cooldown):
|
||||
logger.info(
|
||||
f"Auto-reply to {sender} skipped: within {cooldown}s cooldown window"
|
||||
)
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
system_logger.record(
|
||||
"自动回复已跳过（冷却中）",
|
||||
detail=f"{sender} 在 {cooldown} 秒冷却窗口内重复发送，未重复回复",
|
||||
level="info",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
# 提前标记回复时间，确保冷却窗口内（含延迟期间）的后续消息都被抑制
|
||||
if peer_key and cooldown > 0:
|
||||
self._last_reply_at[peer_key] = time.monotonic()
|
||||
|
||||
if content:
|
||||
self._conv_previews[sender] = content
|
||||
|
||||
delay_seconds = await self._resolve_reply_delay_seconds()
|
||||
if not self._running:
|
||||
return
|
||||
|
||||
async def send_reply() -> None:
|
||||
await self._send_auto_reply(
|
||||
sender=sender,
|
||||
content=content,
|
||||
conv_id=conv_id,
|
||||
replies=replies,
|
||||
peer_key=peer_key,
|
||||
cooldown=cooldown,
|
||||
log_kwargs=log_kwargs,
|
||||
)
|
||||
|
||||
if delay_seconds > 0:
|
||||
position = await self._reply_queue.enqueue(
|
||||
delay_seconds,
|
||||
send_reply,
|
||||
description=f"回复 {sender}",
|
||||
details={
|
||||
"sender_name": sender,
|
||||
"sender_id": peer_uid or conv_id or None,
|
||||
"sender_avatar": log_kwargs.get("sender_avatar"),
|
||||
"conversation_id": conv_id or None,
|
||||
"incoming_content": content or "",
|
||||
"incoming_contents": [content or ""],
|
||||
"message_count": 1,
|
||||
"replies": list(replies),
|
||||
},
|
||||
merge_keys=queue_merge_keys,
|
||||
)
|
||||
logger.info(
|
||||
"Queued reply to %s for account %s: position=%s interval=%ss",
|
||||
sender,
|
||||
self.account_id,
|
||||
position,
|
||||
delay_seconds,
|
||||
)
|
||||
system_logger.record(
|
||||
"自动回复已进入账号队列",
|
||||
detail=(
|
||||
f"{sender} 当前排在第 {position} 位；账号生效间隔为 {delay_seconds} 秒，"
|
||||
"账号内计时与排位独立；到点后再进入全局带宽队列逐条投递。"
|
||||
),
|
||||
level="info",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
|
||||
# 账号与系统均未配置排队间隔：跳过排队规则，保持原来的立即回复。
|
||||
return send_reply
|
||||
|
||||
async def _send_auto_reply(
|
||||
self,
|
||||
*,
|
||||
sender: str,
|
||||
content: str,
|
||||
conv_id: str,
|
||||
replies: list[str],
|
||||
peer_key: str,
|
||||
cooldown: int,
|
||||
log_kwargs: dict,
|
||||
) -> None:
|
||||
"""发送一项已匹配的自动回复任务，并记录原有消息/系统日志。"""
|
||||
if not self._running:
|
||||
return
|
||||
reply_displays: list[str] = []
|
||||
sent_any = False
|
||||
send_error = ""
|
||||
meta = self._conv_meta.get(conv_id, {})
|
||||
for index, reply in enumerate(replies):
|
||||
if not self._running:
|
||||
send_error = self.last_error or "托管已停止，后续回复已取消"
|
||||
break
|
||||
if index > 0:
|
||||
await asyncio.sleep(0.6)
|
||||
if not self._running:
|
||||
send_error = self.last_error or "托管已停止，后续回复已取消"
|
||||
break
|
||||
reply_display = format_reply_display(reply)
|
||||
reply_displays.append(reply_display)
|
||||
sent = False
|
||||
if conv_id:
|
||||
sent, resolved = await self._send_text(
|
||||
conv_id,
|
||||
reply,
|
||||
conversation_short_id=str(meta.get("conversation_short_id") or ""),
|
||||
)
|
||||
if sent:
|
||||
if resolved:
|
||||
meta = {**meta, **resolved, "conversation_id": conv_id}
|
||||
self._conv_meta[conv_id] = meta
|
||||
else:
|
||||
send_error = self.last_error or "IM API 发送失败"
|
||||
else:
|
||||
send_error = "缺少会话 ID，无法发送自动回复"
|
||||
if sent:
|
||||
sent_any = True
|
||||
|
||||
combined_display = " | ".join(reply_displays)
|
||||
# 日志里存结构化内容（单条直接存 payload，多条用 {"messages":[...]} 包裹），
|
||||
# 这样图片/表情等媒体回复会被前端渲染为真实媒体，而不是被压成 "图片" 占位文字。
|
||||
reply_log_content = serialize_reply_log(replies)
|
||||
if not sent_any:
|
||||
logger.warning(
|
||||
f"IM API send failed for [{sender}]: {send_error}; reply saved to log only"
|
||||
)
|
||||
# 发送彻底失败：清除冷却时间戳，避免把没收到回复的用户锁在冷却窗口内
|
||||
if peer_key and cooldown > 0:
|
||||
self._last_reply_at.pop(peer_key, None)
|
||||
|
||||
await self.log_fn(
|
||||
**log_kwargs,
|
||||
reply=reply_log_content,
|
||||
status="replied" if sent_any else "failed",
|
||||
error=None if sent_any else (send_error or "IM API 发送失败"),
|
||||
)
|
||||
if sent_any:
|
||||
system_logger.record(
|
||||
"自动回复成功",
|
||||
detail=f"已回复 {sender}：{combined_display}",
|
||||
level="success",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
else:
|
||||
system_logger.record(
|
||||
"自动回复失败",
|
||||
detail=f"回复 {sender} 失败：{send_error}（收到：{content}）",
|
||||
level="error",
|
||||
category="send",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
logger.info(
|
||||
f"Auto-reply to {sender}: {content!r} -> {combined_display!r} "
|
||||
f"(sent={sent_any}, count={len(replies)})"
|
||||
)
|
||||
|
||||
async def _index_conversations(self, conversations: list[dict]):
|
||||
my_uid = int(self.session.my_uid or 0)
|
||||
for raw in conversations:
|
||||
conv = enrich_conversation_item(raw, my_uid)
|
||||
conv_id = str(conv.get("conversation_id") or "")
|
||||
name = (conv.get("sender_name") or "").strip()
|
||||
avatar = str(conv.get("sender_avatar") or "").strip()
|
||||
peer_uid = str(conv.get("peer_uid") or "")
|
||||
|
||||
if peer_uid and (is_generic_peer_name(name, peer_uid) or not avatar):
|
||||
profile = await fetch_peer_profile(self.session, peer_uid, self.account_id)
|
||||
if profile.get("nickname"):
|
||||
name = profile["nickname"]
|
||||
conv["sender_name"] = name
|
||||
if profile.get("avatar_url"):
|
||||
avatar = profile["avatar_url"]
|
||||
conv["sender_avatar"] = avatar
|
||||
|
||||
if conv_id:
|
||||
self._conv_meta[conv_id] = {
|
||||
**conv,
|
||||
"sender_name": name,
|
||||
"sender_avatar": avatar or None,
|
||||
"peer_uid": peer_uid,
|
||||
}
|
||||
if name:
|
||||
self._conv_names[conv_id] = name
|
||||
if peer_uid and name:
|
||||
self._conv_names[peer_uid] = name
|
||||
|
||||
async def _poll_conversations(self):
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "conversation poll"):
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
unread_total = await http.get_unread_count()
|
||||
if unread_total:
|
||||
logger.info(f"IM unread total: {unread_total}")
|
||||
conversations = await http.get_conversations()
|
||||
await self._index_conversations(conversations)
|
||||
# Message handling may wait in the global send lane. Do not keep one
|
||||
# of the scarce background HTTP slots occupied while that happens.
|
||||
for conv in conversations:
|
||||
unread = int(conv.get("unread_count") or 0)
|
||||
if unread > 0 or conv.get("content"):
|
||||
await self._handle_incoming(conv)
|
||||
|
||||
async def _verify_account_uid(self):
|
||||
"""启动时用 query/user 接口核验账号真实 UID，修正采集端可能取错的 my_uid/device_id。
|
||||
|
||||
采集端从 tea_cache 推断的 my_uid 可能是访客/对方 id，会导致会话列表为 0、
|
||||
创建会话 INVALID_REQUEST。这里在建连前先校正，保证后续所有请求身份正确。
|
||||
"""
|
||||
if getattr(self.session, "uid_verified", False) and self.session.my_uid:
|
||||
return
|
||||
try:
|
||||
from .auth import DouyinAuth
|
||||
auth = DouyinAuth.from_im_session(self.session)
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "account UID verify"):
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
old = int(self.session.my_uid or 0)
|
||||
resolved = await asyncio.to_thread(http._resolve_authoritative_uid, auth)
|
||||
if resolved and old and int(resolved) != old:
|
||||
system_logger.record(
|
||||
"已自动校正账号 UID",
|
||||
detail=f"采集端识别 UID={old}，接口核验真实 UID={resolved}，已修正后再建立私信连接。",
|
||||
level="info",
|
||||
category="system",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"启动核验账号 UID 失败（沿用采集值）：{e}")
|
||||
|
||||
async def run(self):
|
||||
"""主循环：WebSocket + HTTP 轮询"""
|
||||
self._running = True
|
||||
await self._reply_queue.start()
|
||||
await self._verify_account_uid()
|
||||
# ensure_frontier_ws 可能触发签名/HTTP（阻塞），放线程池避免多账号启动时卡死事件循环
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id, "frontier discovery"):
|
||||
await asyncio.to_thread(ensure_frontier_ws, self.session)
|
||||
has_ws = bool(self.session.frontier_ws_url())
|
||||
cred_summary = format_session_credential_summary(self.session)
|
||||
logger.info(cred_summary)
|
||||
logger.info(
|
||||
f"Starting IM direct service for account {self.account_id} "
|
||||
f"(ws={'yes' if has_ws else 'no'})"
|
||||
)
|
||||
system_logger.record(
|
||||
"私信托管已启动",
|
||||
detail=f"实时接收通道：{'已就绪' if has_ws else '不可用（仅 HTTP 轮询）'}\n{cred_summary}",
|
||||
level="success" if has_ws else "warning",
|
||||
category="system",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
try:
|
||||
from .emoji_pack import ensure_emoji_map, is_fresh
|
||||
|
||||
if not is_fresh():
|
||||
async with controller.background_slot(self.account_id, "emoji preload"):
|
||||
await asyncio.to_thread(ensure_emoji_map, self.session)
|
||||
except Exception as e:
|
||||
logger.debug(f"emoji map preload failed: {e}")
|
||||
|
||||
self._ws_client = DouyinImWsClient(
|
||||
self.session, self._handle_incoming, account_id=self.account_id
|
||||
)
|
||||
await self._ws_client.start()
|
||||
|
||||
try:
|
||||
await self._poll_conversations()
|
||||
except Exception as e:
|
||||
logger.warning(f"Initial conversation poll failed: {e}")
|
||||
system_logger.record(
|
||||
"首次会话轮询失败",
|
||||
detail=f"{e}",
|
||||
level="warning",
|
||||
category="poll",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
loop_count = 0
|
||||
while self._running:
|
||||
# The initial poll above is authoritative. Sleep before the next
|
||||
# recurring tick so startup cannot issue two back-to-back polls.
|
||||
await asyncio.sleep(5)
|
||||
if not self._running:
|
||||
break
|
||||
loop_count += 1
|
||||
try:
|
||||
if loop_count % 3 == 0:
|
||||
await self._poll_conversations()
|
||||
if loop_count % 6 == 0:
|
||||
logger.info(f"IM direct tick #{loop_count} account={self.account_id}")
|
||||
# 关注欢迎语：约每 60s 检测一次新粉丝（独立于私信轮询，失败不影响主循环）
|
||||
if self.follow_tick and loop_count % 12 == 0:
|
||||
try:
|
||||
await self.follow_tick()
|
||||
except Exception as e:
|
||||
logger.error(f"follow welcome tick error: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"IM poll error: {e}")
|
||||
system_logger.record(
|
||||
"会话轮询出错",
|
||||
detail=f"拉取会话/未读时出错：{e}",
|
||||
level="error",
|
||||
category="poll",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
async def stop(self):
|
||||
self._running = False
|
||||
await get_traffic_controller().send_queue.cancel_account(self.account_id)
|
||||
await self._reply_queue.stop()
|
||||
if self._ws_client:
|
||||
await self._ws_client.stop()
|
||||
|
||||
def get_cached_conversations(self) -> list[dict]:
|
||||
"""返回运行中缓存的会话（来自 WS / 轮询）。"""
|
||||
results = []
|
||||
seen = set()
|
||||
for conv_id, meta in self._conv_meta.items():
|
||||
name = (meta.get("sender_name") or "").strip()
|
||||
key = conv_id or name
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
results.append({
|
||||
"conversation_id": conv_id,
|
||||
"sender_name": name or f"会话{conv_id[-8:]}" if conv_id else "未知用户",
|
||||
"sender_avatar": meta.get("sender_avatar") or None,
|
||||
"sender_id": str(meta.get("peer_uid") or meta.get("sender_id") or conv_id or ""),
|
||||
"peer_uid": str(meta.get("peer_uid") or ""),
|
||||
"content": str(meta.get("content") or ""),
|
||||
"unread_count": int(meta.get("unread_count") or 0),
|
||||
})
|
||||
return results
|
||||
|
||||
async def get_reply_queue_snapshot(self) -> list[dict]:
|
||||
"""返回当前账号自动回复队列的可管理快照。"""
|
||||
return await self._reply_queue.snapshot()
|
||||
|
||||
async def send_queued_reply_now(self, job_id: str) -> dict:
|
||||
"""把指定自动回复任务移入账号紧急队列；实际发送仍由单消费者串行执行。"""
|
||||
return await self._reply_queue.send_now(job_id)
|
||||
|
||||
async def _send_text(
|
||||
self,
|
||||
conversation_id: str,
|
||||
content: str,
|
||||
conversation_short_id: str = "",
|
||||
) -> tuple[bool, Optional[dict]]:
|
||||
"""发送一条私信；若因签名凭证失效(7911)失败，刷新 web_protect 后自动重试一次。
|
||||
|
||||
返回 (是否成功, 解析到的会话 meta)。失败原因写入 self.last_error。
|
||||
"""
|
||||
for attempt in range(2):
|
||||
async with DouyinImHttpClient(self.session, account_id=self.account_id) as http:
|
||||
sent = await http.send_text_message(
|
||||
conversation_id,
|
||||
content,
|
||||
conversation_short_id=conversation_short_id,
|
||||
)
|
||||
self.last_error = http.last_error
|
||||
needs_refresh = http.last_send_needs_refresh
|
||||
if sent:
|
||||
resolved = http.last_send_meta.get(conversation_id)
|
||||
self.session.conv_meta.update(http.session.conv_meta)
|
||||
self._session_invalid_strikes = 0 # 发送成功 → 登录有效
|
||||
return True, resolved
|
||||
|
||||
# 仅在“签名凭证失效”时刷新并重试一次
|
||||
if attempt == 0 and needs_refresh and self.refresh_credentials:
|
||||
logger.warning(
|
||||
f"Send hit credential-expiry(7911) for {conversation_id}; "
|
||||
"refreshing web_protect and retrying once..."
|
||||
)
|
||||
try:
|
||||
refreshed = await self.refresh_credentials()
|
||||
except Exception as e:
|
||||
logger.warning(f"refresh_credentials raised: {e}")
|
||||
refreshed = False
|
||||
if refreshed:
|
||||
continue
|
||||
break
|
||||
await self._note_session_invalid(self.last_error)
|
||||
return False, None
|
||||
|
||||
async def _note_session_invalid(self, error: str) -> None:
|
||||
"""根据发送失败原因判断 IM 是否已退出登录；连续 INVALID_REQUEST 即触发自动下线。
|
||||
|
||||
INVALID_REQUEST 来自 create_conversation/发送：会话/签名被抖音判为无效，强相关于「登录失效」。
|
||||
而 8xxx/7xxx 等业务错误（关系/频控/内容）说明请求已到达抖音、登录仍有效，重置计数。
|
||||
"""
|
||||
err = error or ""
|
||||
if "INVALID_REQUEST" not in err:
|
||||
self._session_invalid_strikes = 0
|
||||
return
|
||||
self._session_invalid_strikes += 1
|
||||
if self._session_invalid_strikes < 2 or self._session_invalid_fired:
|
||||
return
|
||||
self._session_invalid_fired = True
|
||||
reason = "IM 会话失效(INVALID_REQUEST)，登录可能已退出"
|
||||
logger.warning(
|
||||
f"Account {self.account_id} {reason}；连续 {self._session_invalid_strikes} 次 -> 自动下线"
|
||||
)
|
||||
system_logger.record(
|
||||
"IM 登录失效，自动下线",
|
||||
detail=f"{reason}（连续 {self._session_invalid_strikes} 次发送返回 INVALID_REQUEST）。"
|
||||
"请停止托管后用浏览器模式重新登录并打开私信页，再重新启动托管。",
|
||||
level="error",
|
||||
category="auth",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
self._running = False # 让主循环尽快退出
|
||||
if self.on_session_invalid:
|
||||
try:
|
||||
await self.on_session_invalid(reason)
|
||||
except Exception as e:
|
||||
logger.error(f"on_session_invalid handler error: {e}")
|
||||
|
||||
async def send_message(self, conversation_id: str, content: str) -> bool:
|
||||
"""手动发送私信"""
|
||||
from .conv_util import normalize_conversation_id
|
||||
from .auth import DouyinAuth
|
||||
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(
|
||||
self.session.cookie_header(),
|
||||
self.session.web_protect_str,
|
||||
self.session.keys_str,
|
||||
)
|
||||
if getattr(self.session, "uid_verified", False) and self.session.my_uid:
|
||||
my_uid = self.session.my_uid
|
||||
else:
|
||||
my_uid = await asyncio.to_thread(lambda: auth.get_uid()) or self.session.my_uid
|
||||
if my_uid:
|
||||
conversation_id = normalize_conversation_id(conversation_id, my_uid)
|
||||
|
||||
meta = self._conv_meta.get(conversation_id, {})
|
||||
sent, resolved = await self._send_text(
|
||||
conversation_id,
|
||||
content,
|
||||
conversation_short_id=str(meta.get("conversation_short_id") or ""),
|
||||
)
|
||||
if sent and resolved:
|
||||
self._conv_meta[conversation_id] = {
|
||||
**meta,
|
||||
**resolved,
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
return sent
|
||||
@@ -0,0 +1,260 @@
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Optional
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
IM_TOKEN_COOKIES = ("sessionid", "sessionid_ss")
|
||||
|
||||
|
||||
def is_frontier_ws_url(url: str) -> bool:
|
||||
"""判断是否为抖音 IM frontier 长连接地址。
|
||||
|
||||
真实抓包里 host 可能是 frontier-im.douyin.com，也可能是
|
||||
frontierNN-normal.zijieapi.com 这类内部别名，二者都要认。
|
||||
"""
|
||||
if not url or "token=" not in url:
|
||||
return False
|
||||
return "frontier-im.douyin.com" in url or ("frontier" in url and "zijieapi.com" in url)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DouyinImSession:
|
||||
"""抖音 IM 直连所需会话信息（从 Cookie + 浏览器抓包获得）"""
|
||||
|
||||
cookies: dict = field(default_factory=dict)
|
||||
ws_urls: list = field(default_factory=list)
|
||||
device_id: str = ""
|
||||
web_id: str = ""
|
||||
user_agent: str = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
keys_str: str = ""
|
||||
web_protect_str: str = ""
|
||||
my_uid: int = 0
|
||||
# my_uid 是否已用 query/user 接口核验过（采集端推断的 my_uid 可能取错 tea_cache id）
|
||||
uid_verified: bool = False
|
||||
conv_meta: dict = field(default_factory=dict)
|
||||
# 方案 A：直接复用浏览器抓到的真实 frontier 连接凭证（绕开我们自己推导 token/access_key 不准的问题）
|
||||
sdk_cert: str = "" # bd-ticket-guard 客户端证书（frontier sdk_cert / HTTP client-cert）
|
||||
frontier_ts_sign: str = "" # 抓包得到的新鲜 ts_sign（覆盖 web_protect 里可能已过期的）
|
||||
|
||||
@classmethod
|
||||
def from_storage_state(cls, data: dict, extra: Optional[dict] = None) -> "DouyinImSession":
|
||||
extra = extra or {}
|
||||
cookies = {}
|
||||
cookie_items = data.get("cookies", [])
|
||||
priority_names = set(IM_TOKEN_COOKIES)
|
||||
|
||||
def pick_best_cookie(name: str) -> str:
|
||||
matches = [
|
||||
c for c in cookie_items
|
||||
if c.get("name") == name and c.get("value")
|
||||
]
|
||||
if not matches:
|
||||
return ""
|
||||
matches.sort(
|
||||
key=lambda c: (
|
||||
0 if ".douyin.com" in (c.get("domain") or "") else 1,
|
||||
-len(c.get("value") or ""),
|
||||
)
|
||||
)
|
||||
return matches[0].get("value") or ""
|
||||
|
||||
for item in cookie_items:
|
||||
name = item.get("name")
|
||||
if not name or any(c in name for c in "()[]{}'\"\n \t\\"):
|
||||
continue
|
||||
val = item.get("value") or ""
|
||||
if name in priority_names:
|
||||
best = pick_best_cookie(name)
|
||||
if best:
|
||||
cookies[name] = best
|
||||
elif name not in cookies or len(val) > len(cookies.get(name, "")):
|
||||
cookies[name] = val
|
||||
|
||||
device_id = extra.get("device_id") or cookies.get("device_id") or ""
|
||||
web_id = extra.get("web_id") or ""
|
||||
|
||||
keys_str = extra.get("keys_str") or ""
|
||||
web_protect_str = extra.get("web_protect_str") or ""
|
||||
|
||||
# my_uid 优先级：浏览器实时采集(extra) > storage_state 顶层(凭证采集工具手填/抓取)。
|
||||
# 顶层 my_uid 让导入的明文数字 UID 直接生效，避免后端用加密 uid_tt 解析失败而回退联网查询。
|
||||
def _as_uid(v) -> int:
|
||||
try:
|
||||
return int(str(v).strip())
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
my_uid = _as_uid(extra.get("my_uid")) or _as_uid(data.get("my_uid"))
|
||||
user_agent = str(extra.get("user_agent") or data.get("user_agent") or "").strip()
|
||||
|
||||
if not device_id or not web_id or not keys_str or not web_protect_str or not my_uid:
|
||||
for origin in data.get("origins", []):
|
||||
for entry in origin.get("localStorage", []):
|
||||
name = entry.get("name", "")
|
||||
value = entry.get("value", "")
|
||||
if not value:
|
||||
continue
|
||||
if name == "security-sdk/s_sdk_crypt_sdk" and not keys_str:
|
||||
keys_str = value
|
||||
if name == "security-sdk/s_sdk_sign_data_key/web_protect" and not web_protect_str:
|
||||
web_protect_str = value
|
||||
if "tea_cache_tokens" in name and not web_id:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
web_id = str(
|
||||
parsed.get("web_id")
|
||||
or parsed.get("user_unique_id")
|
||||
or ""
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
if name == "web_runtime_security_uid" and not device_id:
|
||||
if str(value or "").isdigit():
|
||||
device_id = value
|
||||
if "tea_cache_tokens" in name and not my_uid:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
uid = parsed.get("user_unique_id")
|
||||
if uid and str(uid).isdigit():
|
||||
my_uid = int(uid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not my_uid:
|
||||
for item in data.get("cookies", []):
|
||||
cname = item.get("name") or ""
|
||||
if cname in ("uid_tt", "uid_tt_ss") and item.get("value"):
|
||||
try:
|
||||
my_uid = int(item.get("value"))
|
||||
break
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
|
||||
if not device_id and my_uid:
|
||||
device_id = str(my_uid)
|
||||
elif not device_id and web_id:
|
||||
device_id = web_id
|
||||
|
||||
ws_urls = list(extra.get("ws_urls") or [])
|
||||
|
||||
# 方案 A：凭证采集工具可携带浏览器抓到的真实 frontier 连接（含 token/sdk_cert/ts_sign）。
|
||||
frontier_ws_url = str(
|
||||
extra.get("frontier_ws_url") or data.get("frontier_ws_url") or ""
|
||||
).strip()
|
||||
sdk_cert = str(extra.get("sdk_cert") or data.get("sdk_cert") or "").strip()
|
||||
frontier_ts_sign = str(
|
||||
extra.get("frontier_ts_sign")
|
||||
or data.get("frontier_ts_sign")
|
||||
or data.get("ts_sign")
|
||||
or ""
|
||||
).strip()
|
||||
if is_frontier_ws_url(frontier_ws_url):
|
||||
# 真实抓包 URL 优先，放在最前面
|
||||
ws_urls = [frontier_ws_url] + [u for u in ws_urls if u != frontier_ws_url]
|
||||
# 从真实 URL 里补抽 sdk_cert / ts_sign（用户只贴了 URL 时）。
|
||||
# 注意：不能用 parse_qs（它会把 + 解成空格，毁掉 base64 证书），用 unquote。
|
||||
def _q(url: str, key: str) -> str:
|
||||
m = re.search(rf"[?&]{re.escape(key)}=([^&\s]+)", url)
|
||||
return unquote(m.group(1)) if m else ""
|
||||
if not sdk_cert:
|
||||
sdk_cert = _q(frontier_ws_url, "sdk_cert")
|
||||
if not frontier_ts_sign:
|
||||
frontier_ts_sign = _q(frontier_ws_url, "ts_sign")
|
||||
|
||||
return cls(
|
||||
cookies=cookies,
|
||||
ws_urls=ws_urls,
|
||||
device_id=str(device_id or ""),
|
||||
web_id=str(web_id or ""),
|
||||
keys_str=keys_str,
|
||||
web_protect_str=web_protect_str,
|
||||
my_uid=my_uid,
|
||||
user_agent=user_agent if user_agent else cls.user_agent,
|
||||
sdk_cert=sdk_cert,
|
||||
frontier_ts_sign=frontier_ts_sign,
|
||||
)
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"cookies": self.cookies,
|
||||
"ws_urls": self.ws_urls,
|
||||
"device_id": self.device_id,
|
||||
"web_id": self.web_id,
|
||||
"user_agent": self.user_agent,
|
||||
"keys_str": self.keys_str,
|
||||
"web_protect_str": self.web_protect_str,
|
||||
"my_uid": self.my_uid,
|
||||
"conv_meta": self.conv_meta,
|
||||
"sdk_cert": self.sdk_cert,
|
||||
"frontier_ts_sign": self.frontier_ts_sign,
|
||||
"saved_at": time.time(),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict) -> "DouyinImSession":
|
||||
if not data:
|
||||
return cls()
|
||||
return cls(
|
||||
cookies=data.get("cookies") or {},
|
||||
ws_urls=data.get("ws_urls") or [],
|
||||
device_id=str(data.get("device_id") or ""),
|
||||
web_id=str(data.get("web_id") or ""),
|
||||
user_agent=data.get("user_agent") or cls.user_agent,
|
||||
keys_str=str(data.get("keys_str") or ""),
|
||||
web_protect_str=str(data.get("web_protect_str") or ""),
|
||||
my_uid=int(data.get("my_uid") or 0),
|
||||
conv_meta=dict(data.get("conv_meta") or {}),
|
||||
sdk_cert=str(data.get("sdk_cert") or ""),
|
||||
frontier_ts_sign=str(data.get("frontier_ts_sign") or ""),
|
||||
)
|
||||
|
||||
def cookie_header(self) -> str:
|
||||
parts = []
|
||||
for name, value in self.cookies.items():
|
||||
if name and value is not None:
|
||||
parts.append(f"{name}={value}")
|
||||
return "; ".join(parts)
|
||||
|
||||
def has_login(self) -> bool:
|
||||
login_keys = {
|
||||
"sessionid",
|
||||
"sessionid_ss",
|
||||
"sid_tt",
|
||||
"sid_guard",
|
||||
"passport_auth_status",
|
||||
"odin_tt",
|
||||
}
|
||||
return any(self.cookies.get(k) for k in login_keys)
|
||||
|
||||
def can_direct_im(self) -> bool:
|
||||
"""是否具备 Cookie 直连 IM 的最低条件（无需浏览器）"""
|
||||
return self.has_login() and bool(
|
||||
self.cookies.get("sessionid") or self.cookies.get("sessionid_ss")
|
||||
)
|
||||
|
||||
def frontier_ws_url(self) -> Optional[str]:
|
||||
"""仅返回 IM frontier 地址，忽略浏览器抓到的 bytelink 等无关 WS。"""
|
||||
for url in self.ws_urls:
|
||||
if is_frontier_ws_url(url):
|
||||
return url
|
||||
return None
|
||||
|
||||
def sanitize_ws_urls(self) -> None:
|
||||
self.ws_urls = [url for url in self.ws_urls if is_frontier_ws_url(url)]
|
||||
|
||||
def common_params(self) -> dict[str, str]:
|
||||
return {
|
||||
"aid": "6383",
|
||||
"app_name": "douyin_web",
|
||||
"device_platform": "webapp",
|
||||
"channel": "channel_pc_web",
|
||||
"pc_client_type": "1",
|
||||
"version_code": "170400",
|
||||
"version_name": "17.4.0",
|
||||
"device_id": self.device_id or self.web_id or "",
|
||||
"webid": self.web_id or self.device_id or "",
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,60 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# NO CHECKED-IN PROTOBUF GENCODE
|
||||
# source: Request.proto
|
||||
# Protobuf Python Version: 5.27.1
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import descriptor_pool as _descriptor_pool
|
||||
from google.protobuf import runtime_version as _runtime_version
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf.internal import builder as _builder
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(
|
||||
_runtime_version.Domain.PUBLIC,
|
||||
5,
|
||||
27,
|
||||
1,
|
||||
'',
|
||||
'Request.proto'
|
||||
)
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rRequest.proto\"&\n\x08\x45xtValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\"\x94\x04\n\x07Request\x12\x0b\n\x03\x63md\x18\x01 \x01(\x05\x12\x13\n\x0bsequence_id\x18\x02 \x01(\x03\x12\x13\n\x0bsdk_version\x18\x03 \x01(\t\x12\r\n\x05token\x18\x04 \x01(\t\x12\r\n\x05refer\x18\x05 \x01(\x05\x12\x12\n\ninbox_type\x18\x06 \x01(\x03\x12\x14\n\x0c\x62uild_number\x18\x07 \x01(\t\x12\x1a\n\x04\x62ody\x18\x08 \x01(\x0b\x32\x0c.RequestBody\x12\x11\n\tdevice_id\x18\t \x01(\t\x12\x0f\n\x07\x63hannel\x18\n \x01(\t\x12\x17\n\x0f\x64\x65vice_platform\x18\x0b \x01(\t\x12\x13\n\x0b\x64\x65vice_type\x18\x0c \x01(\t\x12\x12\n\nos_version\x18\r \x01(\t\x12\x14\n\x0cversion_code\x18\x0e \x01(\t\x12&\n\x07headers\x18\x0f \x03(\x0b\x32\x15.Request.HeadersEntry\x12\x11\n\tconfig_id\x18\x10 \x01(\x05\x12\x1e\n\ntoken_info\x18\x11 \x01(\x0b\x32\n.TokenInfo\x12\x11\n\tauth_type\x18\x12 \x01(\x05\x12\x0b\n\x03\x62iz\x18\x15 \x01(\t\x12\x0e\n\x06\x61\x63\x63\x65ss\x18\x16 \x01(\t\x12\x0f\n\x07ts_sign\x18\x17 \x01(\t\x12\x10\n\x08sdk_cert\x18\x18 \x01(\t\x12\x14\n\x0creuqest_sign\x18\x19 \x01(\t\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xeb\x01\n\x0bRequestBody\x12\x34\n\x11send_message_body\x18\x64 \x01(\x0b\x32\x17.SendMessageRequestBodyH\x00\x12H\n\x1b\x63reate_conversation_v2_body\x18\xe1\x04 \x01(\x0b\x32 .CreateConversationV2RequestBodyH\x00\x12T\n\"get_conversation_info_list_v2_body\x18\xe2\x04 \x01(\x0b\x32%.GetConversationInfoListV2RequestBodyH\x00\x42\x06\n\x04\x62ody\"\xb8\x02\n\x16SendMessageRequestBody\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x02 \x01(\x05\x12\x1d\n\x15\x63onversation_short_id\x18\x03 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x04 \x01(\t\x12\x16\n\x03\x65xt\x18\x05 \x03(\x0b\x32\t.ExtValue\x12\x14\n\x0cmessage_type\x18\x06 \x01(\x05\x12\x0e\n\x06ticket\x18\x07 \x01(\t\x12\x19\n\x11\x63lient_message_id\x18\x08 \x01(\t\x12\x17\n\x0fmentioned_users\x18\t \x03(\x03\x12\x1a\n\x12ignore_badge_count\x18\n \x01(\x08\x12,\n\x0cref_msg_info\x18\x0b \x01(\x0b\x32\x16.ReferencedMessageInfo\"y\n\x15ReferencedMessageInfo\x12\x1b\n\x13original_message_id\x18\x01 \x01(\x03\x12\x1f\n\x17original_message_sender\x18\x02 \x01(\t\x12\"\n\x1aoriginal_message_timestamp\x18\x03 \x01(\x03\"^\n\tTokenInfo\x12\x0f\n\x07mark_id\x18\x01 \x01(\x05\x12\x0c\n\x04type\x18\x02 \x01(\x05\x12\x0e\n\x06\x61pp_id\x18\x03 \x01(\x05\x12\x0f\n\x07user_id\x18\x04 \x01(\x03\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\"\xa2\x02\n\x1f\x43reateConversationV2RequestBody\x12\x19\n\x11\x63onversation_type\x18\x01 \x01(\x05\x12\x14\n\x0cparticipants\x18\x02 \x03(\x03\x12\x12\n\npersistent\x18\x03 \x01(\x08\x12\x15\n\ridempotent_id\x18\x04 \x01(\t\x12\x0c\n\x04name\x18\x05 \x01(\t\x12\x12\n\navatar_url\x18\x06 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x07 \x01(\t\x12=\n\x07\x62iz_ext\x18\x08 \x03(\x0b\x32,.CreateConversationV2RequestBody.BizExtEntry\x1a-\n\x0b\x42izExtEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"`\n$GetConversationInfoListV2RequestBody\x12\x38\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32*.GetConversationInfoListV2ResponseBodyData\"~\n)GetConversationInfoListV2ResponseBodyData\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x1d\n\x15\x63onversation_short_id\x18\x02 \x01(\x03\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x62\x06proto3')
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'Request_pb2', _globals)
|
||||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
DESCRIPTOR._loaded_options = None
|
||||
_globals['_REQUEST_HEADERSENTRY']._loaded_options = None
|
||||
_globals['_REQUEST_HEADERSENTRY']._serialized_options = b'8\001'
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._loaded_options = None
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_options = b'8\001'
|
||||
_globals['_EXTVALUE']._serialized_start=17
|
||||
_globals['_EXTVALUE']._serialized_end=55
|
||||
_globals['_REQUEST']._serialized_start=58
|
||||
_globals['_REQUEST']._serialized_end=590
|
||||
_globals['_REQUEST_HEADERSENTRY']._serialized_start=544
|
||||
_globals['_REQUEST_HEADERSENTRY']._serialized_end=590
|
||||
_globals['_REQUESTBODY']._serialized_start=593
|
||||
_globals['_REQUESTBODY']._serialized_end=828
|
||||
_globals['_SENDMESSAGEREQUESTBODY']._serialized_start=831
|
||||
_globals['_SENDMESSAGEREQUESTBODY']._serialized_end=1143
|
||||
_globals['_REFERENCEDMESSAGEINFO']._serialized_start=1145
|
||||
_globals['_REFERENCEDMESSAGEINFO']._serialized_end=1266
|
||||
_globals['_TOKENINFO']._serialized_start=1268
|
||||
_globals['_TOKENINFO']._serialized_end=1362
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY']._serialized_start=1365
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY']._serialized_end=1655
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_start=1610
|
||||
_globals['_CREATECONVERSATIONV2REQUESTBODY_BIZEXTENTRY']._serialized_end=1655
|
||||
_globals['_GETCONVERSATIONINFOLISTV2REQUESTBODY']._serialized_start=1657
|
||||
_globals['_GETCONVERSATIONINFOLISTV2REQUESTBODY']._serialized_end=1753
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODYDATA']._serialized_start=1755
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODYDATA']._serialized_end=1881
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -0,0 +1,46 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# NO CHECKED-IN PROTOBUF GENCODE
|
||||
# source: Response.proto
|
||||
# Protobuf Python Version: 5.27.1
|
||||
"""Generated protocol buffer code."""
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import descriptor_pool as _descriptor_pool
|
||||
from google.protobuf import runtime_version as _runtime_version
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf.internal import builder as _builder
|
||||
_runtime_version.ValidateProtobufRuntimeVersion(
|
||||
_runtime_version.Domain.PUBLIC,
|
||||
5,
|
||||
27,
|
||||
1,
|
||||
'',
|
||||
'Response.proto'
|
||||
)
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0eResponse.proto\"\x82\x01\n\x08Response\x12\x0b\n\x03\x63md\x18\x01 \x01(\x05\x12\x13\n\x0bsequence_id\x18\x02 \x01(\x03\x12\x12\n\nerror_desc\x18\x03 \x01(\t\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\x12\n\ninbox_type\x18\x05 \x01(\x03\x12\x1b\n\x04\x62ody\x18\x06 \x01(\x0b\x32\r.ResponseBody\"\xf8\x01\n\x0cResponseBody\x12\x30\n\x12new_message_notify\x18\xf4\x03 \x01(\x0b\x32\x11.NewMessageNotifyH\x00\x12N\n\x1b\x63reate_conversation_v2_body\x18\xe1\x04 \x01(\x0b\x32&.GetConversationInfoListV2ResponseBodyH\x00\x12^\n+get_conversation_info_list_v2_response_body\x18\xe2\x04 \x01(\x0b\x32&.GetConversationInfoListV2ResponseBodyH\x00\x42\x06\n\x04\x62ody\"z\n\x10NewMessageNotify\x12\x17\n\x0f\x63onversation_id\x18\x02 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x12\x13\n\x0bnotify_type\x18\x04 \x01(\x05\x12\x1d\n\x07message\x18\x05 \x01(\x0b\x32\x0c.MessageBody\"\xd1\x01\n\x0bMessageBody\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x19\n\x11\x63onversation_type\x18\x02 \x01(\x05\x12\x19\n\x11server_message_id\x18\x03 \x01(\x03\x12\x1d\n\x15index_in_conversation\x18\x04 \x01(\x03\x12\x1d\n\x15\x63onversation_short_id\x18\x05 \x01(\x03\x12\x14\n\x0cmessage_type\x18\x06 \x01(\x05\x12\x0e\n\x06sender\x18\x07 \x01(\x03\x12\x0f\n\x07\x63ontent\x18\x08 \x01(\t\"g\n%GetConversationInfoListV2ResponseBody\x12>\n\x16\x63onversation_info_list\x18\x01 \x03(\x0b\x32\x1e.GetConversationInfoV2Response\"\x82\x01\n\x1dGetConversationInfoV2Response\x12\x17\n\x0f\x63onversation_id\x18\x01 \x01(\t\x12\x1d\n\x15\x63onversation_short_id\x18\x02 \x01(\x03\x12\x19\n\x11\x63onversation_type\x18\x03 \x01(\x05\x12\x0e\n\x06ticket\x18\x04 \x01(\tb\x06proto3')
|
||||
|
||||
_globals = globals()
|
||||
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
|
||||
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'Response_pb2', _globals)
|
||||
if not _descriptor._USE_C_DESCRIPTORS:
|
||||
DESCRIPTOR._loaded_options = None
|
||||
_globals['_RESPONSE']._serialized_start=19
|
||||
_globals['_RESPONSE']._serialized_end=149
|
||||
_globals['_RESPONSEBODY']._serialized_start=152
|
||||
_globals['_RESPONSEBODY']._serialized_end=400
|
||||
_globals['_NEWMESSAGENOTIFY']._serialized_start=402
|
||||
_globals['_NEWMESSAGENOTIFY']._serialized_end=524
|
||||
_globals['_MESSAGEBODY']._serialized_start=527
|
||||
_globals['_MESSAGEBODY']._serialized_end=736
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODY']._serialized_start=738
|
||||
_globals['_GETCONVERSATIONINFOLISTV2RESPONSEBODY']._serialized_end=841
|
||||
_globals['_GETCONVERSATIONINFOV2RESPONSE']._serialized_start=844
|
||||
_globals['_GETCONVERSATIONINFOV2RESPONSE']._serialized_end=974
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,167 @@
|
||||
ea = function() {
|
||||
return (ea = Object.assign || function(e) {
|
||||
for (var t, n = 1, r = arguments.length; n < r; n++)
|
||||
for (var o in t = arguments[n])
|
||||
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
|
||||
return e
|
||||
}
|
||||
).apply(this, arguments)
|
||||
}
|
||||
var H = function(e) {
|
||||
for (var t, n = e.toString(), r = [], o = 0; o < n.length; o++)
|
||||
0 <= (t = n.charCodeAt(o)) && t <= 127 ? r.push(t) : 128 <= t && t <= 2047 ? (r.push(192 | 31 & t >> 6),
|
||||
r.push(128 | 63 & t)) : (2048 <= t && t <= 55295 || 57344 <= t && t <= 65535) && (r.push(224 | 15 & t >> 12),
|
||||
r.push(128 | 63 & t >> 6),
|
||||
r.push(128 | 63 & t));
|
||||
for (var i = 0; i < r.length; i++)
|
||||
r[i] &= 255;
|
||||
return r
|
||||
}
|
||||
V = function(e) {
|
||||
var t = []
|
||||
, n = [];
|
||||
if (void 0 === e)
|
||||
return "";
|
||||
n = H(e);
|
||||
for (var r = 0, o = n.length; r < o; ++r)
|
||||
t.push((5 ^ n[r]).toString(16));
|
||||
return t.join("")
|
||||
}
|
||||
function generate_account_sdk_source_info(){
|
||||
let browserInfo = {
|
||||
"hardwareConcurrency": 20,
|
||||
"webdriver": false,
|
||||
"chromedriver": false,
|
||||
"shelldriver": false,
|
||||
"plugins": 5,
|
||||
"permissions": [
|
||||
{
|
||||
"name": "notifications",
|
||||
"state": "prompt"
|
||||
}
|
||||
],
|
||||
"innerHeight": 1442,
|
||||
"innerWidth": 1166,
|
||||
"outerHeight": 1552,
|
||||
"outerWidth": 2560,
|
||||
"stoargeStatus": {
|
||||
"indexedDB": {
|
||||
"idb": "object",
|
||||
"open": "function",
|
||||
"indexedDB": "object",
|
||||
"IDBKeyRange": "function",
|
||||
"openDatabase": "undefined",
|
||||
"isSafari": false,
|
||||
"hasFetch": false
|
||||
},
|
||||
"localStorage": {
|
||||
"isSupportLStorage": true,
|
||||
"size": 46382,
|
||||
"write": true
|
||||
},
|
||||
"storageQuotaStatus": {
|
||||
"usage": 149822,
|
||||
"quota": 128849645568,
|
||||
"isPrivate": false
|
||||
}
|
||||
},
|
||||
"notificationPermission": "default",
|
||||
"performance": {
|
||||
// "timeOrigin": 1723036093298,
|
||||
"timeOrigin": new Date().getTime(),
|
||||
"usedJSHeapSize": 137509473,
|
||||
"navigationTiming": {
|
||||
"decodedBodySize": 633258,
|
||||
"entryType": "navigation",
|
||||
"initiatorType": "navigation",
|
||||
"name": "https://www.douyin.com/?recommend=1",
|
||||
"renderBlockingStatus": "non-blocking",
|
||||
"serverTiming": "inner,tt_agw,cdn-cache,edge,origin",
|
||||
"guleStart": 1496.2999999523163,
|
||||
"guleDuration": 14.799999952316284
|
||||
}
|
||||
}
|
||||
}
|
||||
let data = {
|
||||
request_host: "www.douyin.com",
|
||||
request_pathname: '/'
|
||||
}
|
||||
return V(JSON.stringify(ea(ea({}, browserInfo || {}), data)))
|
||||
}
|
||||
// console.log(generate_account_sdk_source_info())
|
||||
|
||||
|
||||
eD = function(e, t) {
|
||||
var n, r = 0, o = 0;
|
||||
if ("object" != typeof e || !t || t.length <= 0)
|
||||
return e;
|
||||
for (var i = ej({
|
||||
mix_mode: r
|
||||
}, e), a = 0, c = t.length; a < c; ++a)
|
||||
void 0 !== (n = i[t[a]]) && (r |= 1,
|
||||
o |= 1,
|
||||
i[t[a]] = eN(n));
|
||||
return i.mix_mode = r,
|
||||
i.fixed_mix_mode = o,
|
||||
i
|
||||
}
|
||||
var ek = function(e) {
|
||||
for (var t, n = e.toString(), r = [], o = 0; o < n.length; o++)
|
||||
0 <= (t = n.charCodeAt(o)) && t <= 127 ? r.push(t) : 128 <= t && t <= 2047 ? (r.push(192 | 31 & t >> 6),
|
||||
r.push(128 | 63 & t)) : (2048 <= t && t <= 55295 || 57344 <= t && t <= 65535) && (r.push(224 | 15 & t >> 12),
|
||||
r.push(128 | 63 & t >> 6),
|
||||
r.push(128 | 63 & t));
|
||||
for (var i = 0; i < r.length; i++)
|
||||
r[i] &= 255;
|
||||
return r
|
||||
}
|
||||
eN = function(e) {
|
||||
var t = []
|
||||
, n = [];
|
||||
if (void 0 === e)
|
||||
return "";
|
||||
n = ek(e);
|
||||
for (var r = 0, o = n.length; r < o; ++r)
|
||||
t.push((5 ^ n[r]).toString(16));
|
||||
return t.join("")
|
||||
}
|
||||
ej = function() {
|
||||
return (ej = Object.assign || function(e) {
|
||||
for (var t, n = 1, r = arguments.length; n < r; n++)
|
||||
for (var o in t = arguments[n])
|
||||
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
|
||||
return e
|
||||
}
|
||||
).apply(this, arguments)
|
||||
};
|
||||
function generateSecretPhoneNum(phoneNum) {
|
||||
return eD({
|
||||
"mobile": "+86 " + phoneNum,
|
||||
"type": 24,
|
||||
"is6Digits": 1
|
||||
}, ["mobile", "type"])
|
||||
}
|
||||
// let res = eD({
|
||||
// "mobile": "+86 15751076989",
|
||||
// "type": 24,
|
||||
// "is6Digits": 1
|
||||
// }, ["mobile", "type"])
|
||||
//
|
||||
// console.log(res)
|
||||
nx = function() {
|
||||
return (nx = Object.assign || function(e) {
|
||||
for (var t, n = 1, r = arguments.length; n < r; n++)
|
||||
for (var o in t = arguments[n])
|
||||
Object.prototype.hasOwnProperty.call(t, o) && (e[o] = t[o]);
|
||||
return e
|
||||
}
|
||||
).apply(this, arguments)
|
||||
}
|
||||
function generateSecretCode(phoneNum, code) {
|
||||
return eD({
|
||||
"mobile": "+86 " + phoneNum,
|
||||
"code": code,
|
||||
"service": "https://www.douyin.com"
|
||||
}, ["mobile", "code", "password"])
|
||||
}
|
||||
// console.log(generateSecretCode("15751076989", "090625"))
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"name": "static",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"dependencies": {
|
||||
"jsrsasign": "^11.1.0"
|
||||
}
|
||||
},
|
||||
"node_modules/jsrsasign": {
|
||||
"version": "11.1.3",
|
||||
"resolved": "https://registry.npmjs.org/jsrsasign/-/jsrsasign-11.1.3.tgz",
|
||||
"integrity": "sha512-nPnK5D/4lv0Dwr7TlzrKtAd8JlLZwFTqTUUB3NQCbtdobcRcohGFxjbPySDVh74iWUudcCsapYT6OxoyhJLhhA==",
|
||||
"license": "MIT"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"jsrsasign": "^11.1.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,537 @@
|
||||
"""Process-wide traffic control for hosted Douyin accounts.
|
||||
|
||||
The backend normally runs one asyncio event loop. A controller is kept per
|
||||
event loop so unit tests that create a fresh loop for every test do not reuse
|
||||
asyncio primitives bound to an older loop.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import weakref
|
||||
from collections import deque
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Awaitable, Callable, Optional, TypeVar
|
||||
|
||||
|
||||
logger = logging.getLogger("douyin_im.traffic")
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def _env_float(name: str, default: float, minimum: float = 0.0) -> float:
|
||||
try:
|
||||
return max(minimum, float(os.getenv(name, str(default))))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def _env_int(name: str, default: int, minimum: int = 1) -> int:
|
||||
try:
|
||||
return max(minimum, int(os.getenv(name, str(default))))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@dataclass(eq=False)
|
||||
class _SendJob:
|
||||
account_id: int
|
||||
operation: Callable[[], Awaitable[Any]]
|
||||
future: asyncio.Future
|
||||
description: str
|
||||
queued_at: float
|
||||
cancelled: bool = False
|
||||
operation_task: Optional[asyncio.Task] = None
|
||||
|
||||
|
||||
class GlobalSendQueue:
|
||||
"""One paced, account-fair dispatcher for every outbound IM write.
|
||||
|
||||
Each account owns a FIFO. After one job is selected, that account goes to
|
||||
the back of the rotation. A busy account therefore cannot starve smaller
|
||||
accounts, while messages within one account retain their original order.
|
||||
"""
|
||||
|
||||
def __init__(self, interval_seconds: float = 1.0) -> None:
|
||||
self.interval_seconds = max(0.0, float(interval_seconds or 0.0))
|
||||
self._queues: dict[int, deque[_SendJob]] = {}
|
||||
self._rotation: deque[int] = deque()
|
||||
self._in_rotation: set[int] = set()
|
||||
self._active_job: Optional[_SendJob] = None
|
||||
self._active_account: Optional[int] = None
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._running = False
|
||||
self._state_lock = asyncio.Lock()
|
||||
self._wake = asyncio.Event()
|
||||
self._next_allowed_at = 0.0
|
||||
|
||||
@property
|
||||
def queued_count(self) -> int:
|
||||
return sum(len(queue) for queue in self._queues.values())
|
||||
|
||||
@property
|
||||
def pending_count(self) -> int:
|
||||
return self.queued_count + (1 if self._active_job is not None else 0)
|
||||
|
||||
async def start(self) -> None:
|
||||
async with self._state_lock:
|
||||
if self._task and not self._task.done():
|
||||
return
|
||||
self._running = True
|
||||
self._wake.clear()
|
||||
self._task = asyncio.create_task(
|
||||
self._run(),
|
||||
name="douyin-global-send-queue",
|
||||
)
|
||||
|
||||
async def submit(
|
||||
self,
|
||||
account_id: int,
|
||||
operation: Callable[[], Awaitable[T]],
|
||||
description: str = "",
|
||||
) -> T:
|
||||
await self.start()
|
||||
loop = asyncio.get_running_loop()
|
||||
account_key = int(account_id or 0)
|
||||
future: asyncio.Future = loop.create_future()
|
||||
job = _SendJob(
|
||||
account_id=account_key,
|
||||
operation=operation,
|
||||
future=future,
|
||||
description=str(description or "send"),
|
||||
queued_at=time.time(),
|
||||
)
|
||||
|
||||
async with self._state_lock:
|
||||
queue = self._queues.setdefault(account_key, deque())
|
||||
queue.append(job)
|
||||
if account_key != self._active_account and account_key not in self._in_rotation:
|
||||
self._rotation.append(account_key)
|
||||
self._in_rotation.add(account_key)
|
||||
position = self.pending_count
|
||||
self._wake.set()
|
||||
|
||||
if position > 1:
|
||||
logger.info(
|
||||
"Outbound queued account=%s position=%s pending=%s (%s)",
|
||||
account_key,
|
||||
position,
|
||||
position,
|
||||
job.description,
|
||||
)
|
||||
|
||||
try:
|
||||
# Shield lets us remove/cancel the exact queued job when its caller
|
||||
# is cancelled instead of asyncio cancelling only the result future.
|
||||
return await asyncio.shield(future)
|
||||
except asyncio.CancelledError:
|
||||
active = await self._cancel_job(job)
|
||||
if active:
|
||||
# Once the network operation has begun, return its real result
|
||||
# instead of reporting a timeout and then delivering later.
|
||||
# This also keeps the single lane occupied until a to_thread
|
||||
# upload has truly finished.
|
||||
return await asyncio.shield(future)
|
||||
raise
|
||||
|
||||
async def _cancel_job(self, job: _SendJob) -> bool:
|
||||
async with self._state_lock:
|
||||
if job.cancelled:
|
||||
return bool(job.operation_task and not job.operation_task.done())
|
||||
if (
|
||||
self._active_job is job
|
||||
and job.operation_task is not None
|
||||
):
|
||||
# An active image upload may be running in asyncio.to_thread().
|
||||
# Cancelling its awaiter does not stop the thread and would let
|
||||
# the next send overlap it. Let the active job drain and make
|
||||
# the cancelled caller wait for its authoritative result.
|
||||
return True
|
||||
|
||||
job.cancelled = True
|
||||
queue = self._queues.get(job.account_id)
|
||||
if queue:
|
||||
try:
|
||||
queue.remove(job)
|
||||
except ValueError:
|
||||
pass
|
||||
if not queue:
|
||||
self._queues.pop(job.account_id, None)
|
||||
self._remove_from_rotation(job.account_id)
|
||||
if not job.future.done():
|
||||
job.future.cancel()
|
||||
self._wake.set()
|
||||
return False
|
||||
|
||||
def _remove_from_rotation(self, account_id: int) -> None:
|
||||
if account_id not in self._in_rotation:
|
||||
return
|
||||
self._in_rotation.discard(account_id)
|
||||
try:
|
||||
self._rotation.remove(account_id)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def _pop_next_locked(self) -> Optional[_SendJob]:
|
||||
while self._rotation:
|
||||
account_id = self._rotation.popleft()
|
||||
self._in_rotation.discard(account_id)
|
||||
queue = self._queues.get(account_id)
|
||||
if not queue:
|
||||
self._queues.pop(account_id, None)
|
||||
continue
|
||||
|
||||
while queue and (queue[0].cancelled or queue[0].future.cancelled()):
|
||||
queue.popleft()
|
||||
if not queue:
|
||||
self._queues.pop(account_id, None)
|
||||
continue
|
||||
|
||||
job = queue.popleft()
|
||||
if not queue:
|
||||
self._queues.pop(account_id, None)
|
||||
self._active_job = job
|
||||
self._active_account = account_id
|
||||
return job
|
||||
return None
|
||||
|
||||
async def _finish_job(self, job: _SendJob) -> None:
|
||||
async with self._state_lock:
|
||||
if self._active_job is job:
|
||||
self._active_job = None
|
||||
self._active_account = None
|
||||
|
||||
queue = self._queues.get(job.account_id)
|
||||
while queue and (queue[0].cancelled or queue[0].future.cancelled()):
|
||||
queue.popleft()
|
||||
if queue:
|
||||
if job.account_id not in self._in_rotation:
|
||||
# Reinsert only after the active operation finishes. New
|
||||
# accounts that arrived while it ran get their turn first.
|
||||
self._rotation.append(job.account_id)
|
||||
self._in_rotation.add(job.account_id)
|
||||
else:
|
||||
self._queues.pop(job.account_id, None)
|
||||
self._remove_from_rotation(job.account_id)
|
||||
self._wake.set()
|
||||
|
||||
async def _run(self) -> None:
|
||||
try:
|
||||
while True:
|
||||
async with self._state_lock:
|
||||
if not self._running:
|
||||
return
|
||||
job = self._pop_next_locked()
|
||||
if job is None:
|
||||
self._wake.clear()
|
||||
|
||||
if job is None:
|
||||
await self._wake.wait()
|
||||
continue
|
||||
|
||||
try:
|
||||
remaining = self._next_allowed_at - asyncio.get_running_loop().time()
|
||||
if remaining > 0:
|
||||
await asyncio.sleep(remaining)
|
||||
|
||||
if job.cancelled or job.future.cancelled():
|
||||
continue
|
||||
|
||||
operation_task = asyncio.create_task(job.operation())
|
||||
async with self._state_lock:
|
||||
job.operation_task = operation_task
|
||||
cancelled_before_start = job.cancelled or job.future.cancelled()
|
||||
if cancelled_before_start:
|
||||
operation_task.cancel()
|
||||
|
||||
try:
|
||||
result = await operation_task
|
||||
except asyncio.CancelledError:
|
||||
if not job.future.done():
|
||||
job.future.cancel()
|
||||
if not self._running:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if not job.future.done():
|
||||
job.future.set_exception(exc)
|
||||
else:
|
||||
if not job.future.done():
|
||||
job.future.set_result(result)
|
||||
finally:
|
||||
self._next_allowed_at = (
|
||||
asyncio.get_running_loop().time() + self.interval_seconds
|
||||
)
|
||||
await self._finish_job(job)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
async def snapshot(self) -> dict[str, Any]:
|
||||
async with self._state_lock:
|
||||
per_account = {
|
||||
account_id: len(queue)
|
||||
for account_id, queue in self._queues.items()
|
||||
if queue
|
||||
}
|
||||
return {
|
||||
"running": bool(self._task and not self._task.done()),
|
||||
"pending_count": sum(per_account.values())
|
||||
+ (1 if self._active_job is not None else 0),
|
||||
"queued_count": sum(per_account.values()),
|
||||
"active_account_id": self._active_account,
|
||||
"active_description": (
|
||||
self._active_job.description if self._active_job else ""
|
||||
),
|
||||
"interval_seconds": self.interval_seconds,
|
||||
"per_account": per_account,
|
||||
}
|
||||
|
||||
async def cancel_account(self, account_id: int) -> int:
|
||||
"""Cancel callers for one stopped account without overlapping traffic.
|
||||
|
||||
Waiting jobs are removed. An already-running operation is allowed to
|
||||
drain inside the dispatcher (notably, Python cannot stop a running
|
||||
to_thread image upload), but its caller is released immediately and no
|
||||
later job can start until that drain finishes.
|
||||
"""
|
||||
account_key = int(account_id or 0)
|
||||
cancelled = 0
|
||||
async with self._state_lock:
|
||||
queue = self._queues.pop(account_key, deque())
|
||||
self._remove_from_rotation(account_key)
|
||||
for job in queue:
|
||||
job.cancelled = True
|
||||
if not job.future.done():
|
||||
job.future.cancel()
|
||||
cancelled += 1
|
||||
|
||||
if self._active_job and self._active_job.account_id == account_key:
|
||||
self._active_job.cancelled = True
|
||||
if not self._active_job.future.done():
|
||||
self._active_job.future.cancel()
|
||||
cancelled += 1
|
||||
self._wake.set()
|
||||
return cancelled
|
||||
|
||||
async def stop(self) -> None:
|
||||
drain_timeout = _env_float("KEFU_SEND_SHUTDOWN_DRAIN_SECONDS", 30.0)
|
||||
async with self._state_lock:
|
||||
self._running = False
|
||||
task = self._task
|
||||
self._task = None
|
||||
active_job = self._active_job
|
||||
active_task = active_job.operation_task if active_job else None
|
||||
futures = [
|
||||
job.future
|
||||
for queue in self._queues.values()
|
||||
for job in queue
|
||||
]
|
||||
for queue in self._queues.values():
|
||||
for job in queue:
|
||||
job.cancelled = True
|
||||
if active_job:
|
||||
active_job.cancelled = True
|
||||
futures.append(active_job.future)
|
||||
self._queues.clear()
|
||||
self._rotation.clear()
|
||||
self._in_rotation.clear()
|
||||
self._wake.set()
|
||||
|
||||
for future in futures:
|
||||
if not future.done():
|
||||
future.cancel()
|
||||
|
||||
if active_task and not active_task.done() and drain_timeout > 0:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.shield(active_task),
|
||||
timeout=drain_timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"Active outbound task did not drain within %.1fs during shutdown (%s)",
|
||||
drain_timeout,
|
||||
active_job.description if active_job else "send",
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
# The dispatcher owns normal operation error reporting/results.
|
||||
pass
|
||||
|
||||
if active_task and not active_task.done():
|
||||
active_task.cancel()
|
||||
if task and task is not asyncio.current_task():
|
||||
if not task.done():
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.shield(task), timeout=1.0)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
task.cancel()
|
||||
if not task.done():
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
async with self._state_lock:
|
||||
self._active_job = None
|
||||
self._active_account = None
|
||||
self._next_allowed_at = 0.0
|
||||
self._wake.clear()
|
||||
|
||||
|
||||
class TrafficController:
|
||||
"""Independent lanes for message writes and lower-priority network work."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.send_queue = GlobalSendQueue(
|
||||
interval_seconds=_env_float(
|
||||
"KEFU_GLOBAL_SEND_INTERVAL_SECONDS",
|
||||
1.0,
|
||||
)
|
||||
)
|
||||
self._background = asyncio.Semaphore(
|
||||
_env_int("KEFU_BACKGROUND_NETWORK_CONCURRENCY", 2)
|
||||
)
|
||||
self._browser = asyncio.Semaphore(
|
||||
_env_int("KEFU_BROWSER_START_CONCURRENCY", 1)
|
||||
)
|
||||
self._media_proxy = asyncio.Semaphore(
|
||||
_env_int("KEFU_MEDIA_PROXY_CONCURRENCY", 4)
|
||||
)
|
||||
self._background_owner: contextvars.ContextVar[tuple[Optional[asyncio.Task], int]] = (
|
||||
contextvars.ContextVar(
|
||||
f"douyin_background_owner_{id(self)}",
|
||||
default=(None, 0),
|
||||
)
|
||||
)
|
||||
self.background_waiting = 0
|
||||
self.background_active = 0
|
||||
self.browser_waiting = 0
|
||||
self.browser_active = 0
|
||||
self.media_proxy_active = 0
|
||||
|
||||
@asynccontextmanager
|
||||
async def background_slot(self, account_id: int = 0, description: str = "request"):
|
||||
current_task = asyncio.current_task()
|
||||
owner_task, depth = self._background_owner.get()
|
||||
if owner_task is current_task and depth > 0:
|
||||
token = self._background_owner.set((current_task, depth + 1))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._background_owner.reset(token)
|
||||
return
|
||||
|
||||
started = asyncio.get_running_loop().time()
|
||||
self.background_waiting += 1
|
||||
try:
|
||||
await self._background.acquire()
|
||||
except BaseException:
|
||||
self.background_waiting -= 1
|
||||
raise
|
||||
self.background_waiting -= 1
|
||||
self.background_active += 1
|
||||
token = self._background_owner.set((current_task, 1))
|
||||
waited = asyncio.get_running_loop().time() - started
|
||||
if waited >= 1.0:
|
||||
logger.info(
|
||||
"Background request dequeued account=%s waited=%.2fs (%s)",
|
||||
account_id,
|
||||
waited,
|
||||
description,
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._background_owner.reset(token)
|
||||
self.background_active -= 1
|
||||
self._background.release()
|
||||
|
||||
@asynccontextmanager
|
||||
async def browser_slot(self, account_id: int = 0, description: str = "browser-login"):
|
||||
started = asyncio.get_running_loop().time()
|
||||
self.browser_waiting += 1
|
||||
try:
|
||||
await self._browser.acquire()
|
||||
except BaseException:
|
||||
self.browser_waiting -= 1
|
||||
raise
|
||||
self.browser_waiting -= 1
|
||||
self.browser_active += 1
|
||||
waited = asyncio.get_running_loop().time() - started
|
||||
if waited >= 1.0:
|
||||
logger.info(
|
||||
"Browser task dequeued account=%s waited=%.2fs (%s)",
|
||||
account_id,
|
||||
waited,
|
||||
description,
|
||||
)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.browser_active -= 1
|
||||
self._browser.release()
|
||||
|
||||
@asynccontextmanager
|
||||
async def media_proxy_slot(self):
|
||||
await self._media_proxy.acquire()
|
||||
self.media_proxy_active += 1
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self.media_proxy_active -= 1
|
||||
self._media_proxy.release()
|
||||
|
||||
async def snapshot(self) -> dict[str, Any]:
|
||||
return {
|
||||
"send": await self.send_queue.snapshot(),
|
||||
"background": {
|
||||
"active": self.background_active,
|
||||
"waiting": self.background_waiting,
|
||||
},
|
||||
"browser": {
|
||||
"active": self.browser_active,
|
||||
"waiting": self.browser_waiting,
|
||||
},
|
||||
"media_proxy": {"active": self.media_proxy_active},
|
||||
}
|
||||
|
||||
async def stop(self) -> None:
|
||||
await self.send_queue.stop()
|
||||
|
||||
|
||||
_CONTROLLERS: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, TrafficController]" = (
|
||||
weakref.WeakKeyDictionary()
|
||||
)
|
||||
|
||||
|
||||
def get_traffic_controller() -> TrafficController:
|
||||
loop = asyncio.get_running_loop()
|
||||
controller = _CONTROLLERS.get(loop)
|
||||
if controller is None:
|
||||
controller = TrafficController()
|
||||
_CONTROLLERS[loop] = controller
|
||||
return controller
|
||||
|
||||
|
||||
async def submit_outbound(
|
||||
account_id: int,
|
||||
operation: Callable[[], Awaitable[T]],
|
||||
description: str = "",
|
||||
) -> T:
|
||||
return await get_traffic_controller().send_queue.submit(
|
||||
account_id,
|
||||
operation,
|
||||
description=description,
|
||||
)
|
||||
|
||||
|
||||
async def shutdown_traffic_controller() -> None:
|
||||
loop = asyncio.get_running_loop()
|
||||
controller = _CONTROLLERS.pop(loop, None)
|
||||
if controller is not None:
|
||||
await controller.stop()
|
||||
@@ -0,0 +1,230 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import threading
|
||||
from typing import Awaitable, Callable, Optional
|
||||
|
||||
from websocket import WebSocketApp
|
||||
|
||||
from utils import system_logger
|
||||
from .protocol import parse_ws_payload
|
||||
from .session import DouyinImSession
|
||||
|
||||
logger = logging.getLogger("douyin_im.ws")
|
||||
|
||||
MessageHandler = Callable[[dict], Awaitable[None]]
|
||||
|
||||
|
||||
class DouyinImWsClient:
|
||||
"""直连 frontier-im WebSocket（websocket-client，与 DouYin_Spider 一致）"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session: DouyinImSession,
|
||||
on_message: MessageHandler,
|
||||
account_id: int | None = None,
|
||||
):
|
||||
self.session = session
|
||||
self.on_message = on_message
|
||||
self.account_id = account_id
|
||||
self._running = False
|
||||
self._task: Optional[asyncio.Task] = None
|
||||
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
||||
self._ws_app: Optional[WebSocketApp] = None
|
||||
self._ws_lock = threading.Lock()
|
||||
|
||||
async def start(self):
|
||||
url = self.session.frontier_ws_url()
|
||||
if not url:
|
||||
logger.warning("No frontier WebSocket URL captured; WS listener disabled")
|
||||
system_logger.record(
|
||||
"实时接收未启用：未获取到 frontier WebSocket 地址",
|
||||
detail="缺少有效的 device_id 或 sessionid，无法建立实时私信通道，将仅依赖 HTTP 轮询。",
|
||||
level="warning",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
return
|
||||
self._running = True
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._task = asyncio.create_task(self._run_loop(url))
|
||||
|
||||
async def stop(self):
|
||||
self._running = False
|
||||
with self._ws_lock:
|
||||
if self._ws_app:
|
||||
try:
|
||||
self._ws_app.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._ws_app = None
|
||||
if self._task:
|
||||
self._task.cancel()
|
||||
try:
|
||||
await self._task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._task = None
|
||||
|
||||
async def _run_ws_thread(self, url: str):
|
||||
"""在独立守护线程中跑 run_forever，直到连接断开/关闭。
|
||||
|
||||
不能用共享默认线程池（run_in_executor(None)/asyncio.to_thread）：
|
||||
WS 长连接会永久占用一个池线程，账号数超过池大小（默认 64）后，
|
||||
所有账号的签名/轮询任务被饿死，表现为“启动几十个账号后全部卡死超时”。
|
||||
"""
|
||||
loop = asyncio.get_running_loop()
|
||||
done = asyncio.Event()
|
||||
error: list[BaseException] = []
|
||||
|
||||
def _runner():
|
||||
try:
|
||||
self._connect_sync(url)
|
||||
except BaseException as e:
|
||||
error.append(e)
|
||||
finally:
|
||||
try:
|
||||
loop.call_soon_threadsafe(done.set)
|
||||
except RuntimeError:
|
||||
pass # 事件循环已关闭
|
||||
|
||||
thread = threading.Thread(
|
||||
target=_runner,
|
||||
name=f"im-ws-{self.account_id or 'na'}",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
try:
|
||||
await done.wait()
|
||||
except asyncio.CancelledError:
|
||||
# stop() 会 close ws_app 使 run_forever 退出，线程随之结束
|
||||
raise
|
||||
if error:
|
||||
raise error[0]
|
||||
|
||||
async def _run_loop(self, url: str):
|
||||
retry = 0
|
||||
while self._running:
|
||||
from .frontier import ensure_frontier_ws
|
||||
from .traffic_control import get_traffic_controller
|
||||
|
||||
# ensure_frontier_ws 可能触发签名/HTTP（阻塞），放线程池避免卡事件循环
|
||||
controller = get_traffic_controller()
|
||||
async with controller.background_slot(self.account_id or 0, "websocket prepare"):
|
||||
await asyncio.to_thread(ensure_frontier_ws, self.session)
|
||||
connect_url = self.session.frontier_ws_url() or url
|
||||
try:
|
||||
logger.info(f"Connecting IM WebSocket: {connect_url[:100]}...")
|
||||
await self._run_ws_thread(connect_url)
|
||||
retry = 0
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.warning(f"IM WebSocket error: {e}")
|
||||
system_logger.record(
|
||||
"实时接收连接异常",
|
||||
detail=f"建立 frontier WebSocket 失败：{e}",
|
||||
level="error",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
if not self._running:
|
||||
break
|
||||
retry += 1
|
||||
# Stable per-account jitter prevents every hosted account from
|
||||
# reconnecting in the same second after a shared network outage.
|
||||
jitter = ((int(self.account_id or 0) * 2654435761) % 5000) / 1000.0
|
||||
wait = min(30.0, 2.0 * retry) + jitter
|
||||
logger.info(f"IM WebSocket reconnect in {wait:.1f}s...")
|
||||
system_logger.record(
|
||||
f"实时接收断开，{wait:.1f}s 后重连",
|
||||
detail="frontier WebSocket 连接已断开，正在自动重连。",
|
||||
level="warning",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
await asyncio.sleep(wait)
|
||||
|
||||
def _connect_sync(self, url: str):
|
||||
if not self._loop:
|
||||
return
|
||||
|
||||
def on_open(_ws):
|
||||
logger.info("IM WebSocket connected")
|
||||
system_logger.record(
|
||||
"实时接收通道已连接",
|
||||
detail="frontier WebSocket 已建立，可实时接收私信。",
|
||||
level="success",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
def on_message(_ws, message):
|
||||
asyncio.run_coroutine_threadsafe(self._dispatch(message), self._loop)
|
||||
|
||||
def on_error(_ws, error):
|
||||
if self._running:
|
||||
logger.warning(f"IM WebSocket error: {error}")
|
||||
system_logger.record(
|
||||
"实时接收通道报错",
|
||||
detail=f"{error}",
|
||||
level="error",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
def on_close(_ws, code, msg):
|
||||
logger.info(f"IM WebSocket closed: code={code}, msg={msg}")
|
||||
if self._running:
|
||||
system_logger.record(
|
||||
"实时接收通道关闭",
|
||||
detail=f"code={code}, msg={msg}",
|
||||
level="warning",
|
||||
category="ws",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
|
||||
headers = {
|
||||
"User-Agent": self.session.user_agent,
|
||||
"Pragma": "no-cache",
|
||||
"Cache-Control": "no-cache",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Sec-WebSocket-Protocol": "binary, base64, pbbp2",
|
||||
"Sec-WebSocket-Extensions": "permessage-deflate; client_max_window_bits",
|
||||
}
|
||||
ws_app = WebSocketApp(
|
||||
url,
|
||||
header=headers,
|
||||
cookie=self.session.cookie_header(),
|
||||
on_open=on_open,
|
||||
on_message=on_message,
|
||||
on_error=on_error,
|
||||
on_close=on_close,
|
||||
)
|
||||
with self._ws_lock:
|
||||
self._ws_app = ws_app
|
||||
try:
|
||||
ws_app.run_forever(origin="https://www.douyin.com", ping_interval=20, ping_timeout=10)
|
||||
finally:
|
||||
with self._ws_lock:
|
||||
if self._ws_app is ws_app:
|
||||
self._ws_app = None
|
||||
|
||||
async def _dispatch(self, raw):
|
||||
if isinstance(raw, str):
|
||||
payload = raw.encode("utf-8", errors="ignore")
|
||||
else:
|
||||
payload = raw
|
||||
items = parse_ws_payload(payload)
|
||||
for item in items:
|
||||
try:
|
||||
await self.on_message(item)
|
||||
except Exception as e:
|
||||
logger.debug(f"WS message handler error: {e}")
|
||||
system_logger.record(
|
||||
"实时消息处理失败",
|
||||
detail=f"处理收到的私信时出错：{e}",
|
||||
level="error",
|
||||
category="recv",
|
||||
account_id=self.account_id,
|
||||
)
|
||||
Reference in New Issue
Block a user