Files
dy/backend/tests/test_im_receive_path.py
T
2026-08-27 18:32:03 +08:00

712 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""接收私信链路的回归测试。
覆盖三个曾让「托管中收不到抖音下发的私信」的缺陷:
1. frontier 长连接地址用了账号 UID 而不是设备号,握手成功却订阅错地址;
2. 浏览器本次登录抓到的真实 frontier 地址被 DB 里的旧地址挤掉;
3. PushFrame 负载是 gzip / payloadType 不是 'pb' 时整帧被丢弃。
另外覆盖会话列表接口被抖音拒绝时不能再伪装成「收件箱为空」。
"""
from __future__ import annotations
import gzip
import json
import os
import sys
import unittest
from datetime import datetime
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ.setdefault("KEFU_DB_TYPE", "sqlite")
os.environ.setdefault("KEFU_DATABASE_URL", "")
os.environ.setdefault("KEFU_DB_PATH", str(BACKEND_DIR / "kefu.db"))
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im import frontier as frontier_module
from rpa_engine.douyin_im.auth import DouyinAuth
from rpa_engine.douyin_im.frontier import ensure_frontier_ws
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
from rpa_engine.douyin_im.protocol import parse_ws_payload
from rpa_engine.douyin_im.session import DouyinImSession
from rpa_engine.douyin_im.static import Live_pb2, Response_pb2
from rpa_engine.playwright_worker import DouyinWorker
def _build_push_frame(
*,
conversation_id: str,
sender: int,
content: str,
message_type: int = 7,
server_message_id: int = 7665317099296081465,
encoding: str = "",
payload_type: str = "pb",
) -> bytes:
response = Response_pb2.Response()
notify = response.body.new_message_notify
notify.conversation_id = conversation_id
message = notify.message
message.conversation_id = conversation_id
message.conversation_type = 1
message.server_message_id = server_message_id
message.message_type = message_type
message.sender = sender
message.content = content
body = response.SerializeToString()
if encoding == "gzip":
body = gzip.compress(body)
frame = Live_pb2.PushFrame()
frame.seqId = 1
frame.service = 6
frame.method = 2
frame.payloadEncoding = encoding
frame.payloadType = payload_type
frame.payload = body
return frame.SerializeToString()
class FrontierAddressTests(unittest.TestCase):
"""frontier 按 device_id 寻址,不能用账号 UID 顶替。"""
def _session(self) -> DouyinImSession:
return DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
# query/user 返回的 id:本浏览器的设备注册号
device_id="7678285795559818786",
web_id="7678286623234475535",
my_uid=2609567359568155,
uid_verified=True,
)
def test_built_url_uses_device_id_not_account_uid(self):
session = self._session()
url = ensure_frontier_ws(session)
self.assertIsNotNone(url)
self.assertIn("device_id=7678285795559818786", url)
self.assertNotIn("device_id=2609567359568155", url)
def test_missing_device_id_falls_back_to_query_user_lookup(self):
session = self._session()
session.device_id = ""
session.web_id = ""
with patch.object(
frontier_module,
"fetch_device_id",
return_value="7678285795559818786",
) as fetch:
url = ensure_frontier_ws(session)
fetch.assert_called_once()
self.assertIn("device_id=7678285795559818786", url)
def test_proto_auth_keeps_device_id_for_verified_uid(self):
session = self._session()
auth = DouyinAuth.from_im_session(session)
self.assertEqual(auth.device_id, "7678285795559818786")
class CapturedFrontierUrlTests(unittest.IsolatedAsyncioTestCase):
"""浏览器本次抓到的真实地址必须压过 DB 里的旧地址。"""
async def test_fresh_browser_ws_url_wins_over_cached_url(self):
cached = (
"wss://frontier-im.douyin.com/ws/v2?aid=6383&device_platform=douyin_pc"
"&fpid=9&device_id=7678285795559818786&token=stale&access_key=stale"
)
captured = (
"wss://frontier31-normal.zijieapi.com/ws/v2?aid=6383&fpid=9"
"&device_id=7678285795559818786&access_key=realkey&token=realtoken"
)
saved = DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
ws_urls=[cached],
device_id="7678285795559818786",
my_uid=2609567359568155,
)
row = SimpleNamespace(
im_session_data=json.dumps(saved.to_dict()),
cookie_updated_at=datetime.utcnow(),
uid=None,
profile_updated_at=None,
)
result = MagicMock()
result.one_or_none.return_value = row
db = SimpleNamespace(execute=AsyncMock(return_value=result), close=AsyncMock())
worker = DouyinWorker(account_id=400, login_mode="browser")
worker.get_db = AsyncMock(return_value=db)
worker._load_raw_user_agent = AsyncMock(return_value="test-agent")
session = await worker._build_im_session_from_storage(
{"cookies": [{"name": "sessionid", "value": "6313fec013ec0000000000000000abcd"}]},
{"ws_urls": [captured]},
)
self.assertEqual(session.ws_urls[0], captured)
self.assertEqual(ensure_frontier_ws(session), captured)
class PushFramePayloadTests(unittest.TestCase):
"""PushFrame 负载的编码/类型不能再决定整帧被不被丢弃。"""
def test_plain_protobuf_frame_is_parsed(self):
raw = _build_push_frame(
conversation_id="0:1:869032150442612:2609567359568155",
sender=869032150442612,
content=json.dumps({"text": "你好", "aweType": 700}, ensure_ascii=False),
)
messages = parse_ws_payload(raw)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]["sender_uid"], "869032150442612")
self.assertEqual(
messages[0]["conversation_id"], "0:1:869032150442612:2609567359568155"
)
def test_gzip_encoded_frame_is_parsed(self):
raw = _build_push_frame(
conversation_id="0:1:869032150442612:2609567359568155",
sender=869032150442612,
content=json.dumps({"text": "在吗", "aweType": 700}, ensure_ascii=False),
encoding="gzip",
)
messages = parse_ws_payload(raw)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]["sender_uid"], "869032150442612")
def test_non_pb_payload_type_is_still_parsed(self):
# 现网 frontier 帧会带 payloadType='text/json';只认 'pb' 会整帧丢弃。
raw = _build_push_frame(
conversation_id="0:1:869032150442612:2609567359568155",
sender=869032150442612,
content=json.dumps({"text": "在吗", "aweType": 700}, ensure_ascii=False),
payload_type="text/json",
)
messages = parse_ws_payload(raw)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]["sender_uid"], "869032150442612")
def test_empty_payload_control_frame_yields_no_message(self):
frame = Live_pb2.PushFrame()
frame.service = 6
frame.method = 2
frame.payloadEncoding = "utf-8"
frame.payloadType = "text/json"
self.assertEqual(parse_ws_payload(frame.SerializeToString()), [])
class InboxProtobufTests(unittest.TestCase):
"""imapi 只认 protobuf:解析真实响应形状,而不是 JSON。"""
@staticmethod
def _response(*, cmd=200, status=0, message="OK", messages=()):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_msg, _pb_str
entries = b""
for m in messages:
entries += _pb_msg(
1,
_pb_str(1, m["conversation_id"])
+ _pb_int(3, m["server_message_id"])
+ _pb_int(5, m.get("conversation_short_id", 0))
+ _pb_int(6, m.get("message_type", 7))
+ _pb_int(7, m["sender"])
+ _pb_str(8, m.get("content", "")),
)
return (
_pb_int(1, cmd)
+ _pb_int(3, status)
+ _pb_str(4, message)
+ _pb_msg(6, _pb_msg(cmd, entries))
)
def test_status_is_read_from_the_protobuf_envelope(self):
from rpa_engine.douyin_im.http_client import _pb_response_status
ok = self._response()
self.assertEqual(_pb_response_status(ok), (0, "OK"))
rejected = self._response(status=1, message="unexepcted session length")
self.assertEqual(
_pb_response_status(rejected),
(1, "unexepcted session length"),
)
def test_message_bodies_are_extracted_from_the_inbox_response(self):
from rpa_engine.douyin_im.http_client import _pb_parse_inbox_messages
raw = self._response(
messages=[
{
"conversation_id": "0:1:2609567359568155:869032150442612",
"server_message_id": 7678140298052355621,
"conversation_short_id": 7654765893796266545,
"message_type": 7,
"sender": 869032150442612,
"content": '{"text":"你好"}',
}
]
)
parsed = _pb_parse_inbox_messages(raw, 200)
self.assertEqual(len(parsed), 1)
self.assertEqual(
parsed[0]["conversation_id"],
"0:1:2609567359568155:869032150442612",
)
self.assertEqual(parsed[0]["server_message_id"], "7678140298052355621")
self.assertEqual(parsed[0]["sender"], "869032150442612")
self.assertIn("你好", parsed[0]["content"])
def test_unrelated_protobuf_is_not_mistaken_for_a_message(self):
from rpa_engine.douyin_im.http_client import (
_pb_int, _pb_msg, _pb_parse_inbox_messages, _pb_str,
)
# 一段带字符串字段 1 但不是 conversation_id 的子消息
noise = _pb_msg(6, _pb_msg(200, _pb_msg(1, _pb_str(1, "not-a-conv") + _pb_int(3, 5))))
self.assertEqual(_pb_parse_inbox_messages(noise, 200), [])
def test_empty_inbox_yields_no_messages(self):
from rpa_engine.douyin_im.http_client import _pb_parse_inbox_messages
self.assertEqual(_pb_parse_inbox_messages(self._response(), 200), [])
class InboxCursorAndListTests(unittest.IsolatedAsyncioTestCase):
"""轮询用小窗口,用户点开列表用全量——同一个 cmd,只是游标不同。"""
def _client(self) -> DouyinImHttpClient:
session = DouyinImSession(
cookies={"sessionid": "s", "x_tt_token": "00" + "a" * 353},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
return DouyinImHttpClient(session, account_id=405)
@staticmethod
def _cursor_from_payload(payload: bytes) -> int:
from rpa_engine.douyin_im.http_client import _pb_parse_fields
for fn, wt, val in _pb_parse_fields(payload):
if fn != 8 or wt != 2:
continue
for bfn, bwt, bval in _pb_parse_fields(val):
if bfn != 200 or bwt != 2:
continue
for cfn, cwt, cval in _pb_parse_fields(bval):
if cfn == 1 and cwt == 0:
return int(cval)
return -1
async def _capture_cursor(self, **kwargs) -> int:
from rpa_engine.douyin_im.http_client import _pb_int, _pb_str
client = self._client()
captured: dict = {}
async def fake_post(url, auth, payload, **_kw):
captured["payload"] = payload
return SimpleNamespace(
content=_pb_int(1, 200) + _pb_int(3, 0) + _pb_str(4, "OK"),
raise_for_status=lambda: None,
)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(**kwargs)
return self._cursor_from_payload(captured["payload"])
async def test_poll_window_sends_a_recent_microsecond_cursor(self):
import time as _time
cursor = await self._capture_cursor(lookback_seconds=1800)
now_us = int(_time.time() * 1_000_000)
self.assertGreater(cursor, 0)
# 游标应落在「大约半小时前」,允许几秒误差
self.assertLess(now_us - cursor, int(1810 * 1_000_000))
self.assertGreater(now_us - cursor, int(1790 * 1_000_000))
async def test_zero_lookback_means_no_cursor_not_now(self):
# lookback=0 若被算成 now,就只要「比此刻更新」的消息,永远是空列表。
self.assertEqual(await self._capture_cursor(lookback_seconds=0), 0)
@staticmethod
def _page(*, entries=(), next_cursor=0, has_more=False, cmd=200):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_msg, _pb_str
inner = b""
for short_id, conv_id in entries:
inner += _pb_msg(6, _pb_int(1, short_id) + _pb_str(4, conv_id))
inner += _pb_int(2, next_cursor) + _pb_int(3, 1 if has_more else 0)
return (
_pb_int(1, cmd)
+ _pb_int(3, 0)
+ _pb_str(4, "OK")
+ _pb_msg(6, _pb_msg(cmd, inner))
)
async def test_paging_follows_the_cursor_and_dedupes_conversations(self):
client = self._client()
pages = [
self._page(
entries=[(1, "0:1:10001:20001"), (2, "0:1:10001:20002")],
next_cursor=111,
has_more=True,
),
self._page(
# 第二页重复一个、新增一个
entries=[(2, "0:1:10001:20002"), (3, "0:1:10001:20003")],
next_cursor=222,
has_more=True,
),
]
cursors: list[int] = []
async def fake_post(url, auth, payload, **_kw):
cursors.append(self._cursor_from_payload(payload))
return SimpleNamespace(
content=pages[len(cursors) - 1], raise_for_status=lambda: None
)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(lookback_seconds=0, max_pages=2)
self.assertEqual(cursors, [0, 111])
self.assertEqual(
[c["conversation_id"] for c in client._last_inbox_conversations],
["0:1:10001:20001", "0:1:10001:20002", "0:1:10001:20003"],
)
# 预算用完但抖音还说 has_more:必须承认列表不完整
self.assertTrue(client.inbox_truncated)
async def test_last_page_is_not_reported_as_truncated(self):
client = self._client()
page = self._page(entries=[(1, "0:1:10001:20001")], has_more=False)
async def fake_post(url, auth, payload, **_kw):
return SimpleNamespace(content=page, raise_for_status=lambda: None)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(lookback_seconds=0, max_pages=5)
self.assertFalse(client.inbox_truncated)
async def test_a_stalled_cursor_stops_paging(self):
client = self._client()
# 抖音回 has_more=1 但游标不前进:不能无限翻同一页
page = self._page(
entries=[(1, "0:1:10001:20001")], next_cursor=0, has_more=True
)
calls = {"n": 0}
async def fake_post(url, auth, payload, **_kw):
calls["n"] += 1
return SimpleNamespace(content=page, raise_for_status=lambda: None)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(lookback_seconds=0, max_pages=10)
self.assertEqual(calls["n"], 1)
async def test_conversations_without_recent_messages_still_listed(self):
client = self._client()
client.fetch_inbox_messages = AsyncMock(return_value=[])
client._last_inbox_conversations = [
{"conversation_id": "0:1:10001:20001", "conversation_short_id": "555"},
{"conversation_id": "0:1:10001:20002", "conversation_short_id": "666"},
]
rows = await client.get_conversations(
enrich_profiles=False, lookback_seconds=0
)
self.assertEqual(
{r["conversation_id"] for r in rows},
{"0:1:10001:20001", "0:1:10001:20002"},
)
self.assertEqual(
client.session.conv_meta["0:1:10001:20002"]["conversation_short_id"],
"666",
)
async def test_control_frames_never_become_a_conversation_preview(self):
client = self._client()
client.fetch_inbox_messages = AsyncMock(
return_value=[
{
"conversation_id": "0:1:10001:20001",
"server_message_id": "100",
"message_type": 7,
"sender": "20001",
"content": '{"text":"真实消息"}',
},
{
"conversation_id": "0:1:10001:20001",
"server_message_id": "200",
"message_type": 50001,
"sender": "20001",
"content": '{"command_type":6,"conversation_id":"0:1:10001:20001"}',
},
]
)
rows = await client.get_conversations(enrich_profiles=False)
# 控制帧 server_message_id 更大,但不能顶掉真实消息成为预览,
# 否则 _handle_incoming 会拿它去匹配自动回复。
self.assertEqual(len(rows), 1)
self.assertIn("真实消息", rows[0]["content"])
self.assertEqual(rows[0]["server_message_id"], "100")
class ReadRequestTokenTests(unittest.TestCase):
"""读接口的 Request.token 必须是 x_tt_token。
带 auth.ticket 时抖音照样回 status_code=0 "OK",但把调用方当匿名用户,
正文恒为空——和「收件箱没有消息」完全无法区分,是最难发现的那类故障。
实测同一请求只换 tokenauth.ticket 73 字节 0 条,x_tt_token 113KB 47 条。
"""
def _auth(self):
session = DouyinImSession(
cookies={
"sessionid": "6313fec013ec0000000000000000abcd",
"x_tt_token": "00" + "a" * 353,
},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
return DouyinAuth.from_im_session(session)
def test_read_request_uses_x_tt_token(self):
from rpa_engine.douyin_im.proto_builder import ProtoBuilder
auth = self._auth()
request = ProtoBuilder.build_read_request(auth, 200)
self.assertEqual(request.token, "00" + "a" * 353)
self.assertNotEqual(request.token, auth.ticket)
def test_read_request_keeps_ticket_when_cookie_missing(self):
from rpa_engine.douyin_im.proto_builder import ProtoBuilder
session = DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
auth = DouyinAuth.from_im_session(session)
request = ProtoBuilder.build_read_request(auth, 200)
self.assertEqual(request.token, auth.ticket or "")
def test_send_request_is_left_on_the_normal_envelope(self):
from rpa_engine.douyin_im.proto_builder import ProtoBuilder
# 发送接口另有 bd-ticket-guard 签名且线上可用,不能顺手改掉它的 token。
auth = self._auth()
request = ProtoBuilder.build_normal_request(auth, 100)
self.assertEqual(request.token, auth.ticket or "")
class AuthoritativeUidTests(unittest.TestCase):
"""imapi 响应字段 13 是抖音认定的本账号 IM uid。"""
def test_response_uid_corrects_a_wrong_my_uid(self):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_str
session = DouyinImSession(
cookies={"sessionid": "s"},
my_uid=7678285795559818786, # 误把 frontier 设备号当成了 IM uid
)
client = DouyinImHttpClient(session, account_id=404)
content = (
_pb_int(1, 200)
+ _pb_int(3, 0)
+ _pb_str(4, "OK")
+ _pb_int(13, 2609567359568155)
)
client._adopt_authoritative_uid(content)
self.assertEqual(session.my_uid, 2609567359568155)
self.assertTrue(session.uid_verified)
def test_matching_uid_is_left_alone(self):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_str
session = DouyinImSession(cookies={"sessionid": "s"}, my_uid=2609567359568155)
client = DouyinImHttpClient(session, account_id=404)
content = _pb_int(1, 200) + _pb_int(3, 0) + _pb_str(4, "OK") + _pb_int(
13, 2609567359568155
)
client._adopt_authoritative_uid(content)
self.assertEqual(session.my_uid, 2609567359568155)
self.assertFalse(session.uid_verified)
class ConversationListRejectionTests(unittest.IsolatedAsyncioTestCase):
"""接口被拒不能再伪装成「收件箱为空」。"""
def _client(self) -> DouyinImHttpClient:
session = DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
return DouyinImHttpClient(session, account_id=401)
async def test_rejected_protobuf_response_is_reported(self):
client = self._client()
raw = InboxProtobufTests._response(
status=1, message="unexepcted session length"
)
post = AsyncMock(
return_value=SimpleNamespace(
content=raw, raise_for_status=lambda: None
)
)
with (
patch.object(client, "_post_protobuf", post),
patch(
"rpa_engine.douyin_im.auth.DouyinAuth.from_im_session",
return_value=SimpleNamespace(source_ip=""),
),
patch(
"rpa_engine.douyin_im.proto_builder.ProtoBuilder.build_normal_request",
return_value=SimpleNamespace(SerializeToString=lambda: b""),
),
patch.object(client, "_report_conversation_list_rejected") as report,
):
self.assertEqual(await client.fetch_inbox_messages(), [])
post.assert_awaited_once()
report.assert_called_once_with("unexepcted session length")
async def test_rejection_marks_the_endpoint_unsupported(self):
client = self._client()
client._report_conversation_list_rejected("unexepcted session length")
self.assertTrue(client.conversation_list_unsupported)
self.assertIn("unexepcted session length", client.last_error)
async def test_empty_but_successful_inbox_is_not_reported_as_failure(self):
client = self._client()
client.fetch_inbox_messages = AsyncMock(return_value=[])
with patch.object(
client, "_report_conversation_list_rejected"
) as report:
self.assertEqual(
await client.get_conversations(enrich_profiles=False), []
)
report.assert_not_called()
self.assertEqual(client.last_error, "")
self.assertFalse(client.conversation_list_unsupported)
class LoggedOutDetectionTests(unittest.IsolatedAsyncioTestCase):
"""抖音回「用户未登录」时必须明确报出来,不能当成资料接口抖动。"""
def test_status_code_8_is_reported_as_logged_out(self):
from rpa_engine import account_profile as ap
auth = SimpleNamespace(cookie={}, msToken="t", get_uid=lambda: "938334054809296")
payloads = [
{"status_code": 0, "user_uid": "938334054809296"},
{"status_code": 8, "status_msg": "用户未登录", "user": None},
{"status_code": 8, "status_msg": "用户未登录", "user": None},
]
responses = [SimpleNamespace(json=lambda v=v: v) for v in payloads]
with (
patch.object(ap, "_build_auth", return_value=(auth, "ua")),
patch.object(ap.requests, "get", side_effect=responses),
patch.object(ap, "generate_a_bogus", return_value="a-bogus"),
patch.object(ap, "generate_webid", return_value="web-id"),
patch.object(ap, "_requests_proxies", return_value=None),
):
detail = ap.fetch_douyin_profile_detail_sync("cookie-json", "ua")
self.assertTrue(detail["logged_out"])
self.assertFalse(detail["fetched"])
self.assertIn("用户未登录", detail["message"])
async def test_hosting_reports_logged_out_once(self):
worker = DouyinWorker(account_id=403, login_mode="im_direct")
with patch(
"rpa_engine.playwright_worker.system_logger.record"
) as record:
await worker._report_douyin_logged_out("抖音返回「用户未登录」")
await worker._report_douyin_logged_out("抖音返回「用户未登录」")
record.assert_called_once()
self.assertEqual(record.call_args.kwargs["level"], "error")
class ReconciliationBackoffTests(unittest.IsolatedAsyncioTestCase):
"""被抖音拒绝过的接口不能每 120 秒再白打一次。"""
def _service(self):
from rpa_engine.douyin_im.service import DouyinImService
return DouyinImService(
session=DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
device_id="7678285795559818786",
my_uid=2609567359568155,
),
match_reply=AsyncMock(return_value=[]),
log_fn=AsyncMock(),
account_id=402,
)
async def test_second_poll_skips_a_rejected_endpoint(self):
service = self._service()
client = MagicMock()
client.get_conversations = AsyncMock(return_value=[])
client.conversation_list_unsupported = True
client.__aenter__ = AsyncMock(return_value=client)
client.__aexit__ = AsyncMock(return_value=False)
with patch(
"rpa_engine.douyin_im.service.DouyinImHttpClient",
return_value=client,
):
self.assertEqual(await service._poll_conversations(), [])
self.assertTrue(service._conversation_list_unsupported)
# 第二轮完全不再构造 HTTP 客户端 / 发请求
self.assertEqual(await service._poll_conversations(), [])
client.get_conversations.assert_awaited_once()
if __name__ == "__main__":
unittest.main()