ZYT/dy
Template
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

更新

Browse Source
This commit is contained in:
Your Name
2026-08-27 18:32:03 +08:00
parent 4ac6990efe
commit 1f3addcf79
50 changed files with 9145 additions and 1760 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.bak/kefu.db.before_uid_fix_20260827_180651
BIN
View File
Binary file not shown.
backend/.bak/kefu1.db.before_uid_fix_20260827_180652
BIN
View File
Binary file not shown.
backend/debug_qr/frame0_IMG_16_58.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.7 KiB

backend/debug_qr/frame0_IMG_184_1013.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 109 KiB

backend/debug_qr/frame0_IMG_184_412.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 104 KiB

backend/debug_qr/frame0_IMG_184_717.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 122 KiB

backend/debug_qr/frame0_IMG_539_108.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 112 KiB

backend/debug_qr/frame0_IMG_539_412.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 108 KiB

backend/debug_qr/frame0_IMG_539_708.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 101 KiB

backend/debug_qr/frame0_IMG_894_108.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 13 KiB

backend/debug_qr/frame0_IMG_894_412.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 52 KiB

backend/debug_qr/frame0_IMG_894_708.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 106 KiB

backend/debug_qr/page_after_login_click.html
+1556
View File
File diff suppressed because one or more lines are too long
backend/debug_qr/page_initial.html
+1546
View File
File diff suppressed because one or more lines are too long
backend/debug_qr/report.json
+855
View File
@@ -0,0 +1,855 @@
{
"frames": [
{
"index": 0,
"url": "https://www.douyin.com/",
"qrcodes": [
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "AXNt5Hoz",
"outer": "<img class=\"AXNt5Hoz\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "Wzqh8kMJ",
"outer": "<img class=\"Wzqh8kMJ\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "MaDupF4a",
"outer": "<img class=\"MaDupF4a\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "RSP3dVtx",
"outer": "<img class=\"RSP3dVtx\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "M3dFOzE4",
"outer": "<img class=\"M3dFOzE4\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "sB_GUV4n",
"outer": "<img class=\"sB_GUV4n\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "BD9BarA8",
"outer": "<img class=\"BD9BarA8\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "jMPyhzfG",
"outer": "<img class=\"jMPyhzfG\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "efPPcdLl",
"outer": "<img class=\"efPPcdLl\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "RnpNMA46",
"outer": "<img class=\"RnpNMA46\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "FqRV7w1P",
"outer": "<img class=\"FqRV7w1P\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "_Whzlv1b",
"outer": "<img class=\"_Whzlv1b\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "j1TwxzPC",
"outer": "<img class=\"j1TwxzPC\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "zYgniQaG",
"outer": "<img class=\"zYgniQaG\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "uJKU1tdN",
"outer": "<img class=\"uJKU1tdN\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "HkY6seUs",
"outer": "<img class=\"HkY6seUs\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "khDBMSjy",
"outer": "<img class=\"khDBMSjy\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "GkION2OQ",
"outer": "<img class=\"GkION2OQ\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "q2uupcgz",
"outer": "<img class=\"q2uupcgz\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 16,
"y": 58,
"width": 128,
"height": 40
},
"src_prefix": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "n6fjbOcQ",
"outer": "<img class=\"n6fjbOcQ\" src=\"https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png\">",
"screenshot": "frame0_IMG_16_58.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 539.328125,
"y": 108,
"width": 339.328125,
"height": 190.859375
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_ea6e0dc453683550de835cc2ded929b0~tplv-dy-resize-walign-adapt-aq:540:q7",
"alt": "法国搞笑三人组新作，结尾太好笑了 法国喜剧#电影长尾豹马修 #喜剧电影解说",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_ea6e0dc453683550de835cc2ded929b0~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&amp;from=1189464143&amp;lk3s=46e5c84f&amp;s=PackSourceEnum_DOUYIN_WEB_NEW_PAGE&amp;sc=cover&amp;se=false&amp;x-expires=1789030800&amp;x-signatu",
"screenshot": "frame0_IMG_539_108.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 894.65625,
"y": 108,
"width": 339.34375,
"height": 190.875
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_dfc1793c444e4e9731b08c24d409cfff~tplv-dy-resize-walign-adapt-aq:540:q7",
"alt": "你我怎么两清……#戴上耳机 #甲乙丙丁 #李佳薇 #音乐分享",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_dfc1793c444e4e9731b08c24d409cfff~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&amp;from=1189464143&amp;lk3s=46e5c84f&amp;s=PackSourceEnum_DOUYIN_WEB_NEW_PAGE&amp;sc=cover&amp;se=false&amp;x-expires=1789030800&amp;x-signatu",
"screenshot": "frame0_IMG_894_108.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 184,
"y": 412.875,
"width": 339.328125,
"height": 190.859375
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_032fb5c8e6ddd4703e115f1139fe01cf~tplv-dy-resize-walign-adapt-aq:540:q7",
"alt": "被外卖大哥不小心蹭了车，但没想到他的手机铃声竟然是我的歌…但也正因如此我才有幸走进了一个父与子的故事里#人间观察计划#外卖小哥 #看见100种生活#日常分享 #雪下的时候",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_032fb5c8e6ddd4703e115f1139fe01cf~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&amp;from=1189464143&amp;lk3s=46e5c84f&amp;s=PackSourceEnum_DOUYIN_WEB_NEW_PAGE&amp;sc=cover&amp;se=false&amp;x-expires=1789030800&amp;x-signatu",
"screenshot": "frame0_IMG_184_412.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 539.328125,
"y": 412.875,
"width": 339.328125,
"height": 190.859375
},
"src_prefix": "https://p9-pc-sign.douyinpic.com/tos-cn-i-dy/ef60f35dda7a4b1396df4b5b5abfb632~tplv-dy-vqe2-sr-opt1:640:480:q80.webp?from",
"alt": "一口气听完当年火遍全网的说唱，谁的DNA动了#中文说唱 #马思唯 #kkluv #创作者扶持计划 #抖音精选",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p9-pc-sign.douyinpic.com/tos-cn-i-dy/ef60f35dda7a4b1396df4b5b5abfb632~tplv-dy-vqe2-sr-opt1:640:480:q80.webp?from=1189464143&amp;lk3s=46e5c84f&amp;x-expires=1788685200&amp;x-signature=YYrBL5uYHi0RBKP8Do8J0iMGHms%3D\" alt=\"一口气听完当年火遍全网的说唱，谁的DNA动了#中文说唱 #马思唯 #kkluv #创作者扶持计划 #抖音精选\" class=",
"screenshot": "frame0_IMG_539_412.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 894.65625,
"y": 412.875,
"width": 339.34375,
"height": 190.875
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_d4d6aace7531ed9cbd364c4313a21352~tplv-dy-resize-walign-adapt-aq:540:q7",
"alt": "当你穿进老钱班33#老钱班 #侯绿萝#olly懂你漂亮做自己 #olly女性复合维生素",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_d4d6aace7531ed9cbd364c4313a21352~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&amp;from=1189464143&amp;lk3s=46e5c84f&amp;s=PackSourceEnum_DOUYIN_WEB_NEW_PAGE&amp;sc=cover&amp;se=false&amp;x-expires=1789030800&amp;x-signatu",
"screenshot": "frame0_IMG_894_412.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 184,
"y": 717.75,
"width": 339.328125,
"height": 190.859375
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_49bbd3ffa13175f85795123107c70169~tplv-dy-resize-walign-adapt-aq:540:q7",
"alt": "轮回神话5 女儿试炼误入绝境，获S级血统轰动全宇宙！探秘禁忌陵宫，竟发现横扫万界的创世神正是自家咸鱼老爸！#原创动画 #二次元 #剧情 #反转 #扮猪吃虎名场面",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_49bbd3ffa13175f85795123107c70169~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&amp;from=1189464143&amp;lk3s=46e5c84f&amp;s=PackSourceEnum_DOUYIN_WEB_NEW_PAGE&amp;sc=cover&amp;se=false&amp;x-expires=1789030800&amp;x-signatu",
"screenshot": "frame0_IMG_184_717.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 539.328125,
"y": 708.75,
"width": 339.328125,
"height": 190.859375
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/70d71f59a2bab8ab8a238f9276777e7a~tplv-dy-vqe2-sr-opt1:640:480:q80.webp?fr",
"alt": "深度解析《大明王朝1566》 明成祖朱棣定下的锦衣卫选拔标准，一般人还真达不到#大明王朝1566 #历史",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/70d71f59a2bab8ab8a238f9276777e7a~tplv-dy-vqe2-sr-opt1:640:480:q80.webp?from=1189464143&amp;lk3s=46e5c84f&amp;x-expires=1788685200&amp;x-signature=vsIbNW%2BtgYYmFILlWMn38utpuzg%3D\" alt=\"深度解析《大明王朝1566》 明成祖朱棣定下的锦衣卫选拔标准，一般人还真达不到#大明王朝1566 #历史\" clas",
"screenshot": "frame0_IMG_539_708.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 894.65625,
"y": 708.75,
"width": 339.34375,
"height": 190.875
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_61db88f71bdba90e436a783bae92863f~tplv-dy-resize-walign-adapt-aq:540:q7",
"alt": "当大哥不接暗号，鼠鼠带着九格强行认大哥会发生什么呢？ #三角洲行动 #三角洲得吃就行挑战 #鼠鼠我呀得吃了 #三角洲最仁义玩家 #洲人洲事",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_61db88f71bdba90e436a783bae92863f~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&amp;from=1189464143&amp;lk3s=46e5c84f&amp;s=PackSourceEnum_DOUYIN_WEB_NEW_PAGE&amp;sc=cover&amp;se=false&amp;x-expires=1789030800&amp;x-signatu",
"screenshot": "frame0_IMG_894_708.png"
},
{
"selector": "img",
"tag": "IMG",
"visible": true,
"box": {
"x": 184,
"y": 1013.625,
"width": 339.328125,
"height": 190.859375
},
"src_prefix": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_17d53951c4fb7c73dbfa5acddfde35a3~tplv-dy-resize-walign-adapt-aq:540:q7",
"alt": "本想应付体验大学生活的表弟，不料竟意外发现表弟的万能用处 #搞笑 #动漫 #轻漫计划 #充能计划",
"class": "XTdkxrLI discover-video-card-img",
"outer": "<img src=\"https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_17d53951c4fb7c73dbfa5acddfde35a3~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&amp;from=1189464143&amp;lk3s=46e5c84f&amp;s=PackSourceEnum_DOUYIN_WEB_NEW_PAGE&amp;sc=cover&amp;se=false&amp;x-expires=1789030800&amp;x-signatu",
"screenshot": "frame0_IMG_184_1013.png"
}
],
"panels": []
},
{
"index": 1,
"url": "https://lf-rc1.yhgfb-cn-static.com/obj/rc-verifycenter/rmc-nocaptcha/1.0.0.50/index.html",
"qrcodes": [],
"panels": []
}
],
"candidates": [
{
"tag": "IMG",
"index": 0,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "AXNt5Hoz",
"parentText": ""
},
{
"tag": "IMG",
"index": 1,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "Wzqh8kMJ",
"parentText": ""
},
{
"tag": "IMG",
"index": 2,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "MaDupF4a",
"parentText": ""
},
{
"tag": "IMG",
"index": 3,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "RSP3dVtx",
"parentText": ""
},
{
"tag": "IMG",
"index": 4,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "M3dFOzE4",
"parentText": ""
},
{
"tag": "IMG",
"index": 5,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "sB_GUV4n",
"parentText": ""
},
{
"tag": "IMG",
"index": 6,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "BD9BarA8",
"parentText": ""
},
{
"tag": "IMG",
"index": 7,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app.beb4a83a0d39bb5d.png",
"alt": "",
"class": "jMPyhzfG",
"parentText": ""
},
{
"tag": "IMG",
"index": 8,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "efPPcdLl",
"parentText": ""
},
{
"tag": "IMG",
"index": 9,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/get_app_hover.c9e81f511d248ae7.png",
"alt": "",
"class": "RnpNMA46",
"parentText": ""
},
{
"tag": "IMG",
"index": 10,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "FqRV7w1P",
"parentText": ""
},
{
"tag": "IMG",
"index": 11,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "_Whzlv1b",
"parentText": ""
},
{
"tag": "IMG",
"index": 12,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "j1TwxzPC",
"parentText": ""
},
{
"tag": "IMG",
"index": 13,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "zYgniQaG",
"parentText": ""
},
{
"tag": "IMG",
"index": 14,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "uJKU1tdN",
"parentText": ""
},
{
"tag": "IMG",
"index": 15,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "HkY6seUs",
"parentText": ""
},
{
"tag": "IMG",
"index": 16,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "khDBMSjy",
"parentText": ""
},
{
"tag": "IMG",
"index": 17,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app.795ee19b3ececa6b.png",
"alt": "",
"class": "GkION2OQ",
"parentText": ""
},
{
"tag": "IMG",
"index": 18,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "q2uupcgz",
"parentText": ""
},
{
"tag": "IMG",
"index": 19,
"width": 128,
"height": 40,
"x": 16,
"y": 58,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/fold_get_app_hover.911c3dd97d692c11.png",
"alt": "",
"class": "n6fjbOcQ",
"parentText": ""
},
{
"tag": "IMG",
"index": 20,
"width": 128,
"height": 123,
"x": 16,
"y": 701,
"src": "https://lf-douyin-pc-web.douyinstatic.com/obj/douyin-pc-web/ies/douyin_web/media/jxBtnBgV4.4405b8dd83623e92.png",
"alt": "",
"class": "ACBHzWNP",
"parentText": "手机随时看更方便\n下载 APP"
},
{
"tag": "IMG",
"index": 27,
"width": 339.328125,
"height": 190.859375,
"x": 184,
"y": -343,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_32cb3c82390459d1d91bd4f30c5d8ce7~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "【清稚竹马】我还想说，我想你了！#ai漫剧 #原创动画 #漫剧 #校园",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 28,
"width": 339.328125,
"height": 190.859375,
"x": 539.328125,
"y": -343,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_ea6e0dc453683550de835cc2ded929b0~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "法国搞笑三人组新作，结尾太好笑了 法国喜剧#电影长尾豹马修 #喜剧电影解说",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 29,
"width": 339.34375,
"height": 190.875,
"x": 894.65625,
"y": -343,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_dfc1793c444e4e9731b08c24d409cfff~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "你我怎么两清……#戴上耳机 #甲乙丙丁 #李佳薇 #音乐分享",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 30,
"width": 339.328125,
"height": 190.859375,
"x": 184,
"y": -38.125,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_032fb5c8e6ddd4703e115f1139fe01cf~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "被外卖大哥不小心蹭了车，但没想到他的手机铃声竟然是我的歌…但也正因如此我才有幸走进了一个父与子的故事里#人间观察计划#外卖小哥 #看见100种生活#日常分享 #雪下的时候",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 31,
"width": 339.328125,
"height": 190.859375,
"x": 539.328125,
"y": -38.125,
"src": "https://p9-pc-sign.douyinpic.com/tos-cn-i-dy/ef60f35dda7a4b1396df4b5b5abfb632~tplv-dy-vqe2-sr-opt1:640:480:q80.webp?from=1189464143&lk3s=46e5c84f&x-expires=1788685200&x-signature=YYrBL5uYHi0RBKP8Do8J0",
"alt": "一口气听完当年火遍全网的说唱，谁的DNA动了#中文说唱 #马思唯 #kkluv #创作者扶持计划 #抖音精选",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 32,
"width": 339.34375,
"height": 190.875,
"x": 894.65625,
"y": -38.125,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_d4d6aace7531ed9cbd364c4313a21352~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "当你穿进老钱班33#老钱班 #侯绿萝#olly懂你漂亮做自己 #olly女性复合维生素",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 33,
"width": 339.328125,
"height": 190.859375,
"x": 184,
"y": 266.75,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_49bbd3ffa13175f85795123107c70169~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "轮回神话5 女儿试炼误入绝境，获S级血统轰动全宇宙！探秘禁忌陵宫，竟发现横扫万界的创世神正是自家咸鱼老爸！#原创动画 #二次元 #剧情 #反转 #扮猪吃虎名场面",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 34,
"width": 339.328125,
"height": 190.859375,
"x": 539.328125,
"y": 266.75,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/70d71f59a2bab8ab8a238f9276777e7a~tplv-dy-vqe2-sr-opt1:640:480:q80.webp?from=1189464143&lk3s=46e5c84f&x-expires=1788685200&x-signature=vsIbNW%2BtgYYmFILlW",
"alt": "深度解析《大明王朝1566》 明成祖朱棣定下的锦衣卫选拔标准，一般人还真达不到#大明王朝1566 #历史",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 35,
"width": 339.34375,
"height": 190.875,
"x": 894.65625,
"y": 266.75,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_61db88f71bdba90e436a783bae92863f~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "当大哥不接暗号，鼠鼠带着九格强行认大哥会发生什么呢？ #三角洲行动 #三角洲得吃就行挑战 #鼠鼠我呀得吃了 #三角洲最仁义玩家 #洲人洲事",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
},
{
"tag": "IMG",
"index": 36,
"width": 339.328125,
"height": 190.859375,
"x": 184,
"y": 571.625,
"src": "https://p3-pc-sign.douyinpic.com/image-cut-tos/dp_17d53951c4fb7c73dbfa5acddfde35a3~tplv-dy-resize-walign-adapt-aq:540:q75.jpeg?biz_tag=aweme_video&from=1189464143&lk3s=46e5c84f&s=PackSourceEnum_DOUYIN",
"alt": "本想应付体验大学生活的表弟，不料竟意外发现表弟的万能用处 #搞笑 #动漫 #轻漫计划 #充能计划",
"class": "XTdkxrLI discover-video-card-img",
"parentText": ""
}
]
}
backend/debug_qr_inspect.py
+239
View File
@@ -0,0 +1,239 @@
"""抖音登录二维码元素结构诊断脚本"""
import asyncio
import base64
import json
import os
import sys
from playwright.async_api import async_playwright
ROOT = os.path.dirname(os.path.abspath(__file__))
PROJECT_ROOT = os.path.dirname(ROOT)
BROWSERS_PATH = os.path.join(PROJECT_ROOT, "playwright-browsers")
os.environ.setdefault("PLAYWRIGHT_BROWSERS_PATH", BROWSERS_PATH)
OUT_DIR = os.path.join(ROOT, "debug_qr")
os.makedirs(OUT_DIR, exist_ok=True)
async def inspect():
async with async_playwright() as p:
print("launching browser")
browser = await p.chromium.launch(headless=True)
context = await browser.new_context(
viewport={"width": 1280, "height": 900},
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
)
page = await context.new_page()
print("goto douyin.com")
await page.goto("https://www.douyin.com", wait_until="load")
await asyncio.sleep(6)
print("page url:", page.url)
print("page title:", await page.title())
html = await page.content()
with open(os.path.join(OUT_DIR, "page_initial.html"), "w", encoding="utf-8") as f:
f.write(html)
print("saved page_initial.html")
# 点击登录按钮，触发登录弹窗
login_clicked = False
for sel in ["text=登录", "text=登录/注册", "text=立即登录", "button:has-text('登录')", "[class*='login']", "[class*='Login']"]:
try:
el = await page.wait_for_selector(sel, timeout=3000)
if el:
await el.click()
print("clicked via selector:", sel)
login_clicked = True
break
except Exception as e:
print(f"selector {sel} failed: {e}")
if not login_clicked:
for attempt in range(3):
try:
clicked = await page.evaluate("""() => {
const nodes = [...document.querySelectorAll('button, span, div, a, p')];
for (const el of nodes) {
const t = (el.innerText || '').trim();
if ((t.includes('登录') || t.toLowerCase().includes('login')) && el.offsetParent) {
el.click();
return t;
}
}
return '';
}""")
print("clicked via js:", clicked)
if clicked:
break
except Exception as e:
print(f"js click attempt {attempt} err: {e}")
await asyncio.sleep(1)
await asyncio.sleep(4)
html = await page.content()
with open(os.path.join(OUT_DIR, "page_after_login_click.html"), "w", encoding="utf-8") as f:
f.write(html)
print("saved page_after_login_click.html")
try:
await page.screenshot(path=os.path.join(OUT_DIR, "00_viewport.png"), full_page=False, timeout=10000)
print("saved 00_viewport.png")
except Exception as e:
print("viewport screenshot failed:", e)
# 尝试切换「扫码登录」
for frame in page.frames:
try:
switched = await frame.evaluate("""() => {
const nodes = [...document.querySelectorAll('span, div, a, button, p')];
let best = null;
for (const el of nodes) {
const t = (el.innerText || '').trim();
if ((t === '扫码登录' || t === '扫码') && el.offsetParent) {
if (!best || el.children.length < best.children.length) best = el;
}
}
if (best) { best.click(); return 'switched'; }
return '';
}""")
print(f"frame {frame.url[:60]} switched={switched}")
except Exception as e:
print(f"frame switch err: {e}")
await asyncio.sleep(2)
report = {"frames": [], "candidates": []}
# 遍历所有 frame，查找二维码相关元素
for idx, frame in enumerate(page.frames):
frame_report = {"index": idx, "url": frame.url, "qrcodes": [], "panels": []}
selectors = [
"[class*='qrcode'] img",
"[class*='QrCode'] img",
"[class*='qr-code'] img",
"img[class*='qrcode']",
"img[src*='qrcode']",
"img[alt*='二维码']",
"img[alt*='qr']",
"[class*='qrcode'] canvas",
"canvas[class*='qrcode']",
"[class*='scan'] img",
"[class*='scan'] canvas",
"img",
"canvas",
]
for sel in selectors:
try:
els = await frame.query_selector_all(sel)
for el in els:
try:
visible = await el.is_visible()
box = await el.bounding_box()
tag = await el.evaluate("e => e.tagName")
src = await el.get_attribute("src") or ""
alt = await el.get_attribute("alt") or ""
cls = await el.get_attribute("class") or ""
outer = await el.evaluate("e => e.outerHTML.slice(0, 300)")
info = {
"selector": sel,
"tag": tag,
"visible": visible,
"box": box,
"src_prefix": src[:120] if src else "",
"alt": alt,
"class": cls,
"outer": outer,
}
if (tag.lower() in ("img", "canvas") and box and box.get("width", 0) > 40 and visible):
frame_report["qrcodes"].append(info)
# 截图该元素
safe_name = f"frame{idx}_{tag}_{int(box['x'])}_{int(box['y'])}.png"
try:
await el.screenshot(path=os.path.join(OUT_DIR, safe_name))
info["screenshot"] = safe_name
except Exception as e:
info["screenshot_err"] = str(e)
except Exception as e:
print(f" el inspect err: {e}")
except Exception as e:
print(f"frame {idx} selector {sel} err: {e}")
# 登录面板/容器
panel_selectors = [
"[class*='qrcode-container']",
"[class*='qrcodeContainer']",
"[class*='qrcode']",
"[class*='QrCode']",
"[class*='login-scan']",
"[class*='scan-code']",
"#login-pannel",
"[class*='login_panel']",
"[class*='login-panel']",
"[class*='account_login']",
]
for sel in panel_selectors:
try:
els = await frame.query_selector_all(sel)
for el in els:
visible = await el.is_visible()
box = await el.bounding_box()
cls = await el.get_attribute("class") or ""
if visible and box and box.get("width", 0) > 80:
frame_report["panels"].append({
"selector": sel,
"class": cls,
"box": box,
})
safe_name = f"frame{idx}_panel_{int(box['x'])}_{int(box['y'])}.png"
try:
await el.screenshot(path=os.path.join(OUT_DIR, safe_name))
frame_report["panels"][-1]["screenshot"] = safe_name
except Exception as e:
frame_report["panels"][-1]["screenshot_err"] = str(e)
except Exception as e:
pass
report["frames"].append(frame_report)
# 尝试用 JS 暴力查找所有 img/canvas 中可能为二维码的
all_candidates = await page.evaluate("""() => {
const out = [];
document.querySelectorAll('img, canvas').forEach((el, i) => {
const rect = el.getBoundingClientRect();
if (rect.width > 30 && rect.height > 30 && rect.width < 600 && rect.height < 600) {
const style = window.getComputedStyle(el);
out.push({
tag: el.tagName,
index: i,
width: rect.width,
height: rect.height,
x: rect.x,
y: rect.y,
src: el.tagName === 'IMG' ? (el.src || '').slice(0, 200) : '',
alt: el.alt || '',
class: el.className || '',
parentText: (el.parentElement ? el.parentElement.innerText : '').slice(0, 80),
});
}
});
return out;
}""")
report["candidates"] = all_candidates
with open(os.path.join(OUT_DIR, "report.json"), "w", encoding="utf-8") as f:
json.dump(report, f, ensure_ascii=False, indent=2)
print("report saved to", os.path.join(OUT_DIR, "report.json"))
print("found qrcode-like elements:", sum(len(f["qrcodes"]) for f in report["frames"]))
print("found panels:", sum(len(f["panels"]) for f in report["frames"]))
await browser.close()
if __name__ == "__main__":
try:
asyncio.run(inspect())
except Exception as e:
print("FATAL:", e, file=sys.stderr)
import traceback
traceback.print_exc()
raise
backend/kefu.db-shm
BIN
View File
Binary file not shown.
backend/kefu.db-wal
BIN
View File
Binary file not shown.
backend/kefu1.db
BIN
View File
Binary file not shown.
backend/main.py
+170 -1
View File
@@ -4,6 +4,7 @@ import json
import asyncio
import ipaddress
import logging
import time
import uuid
from datetime import datetime, timezone
from io import BytesIO
@@ -92,6 +93,7 @@ from utils.cookie_store import (
cookie_summary,
validate_cookie_json,
analyze_cookie,
extract_user_agent_from_cookie_data,
)
from rpa_engine.device_profiles import list_device_profiles, profile_label_for_ua, resolve_user_agent
from rpa_engine.egress_channels import (
@@ -100,6 +102,22 @@ from rpa_engine.egress_channels import (
)
logger = logging.getLogger("main")
def _ui_conversation_page_budget() -> int:
"""用户点开会话列表时允许翻的收件箱页数。
抖音收件箱按游标分页，一次请求只给一页（实测每页约 100-500KB）；某账号翻
6 页拿到 35 个会话仍未翻完。所以这里必须有预算：只拿一页会把「其中一页」
当成完整列表，不设上限又可能为一次点击拉下好几 MB。
默认 3 页只是个折中——真正花多少流量换多完整的列表是业务取舍，
用 KEFU_UI_CONVERSATION_PAGES 调整；翻不完时仍会并入本地历史，
并且不会把残缺列表伪装成完整列表。
"""
try:
value = int(os.getenv("KEFU_UI_CONVERSATION_PAGES", "3") or 3)
except (TypeError, ValueError):
value = 3
return max(1, min(20, value))
from utils import system_logger
app = FastAPI(title="抖音多账号自动回复管理系统 API")
@@ -113,11 +131,17 @@ app.add_middleware(
)
# RPA 任务管理器
# 自动重登录防抖：同账号 30 分钟内最多触发一次，防止「扫码失败→失效→再重登录」死循环
_AUTO_RELOGIN_COOLDOWN = float(os.getenv("KEFU_AUTO_RELOGIN_COOLDOWN", "1800") or 1800)
class WorkerManager:
def __init__(self):
self.workers = {} # account_id -> DouyinWorker
self._account_locks: dict[int, asyncio.Lock] = {}
self._preparation_locks: dict[int, asyncio.Lock] = {}
self._auto_relogin_tasks: dict[int, asyncio.Task] = {}
self._last_auto_relogin_at: dict[int, float] = {}
def _account_lock(self, account_id: int) -> asyncio.Lock:
return self._account_locks.setdefault(int(account_id), asyncio.Lock())
@@ -145,6 +169,10 @@ class WorkerManager:
account_id,
login_mode=login_mode,
credential_prevalidated=credential_prevalidated,
# 登录态失效（KICK/INVALID_REQUEST/用户未登录）时自动重登录：
# 重新以 browser 模式拉起 worker，浏览器探测未登录 → 弹二维码
# → 用户扫码 → 自动采集凭证并恢复托管。
relogin_hook=self._schedule_auto_relogin,
)
self.workers[account_id] = worker
await worker.start()
@@ -209,6 +237,98 @@ class WorkerManager:
worker = self.workers.get(account_id)
return worker.is_running if worker else False
async def _schedule_auto_relogin(self, account_id: int) -> None:
"""登录态失效后由 worker 回调：防抖 + 后台异步执行自动重登录。
注意：本方法在 service 的发送协程里被 await，必须快速返回，
实际的浏览器重登录流程放到独立 task 中执行。
"""
now = time.monotonic()
last = self._last_auto_relogin_at.get(account_id, 0.0)
if now - last < _AUTO_RELOGIN_COOLDOWN:
logger.info(
f"Account {account_id}: auto relogin skipped "
f"(cooldown {_AUTO_RELOGIN_COOLDOWN}s)"
)
return
self._last_auto_relogin_at[account_id] = now
prev = self._auto_relogin_tasks.get(account_id)
if prev and not prev.done():
logger.info(f"Account {account_id}: auto relogin already in progress")
return
task = asyncio.create_task(
self._auto_relogin_account(account_id),
name=f"auto-relogin-{account_id}",
)
self._auto_relogin_tasks[account_id] = task
def _cleanup(done_task: asyncio.Task) -> None:
if self._auto_relogin_tasks.get(account_id) is done_task:
self._auto_relogin_tasks.pop(account_id, None)
task.add_done_callback(_cleanup)
async def _auto_relogin_account(self, account_id: int) -> None:
"""自动重登录：等旧 worker 退出，置 logging_in，以 browser 模式重启。
新 worker 的浏览器流程会先探测页面登录态：未登录则自动弹二维码
（qr_code_base64 写库，前端账号卡片轮询展示），用户扫码成功后自动
采集 IM 凭证并恢复托管；登录超时/失败则回落到 offline 等人工处理。
"""
try:
# 1) 等旧 worker 完全退出（on_im_session_invalid 已置 is_running=False，
# _run_loop 收尾需要一点时间）
for _ in range(100):
worker = self.workers.get(account_id)
if worker is None or not worker.is_running:
break
await asyncio.sleep(0.2)
# 2) 置 logging_in（前端显示「等待扫码」，二维码由新 worker 生成）
async with AsyncSessionLocal() as db:
account = (
await db.execute(
select(Account).where(Account.id == account_id)
)
).scalar_one_or_none()
if account is None:
return
account.status = "logging_in"
account.qr_code_base64 = None
account.error_message = None
await db.commit()
system_logger.record(
"登录态失效，正在自动重登录",
detail=(
"系统检测到抖音登录态失效，已自动打开登录流程。"
"请留意账号卡片上的二维码，用抖音 App 扫码后托管将自动恢复。"
),
level="warning",
category="auth",
account_id=account_id,
)
# 3) 以 browser 模式重启：浏览器探测未登录 → 弹二维码 → 扫码 → 恢复托管。
# 不等待就绪（wait_until_ready=False），让新 worker 自行走完整登录流程。
await self.start_worker(account_id, login_mode="browser")
except asyncio.CancelledError:
raise
except Exception as exc:
logger.error(f"Account {account_id}: auto relogin failed: {exc}")
try:
async with AsyncSessionLocal() as db:
await db.execute(
update(Account)
.where(Account.id == account_id)
.values(
status="offline",
error_message=f"自动重登录失败：{exc}",
)
)
await db.commit()
except Exception:
pass
manager = WorkerManager()
UPLOAD_DIR = os.path.join(os.path.dirname(__file__), "uploads", "messages")
@@ -443,6 +563,26 @@ def _account_has_cookie(account: Account) -> bool:
return os.path.exists(get_cookie_path(account.id))
def _backfill_user_agent_from_cookie(account: Account, standard_json_str: Optional[str]) -> None:
"""凭证里带登录头（user_agent）且账号未显式配置 UA 时自动回填。
已有自定义 UA（account.user_agent 非空）的账号保持不变，避免覆盖用户选择；
cookie_data 为空或解析失败时静默跳过。回填后发送/接收链路经
_build_account_im_session 统一走 resolve_user_agent(account.user_agent)，
保证 UA 全链路一致。
"""
if account.user_agent:
return
if not standard_json_str:
return
try:
ua = extract_user_agent_from_cookie_data(standard_json_str)
except Exception:
ua = ""
if ua:
account.user_agent = ua
def _build_account_im_session(account: Account) -> DouyinImSession:
cookie_data = _get_account_cookie_data(account)
storage = json.loads(cookie_data) if cookie_data else {}
@@ -941,6 +1081,8 @@ class AccountCookieResponse(BaseModel):
im_status: Optional[str] = None
can_skip_browser: bool = False
should_reset: bool = False
user_agent: Optional[str] = None
user_agent_label: Optional[str] = None
class AccountVideoItem(BaseModel):
@@ -1019,6 +1161,8 @@ async def _build_cookie_response(
im_status=im_detail["im_status"],
can_skip_browser=im_detail["can_skip_browser"],
should_reset=im_detail.get("should_reset", False),
user_agent=account.user_agent or None,
user_agent_label=profile_label_for_ua(account.user_agent),
)
@@ -2054,6 +2198,7 @@ async def update_account_cookie(
account.cookie_path = cookie_path
account.cookie_updated_at = datetime.utcnow()
account.updated_at = datetime.utcnow()
_backfill_user_agent_from_cookie(account, standard_json_str)
await db.execute(
update(AccountProfileDetail)
.where(AccountProfileDetail.account_id == account_id)
@@ -2126,6 +2271,7 @@ async def create_account(
account.cookie_data = standard_json_str
account.cookie_path = cookie_path
account.cookie_updated_at = datetime.utcnow()
_backfill_user_agent_from_cookie(account, standard_json_str)
try:
from rpa_engine.account_profile import apply_douyin_profile
@@ -2760,7 +2906,29 @@ async def get_account_conversations(
if not conversations:
async with DouyinImHttpClient(session, account_id=account_id) as http:
conversations = await http.get_conversations()
# 用户点开会话列表：从头翻，而不是「最近半小时有动静的会话」。
# 抖音没有「一次取回全部会话」的接口，收件箱是按游标分页的，
# 所以这里花一个翻页预算；翻不完时把 DB 历史并进来补齐，
# 免得把其中一页当成完整会话列表展示给用户。
page_budget = _ui_conversation_page_budget()
conversations = await http.get_conversations(
lookback_seconds=0,
max_pages=page_budget,
)
if http.inbox_truncated:
logger.info(
"Account %s conversation list truncated at %d pages; "
"merging local history",
account_id,
page_budget,
)
known = {
str(c.get("conversation_id") or "") for c in conversations
}
my_uid = session.my_uid or 0
for item in await _conversations_from_logs(db, account_id, my_uid):
if str(item.get("conversation_id") or "") not in known:
conversations.append(item)
if not conversations:
my_uid = session.my_uid or 0
@@ -2771,6 +2939,7 @@ async def get_account_conversations(
session.cookie_header(),
session.web_protect_str,
session.keys_str,
user_agent=session.user_agent or "",
)
my_uid = auth.get_uid() or 0
conversations = await _conversations_from_logs(db, account_id, my_uid)
backend/models/db_migrate.py
+40 -2
View File
@@ -53,12 +53,42 @@ def add_index_if_missing(
conn.execute(text(f"CREATE INDEX {index_name} ON {table} ({safe_columns})"))
def widen_mysql_text_columns(conn, table: str, columns: tuple[str, ...]) -> None:
"""Upgrade large account payloads from TEXT to LONGTEXT on MySQL.
SQLite and PostgreSQL TEXT values are not limited to 64 KiB, while MySQL
TEXT is. Browser storage_state and full-page QR/captcha screenshots can
legitimately exceed that size.
"""
if _dialect(conn) != "mysql":
return
allowed = {"cookie_data", "im_session_data", "qr_code_base64"}
try:
reflected = {
item["name"]: str(item.get("type") or "").upper()
for item in inspect(conn).get_columns(table)
}
except Exception:
return
for column in columns:
if column not in allowed or column not in reflected:
continue
if reflected[column] == "LONGTEXT":
continue
conn.execute(
text(f"ALTER TABLE {table} MODIFY COLUMN {column} LONGTEXT NULL")
)
def migrate_accounts_table(conn) -> None:
add_column_if_missing(
conn,
"accounts",
"cookie_data",
{"default": "ALTER TABLE accounts ADD COLUMN cookie_data TEXT"},
{
"default": "ALTER TABLE accounts ADD COLUMN cookie_data TEXT",
"mysql": "ALTER TABLE accounts ADD COLUMN cookie_data LONGTEXT",
},
)
add_column_if_missing(
conn,
@@ -73,7 +103,10 @@ def migrate_accounts_table(conn) -> None:
conn,
"accounts",
"im_session_data",
{"default": "ALTER TABLE accounts ADD COLUMN im_session_data TEXT"},
{
"default": "ALTER TABLE accounts ADD COLUMN im_session_data TEXT",
"mysql": "ALTER TABLE accounts ADD COLUMN im_session_data LONGTEXT",
},
)
add_column_if_missing(
conn,
@@ -138,6 +171,11 @@ def migrate_accounts_table(conn) -> None:
"douyin_uid",
{"default": "ALTER TABLE accounts ADD COLUMN douyin_uid VARCHAR(64)"},
)
widen_mysql_text_columns(
conn,
"accounts",
("cookie_data", "im_session_data", "qr_code_base64"),
)
def migrate_account_videos_table(conn) -> None:
backend/models/models.py
+4 -3
View File
@@ -1,5 +1,6 @@
from datetime import datetime
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, Index, Text, UniqueConstraint
from sqlalchemy.dialects.mysql import LONGTEXT
from sqlalchemy.orm import relationship, validates
from .database import Base
from utils.log_limits import bound_error_log_content, bound_message_log_content
@@ -108,9 +109,9 @@ class Account(Base):
phone = Column(String(20), nullable=True) # 绑定的手机号（可选）
status = Column(String(50), default="offline") # offline, logging_in, online, error
cookie_path = Column(String(255), nullable=True) # 存储 cookie/session 的路径
cookie_data = Column(Text, nullable=True) # Playwright storage_state JSON
cookie_data = Column(Text().with_variant(LONGTEXT(), "mysql"), nullable=True) # Playwright storage_state JSON
cookie_updated_at = Column(DateTime, nullable=True) # Cookie 最近更新时间
im_session_data = Column(Text, nullable=True) # IM 直连会话 (WS URL / device_id 等)
im_session_data = Column(Text().with_variant(LONGTEXT(), "mysql"), nullable=True) # IM 直连会话 (WS URL / device_id 等)
reply_delay_seconds = Column(Integer, default=0) # 账号回复排队间隔；0/NULL=继承系统默认
reply_cooldown_seconds = Column(Integer, nullable=True) # 自动回复冷却秒数；NULL=继承全局设置
follow_welcome_enabled = Column(Boolean, default=False) # 新粉丝关注后自动发送欢迎语
@@ -118,7 +119,7 @@ class Account(Base):
user_agent = Column(Text, nullable=True) # 伪装设备头（User-Agent），空=默认
egress_public_ip = Column(String(64), nullable=True) # 指定公网出口；空=自动选择
egress_auto_attempts = Column(Integer, nullable=False, default=1) # 发送失败时最多串行尝试的出口数
qr_code_base64 = Column(Text, nullable=True) # 当前登录二维码的 base64 字符串
qr_code_base64 = Column(Text().with_variant(LONGTEXT(), "mysql"), nullable=True) # 当前登录二维码的 base64 字符串
error_message = Column(Text, nullable=True) # 错误信息
quota_disabled = Column(Boolean, default=False, index=True) # 额度不足被停用
created_at = Column(DateTime, default=datetime.utcnow)
backend/rpa_engine/account_profile.py
+1137 -1081
View File
File diff suppressed because it is too large Load Diff
backend/rpa_engine/credential.py
+13 -1
View File
@@ -52,7 +52,19 @@ def build_im_session_from_storage(
session.keys_str = saved.keys_str
if saved.web_protect_str and not session.web_protect_str:
session.web_protect_str = saved.web_protect_str
if saved.my_uid and not session.my_uid:
# A UID verified from the account profile must win over collector
# guesses such as web_runtime_security_uid. Persisting this flag
# keeps API/manual-send builders on the same identity as hosting.
if saved.uid_verified and saved.my_uid:
session.my_uid = saved.my_uid
# device_id 必须与 my_uid 指向同一账号：protobuf/frontier 的
# device_id 优先取 session.device_id，凭证里残留的旧设备号
# （如 www 域 web_runtime_security_uid）会导致 device_id != my_uid
# -> 安全网关 decision=KICK。用已核验 UID 同步 device_id。
if str(session.device_id or "") != str(saved.my_uid):
session.device_id = str(saved.my_uid)
session.uid_verified = True
elif saved.my_uid and not session.my_uid:
session.my_uid = saved.my_uid
if saved.device_id and not session.device_id:
session.device_id = saved.device_id
backend/rpa_engine/douyin_im/auth.py
+19 -4
View File
@@ -64,8 +64,15 @@ class DouyinAuth:
self.msToken = None
self.web_id = None
self.source_ip = ""
self.user_agent = None
def perepare_auth(self, cookieStr: str, web_protect_: str = "", keys_: str = ""):
def perepare_auth(
self,
cookieStr: str,
web_protect_: str = "",
keys_: str = "",
user_agent: str = "",
):
self.cookie = trans_cookies(cookieStr)
self.cookie_str = cookieStr
self.msToken = self.cookie["msToken"] if "msToken" in self.cookie else generate_msToken()
@@ -89,6 +96,11 @@ class DouyinAuth:
except Exception as e:
logger.debug(f"keys parse failed: {e}")
if user_agent:
# 让签名上下文记住调用方 UA：query_my_uid / generate_webid 等后续
# 请求会复用它，避免退回硬编码 DEFAULT_USER_AGENT 造成 UA 不一致。
self.user_agent = user_agent
def is_sign_ready(self) -> bool:
return bool(
self.private_key
@@ -109,9 +121,11 @@ class DouyinAuth:
session.cookie_header(),
session.web_protect_str,
session.keys_str,
user_agent=session.user_agent or DEFAULT_USER_AGENT,
)
auth.web_id = session.web_id or session.device_id or None
auth.user_agent = session.user_agent or DEFAULT_USER_AGENT
# device_id 是设备注册号（query/user 的 id），不是账号 UID；
# my_uid 只作为最后兜底，由 resolve_proto_device_id 内部处理。
auth.device_id = resolve_proto_device_id(
session.device_id, session.web_id, session.my_uid
)
@@ -139,9 +153,10 @@ class DouyinAuth:
return self.uid
def query_my_uid(self) -> int:
ua = self.user_agent or DEFAULT_USER_AGENT
url = 'https://www.douyin.com/aweme/v1/web/query/user/'
headers = {
"User-Agent": DEFAULT_USER_AGENT,
"User-Agent": ua,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
}
@@ -156,7 +171,7 @@ class DouyinAuth:
"msToken": self.msToken
}
query = splice_url(params)
abogus = generate_a_bogus(query, user_agent=DEFAULT_USER_AGENT)
abogus = generate_a_bogus(query, user_agent=ua)
params['a_bogus'] = abogus
with source_bound_requests_session(self.source_ip) as client:
backend/rpa_engine/douyin_im/dy_util.py
+9 -2
View File
@@ -123,17 +123,24 @@ def generate_fake_webid(random_length=19):
return random_str
def generate_webid(auth=None, url=""):
def generate_webid(auth=None, url="", user_agent=""):
# 优先用已采集到的 web_id（避免每次发送都发起一次阻塞的 HTTP 请求，导致事件循环卡顿）
cached = getattr(auth, "web_id", None) if auth is not None else None
if cached:
return str(cached)
if url == "":
url = "https://www.douyin.com/discover?modal_id=7376449060384935209"
# UA 优先级：显式参数 > auth.user_agent（from_im_session / perepare_auth 已带）> 全局默认。
# 必须与 a_bogus 签名、其余请求头使用同一个 UA，否则服务端重算失配 -> 7911。
ua = (
user_agent
or (getattr(auth, "user_agent", "") if auth is not None else "")
or DEFAULT_USER_AGENT
)
try:
from .auth import DouyinAuth
headers = {
"User-Agent": DEFAULT_USER_AGENT,
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"upgrade-insecure-requests": "1"
backend/rpa_engine/douyin_im/follower_poll.py
+188 -183
View File
@@ -1,183 +1,188 @@
"""抖音网页版「粉丝列表」拉取，用于检测新粉丝（关注欢迎语功能）。
复用与 peer_profile / account_profile 相同的 a_bogus + msToken + cookie 签名方式，
调用 https://www.douyin.com/aweme/v1/web/user/follower/list/ 拉取本账号最近的粉丝。
返回的每个粉丝含：uid / sec_uid / nickname / follow_status / follower_status。
其中 follow_status 表示「我」与对方的关系：0=未关注 1=我已关注 2=互相关注（互关）。
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any
from .dy_util import (
DEFAULT_USER_AGENT,
generate_a_bogus,
generate_msToken,
generate_webid,
splice_url,
)
from .auth import DouyinAuth
logger = logging.getLogger("douyin_im.follower_poll")
FOLLOWER_LIST_URL = "https://www.douyin.com/aweme/v1/web/user/follower/list/"
def _requests_proxies() -> dict | None:
try:
from rpa_engine.runtime_config import requests_proxies
return requests_proxies()
except Exception:
return None
def _to_int(value: Any) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def _extract_followers(data: dict[str, Any]) -> list[dict[str, Any]]:
raw = data.get("followers")
if not isinstance(raw, list):
return []
out: list[dict[str, Any]] = []
for item in raw:
if not isinstance(item, dict):
continue
uid = str(item.get("uid") or item.get("user_id") or "").strip()
if not uid:
continue
out.append(
{
"uid": uid,
"sec_uid": str(item.get("sec_uid") or item.get("sec_user_id") or "").strip(),
"nickname": str(item.get("nickname") or item.get("nick_name") or "").strip(),
# follow_status：我对对方的关系（2=互关）；follower_status：对方对我的关系
"follow_status": _to_int(item.get("follow_status")),
"follower_status": _to_int(item.get("follower_status")),
}
)
return out
def fetch_recent_followers_sync(
session,
sec_user_id: str,
count: int = 20,
max_time: int = 0,
) -> list[dict[str, Any]]:
"""同步拉取最近粉丝（第一页）。失败返回 []，并在日志里写明原因。"""
import requests
sec_user_id = (sec_user_id or "").strip()
if not sec_user_id:
logger.warning("fetch followers skipped: 缺少本账号 sec_user_id")
return []
try:
auth = DouyinAuth()
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
except Exception as exc:
logger.warning("fetch followers: build auth failed: %s", exc)
return []
ua = session.user_agent or DEFAULT_USER_AGENT
params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"sec_user_id": sec_user_id,
"count": str(count),
"max_time": str(max_time),
"min_time": "0",
"offset": "0",
"source_type": "1",
"gps_access": "0",
"address_book_access": "0",
"is_top": "1",
"update_version_code": "170400",
"pc_client_type": "1",
"version_code": "170400",
"version_name": "17.4.0",
"cookie_enabled": "true",
"screen_width": "1536",
"screen_height": "960",
"browser_language": "zh-CN",
"browser_platform": "Win32",
"browser_name": "Chrome",
"browser_version": "120.0.0.0",
"browser_online": "true",
"os_name": "Windows",
"os_version": "10",
"platform": "PC",
"webid": generate_webid(auth, "https://www.douyin.com/"),
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"msToken": auth.msToken or generate_msToken(),
}
query = splice_url(params)
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
headers = {
"User-Agent": ua,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
}
try:
resp = requests.get(
FOLLOWER_LIST_URL,
params=params,
headers=headers,
cookies=auth.cookie,
timeout=15,
verify=False,
proxies=_requests_proxies(),
)
try:
data = resp.json()
except Exception:
snippet = (resp.text or "")[:200].replace("\n", " ")
logger.warning(
"fetch followers: 非 JSON 响应 (HTTP %s): %s", resp.status_code, snippet
)
return []
if not isinstance(data, dict):
logger.warning("fetch followers: 响应不是 JSON 对象")
return []
status_code = data.get("status_code")
if status_code not in (None, 0):
logger.warning(
"fetch followers: status_code=%s msg=%s",
status_code,
data.get("status_msg") or data.get("message") or "",
)
return []
followers = _extract_followers(data)
logger.info(
"fetch followers ok: 拿到 %s 个粉丝 (has_more=%s total=%s)",
len(followers),
data.get("has_more"),
data.get("total"),
)
return followers
except Exception as exc:
logger.warning("fetch followers failed: %s", exc)
return []
async def fetch_recent_followers(
session,
sec_user_id: str,
count: int = 20,
max_time: int = 0,
) -> list[dict[str, Any]]:
return await asyncio.to_thread(
fetch_recent_followers_sync, session, sec_user_id, count, max_time
)
"""抖音网页版「粉丝列表」拉取，用于检测新粉丝（关注欢迎语功能）。
复用与 peer_profile / account_profile 相同的 a_bogus + msToken + cookie 签名方式，
调用 https://www.douyin.com/aweme/v1/web/user/follower/list/ 拉取本账号最近的粉丝。
返回的每个粉丝含：uid / sec_uid / nickname / follow_status / follower_status。
其中 follow_status 表示「我」与对方的关系：0=未关注 1=我已关注 2=互相关注（互关）。
"""
from __future__ import annotations
import asyncio
import logging
from typing import Any
from .dy_util import (
DEFAULT_USER_AGENT,
generate_a_bogus,
generate_msToken,
generate_webid,
splice_url,
)
from .auth import DouyinAuth
logger = logging.getLogger("douyin_im.follower_poll")
FOLLOWER_LIST_URL = "https://www.douyin.com/aweme/v1/web/user/follower/list/"
def _requests_proxies() -> dict | None:
try:
from rpa_engine.runtime_config import requests_proxies
return requests_proxies()
except Exception:
return None
def _to_int(value: Any) -> int:
try:
return int(value)
except (TypeError, ValueError):
return 0
def _extract_followers(data: dict[str, Any]) -> list[dict[str, Any]]:
raw = data.get("followers")
if not isinstance(raw, list):
return []
out: list[dict[str, Any]] = []
for item in raw:
if not isinstance(item, dict):
continue
uid = str(item.get("uid") or item.get("user_id") or "").strip()
if not uid:
continue
out.append(
{
"uid": uid,
"sec_uid": str(item.get("sec_uid") or item.get("sec_user_id") or "").strip(),
"nickname": str(item.get("nickname") or item.get("nick_name") or "").strip(),
# follow_status：我对对方的关系（2=互关）；follower_status：对方对我的关系
"follow_status": _to_int(item.get("follow_status")),
"follower_status": _to_int(item.get("follower_status")),
}
)
return out
def fetch_recent_followers_sync(
session,
sec_user_id: str,
count: int = 20,
max_time: int = 0,
) -> list[dict[str, Any]]:
"""同步拉取最近粉丝（第一页）。失败返回 []，并在日志里写明原因。"""
import requests
sec_user_id = (sec_user_id or "").strip()
if not sec_user_id:
logger.warning("fetch followers skipped: 缺少本账号 sec_user_id")
return []
try:
auth = DouyinAuth()
auth.perepare_auth(
session.cookie_header(),
session.web_protect_str,
session.keys_str,
user_agent=session.user_agent or DEFAULT_USER_AGENT,
)
except Exception as exc:
logger.warning("fetch followers: build auth failed: %s", exc)
return []
ua = session.user_agent or DEFAULT_USER_AGENT
params = {
"device_platform": "webapp",
"aid": "6383",
"channel": "channel_pc_web",
"sec_user_id": sec_user_id,
"count": str(count),
"max_time": str(max_time),
"min_time": "0",
"offset": "0",
"source_type": "1",
"gps_access": "0",
"address_book_access": "0",
"is_top": "1",
"update_version_code": "170400",
"pc_client_type": "1",
"version_code": "170400",
"version_name": "17.4.0",
"cookie_enabled": "true",
"screen_width": "1536",
"screen_height": "960",
"browser_language": "zh-CN",
"browser_platform": "Win32",
"browser_name": "Chrome",
"browser_version": "120.0.0.0",
"browser_online": "true",
"os_name": "Windows",
"os_version": "10",
"platform": "PC",
"webid": generate_webid(auth, "https://www.douyin.com/"),
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
"msToken": auth.msToken or generate_msToken(),
}
query = splice_url(params)
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
headers = {
"User-Agent": ua,
"Referer": "https://www.douyin.com/",
"Accept": "application/json, text/plain, */*",
}
try:
resp = requests.get(
FOLLOWER_LIST_URL,
params=params,
headers=headers,
cookies=auth.cookie,
timeout=15,
verify=False,
proxies=_requests_proxies(),
)
try:
data = resp.json()
except Exception:
snippet = (resp.text or "")[:200].replace("\n", " ")
logger.warning(
"fetch followers: 非 JSON 响应 (HTTP %s): %s", resp.status_code, snippet
)
return []
if not isinstance(data, dict):
logger.warning("fetch followers: 响应不是 JSON 对象")
return []
status_code = data.get("status_code")
if status_code not in (None, 0):
logger.warning(
"fetch followers: status_code=%s msg=%s",
status_code,
data.get("status_msg") or data.get("message") or "",
)
return []
followers = _extract_followers(data)
logger.info(
"fetch followers ok: 拿到 %s 个粉丝 (has_more=%s total=%s)",
len(followers),
data.get("has_more"),
data.get("total"),
)
return followers
except Exception as exc:
logger.warning("fetch followers failed: %s", exc)
return []
async def fetch_recent_followers(
session,
sec_user_id: str,
count: int = 20,
max_time: int = 0,
) -> list[dict[str, Any]]:
return await asyncio.to_thread(
fetch_recent_followers_sync, session, sec_user_id, count, max_time
)
backend/rpa_engine/douyin_im/frontier.py
+21 -1
View File
@@ -8,7 +8,7 @@ from urllib.parse import unquote
import requests
from .auth import DouyinAuth
from .dy_util import generate_a_bogus, generate_msToken, generate_webid, splice_url
from .dy_util import DEFAULT_USER_AGENT, generate_a_bogus, generate_msToken, generate_webid, splice_url
from .session import DouyinImSession, is_frontier_ws_url
logger = logging.getLogger("douyin_im.frontier")
@@ -42,6 +42,7 @@ def fetch_device_id(session: DouyinImSession) -> str:
session.cookie_header(),
session.web_protect_str,
session.keys_str,
user_agent=session.user_agent or DEFAULT_USER_AGENT,
)
url = "https://www.douyin.com/aweme/v1/web/query/user"
headers = {
@@ -134,12 +135,31 @@ def ensure_frontier_ws(session: DouyinImSession) -> Optional[str]:
logger.info("Using captured real frontier WS URL (with sdk_cert)")
return url
# Only fpid=9 is Douyin private messaging. Other Frontier products (for
# example fpid=971 opened by the generic /chat page shell) also handshake
# successfully but never carry this account's IM push stream.
for url in session.ws_urls:
if (
is_frontier_ws_url(url)
and "zijieapi.com" in url
and "access_key=" in url
and re.search(r"[?&]fpid=9(?:&|$)", url)
and _ws_device_matches_session(session, url)
):
session.ws_urls = [url]
logger.info("Using captured browser frontier WebSocket URL")
return url
for url in session.ws_urls:
if is_frontier_ws_url(url) and _ws_token_looks_encoded(url):
session.ws_urls = [url]
logger.info("Using captured frontier WS URL")
return url
# frontier 按 device_id 寻址推送，它不是账号 UID：抖音 query/user 返回的
# id 才是本浏览器的设备注册号（my_uid 走 DouyinAuth，两者不能互换）。
# 用 my_uid 拼出来的地址握手同样成功，但订阅的是另一个地址，
# 于是长连接一直是「连上但收不到任何私信」。
device_id = resolve_frontier_device_id(session)
if not device_id:
session.ws_urls = []
backend/rpa_engine/douyin_im/http_client.py
+432 -132
View File
@@ -1,6 +1,8 @@
import asyncio
import json
import logging
import re
import time
from typing import Any, Optional
from urllib.parse import urlparse
@@ -13,8 +15,9 @@ from rpa_engine.egress_channels import (
resolve_send_channels,
)
from .conv_util import build_conversation_id, normalize_conversation_id, resolve_peer_uid
from .message_content import format_im_message, serialize_message_content
from .peer_profile import enrich_conversation_item, fetch_peer_profile, is_generic_peer_name
from .protocol import normalize_im_payload, normalize_im_payload_from_bytes, _pick_avatar_url
from .protocol import normalize_im_payload_from_bytes, _pick_avatar_url
from .session import DouyinImSession
logger = logging.getLogger("douyin_im.http")
@@ -23,6 +26,27 @@ IMAPI_BASE = "https://imapi.douyin.com"
# 抖音 IM「按会话拉取消息」cmd（与电商/web 一致）；body 字段号 == cmd。
CMD_GET_MESSAGES_BY_CONVERSATION = 301
# 「按用户拉取收件箱」cmd：抖音网页版打开私信时用它一次性同步各会话最新消息。
CMD_GET_MESSAGES_BY_USER_INIT = 200
# MessageBody 的字段号（与 Response.proto 的 MessageBody 一致）。
_MSG_FIELD_CONVERSATION_ID = 1
_MSG_FIELD_SERVER_MESSAGE_ID = 3
_MSG_FIELD_CONVERSATION_SHORT_ID = 5
_MSG_FIELD_MESSAGE_TYPE = 6
_MSG_FIELD_SENDER = 7
_MSG_FIELD_CONTENT = 8
_CONVERSATION_ID_RE = re.compile(r"^0:\d+:\d+:\d+$")
# 托管轮询只需覆盖长连接断开的时间窗；游标是微秒时间戳，窗口越小响应越小
# （实测同一账号：游标 0 -> 113KB，回看 30 分钟 -> 约 2KB）。
INBOX_POLL_LOOKBACK_SECONDS = 1800.0
# 会话列表被抖音拒绝时的系统日志节流：托管期间每个账号最多每 30 分钟记一次，
# 既保证「收不到私信」有据可查，又不会被每轮轮询刷屏。
_CONVERSATION_REJECT_LOG_INTERVAL = 1800.0
_conversation_reject_logged_at: dict[tuple[int, str], float] = {}
def _pb_varint(n: int) -> bytes:
@@ -94,6 +118,173 @@ def _pb_parse_fields(buf: bytes) -> list[tuple[int, int, Any]]:
return out
def _as_int(value: Any) -> int:
try:
return int(str(value or 0))
except (TypeError, ValueError):
return 0
def _pb_message_body(buf: bytes) -> Optional[dict]:
"""把一段字节按 MessageBody 解析；形状不像就返回 None。
判据是「有合法的 conversation_id + server_message_id」，而不是它出现在
哪个字段号上——收件箱响应里 messages 挂在哪一层随接口而变。
"""
msg: dict = {}
try:
fields = _pb_parse_fields(buf)
except Exception:
return None
for fn, wt, val in fields:
if fn == _MSG_FIELD_CONVERSATION_ID and wt == 2:
try:
conv_id = val.decode("utf-8")
except Exception:
return None
if not _CONVERSATION_ID_RE.match(conv_id):
return None
msg["conversation_id"] = conv_id
elif fn == _MSG_FIELD_SERVER_MESSAGE_ID and wt == 0:
msg["server_message_id"] = str(val)
elif fn == _MSG_FIELD_CONVERSATION_SHORT_ID and wt == 0:
msg["conversation_short_id"] = str(val)
elif fn == _MSG_FIELD_MESSAGE_TYPE and wt == 0:
msg["message_type"] = val
elif fn == _MSG_FIELD_SENDER and wt == 0:
msg["sender"] = str(val)
elif fn == _MSG_FIELD_CONTENT and wt == 2:
msg["content"] = val.decode("utf-8", errors="replace")
if msg.get("conversation_id") and msg.get("server_message_id"):
return msg
return None
def _pb_collect_message_bodies(
buf: bytes,
out: list[dict],
depth: int = 0,
) -> None:
"""递归找出响应体里所有 MessageBody。"""
if depth > 6:
return
parsed = _pb_message_body(buf)
if parsed is not None:
out.append(parsed)
return
try:
fields = _pb_parse_fields(buf)
except Exception:
return
for _fn, wt, val in fields:
if wt == 2 and isinstance(val, bytes) and val:
_pb_collect_message_bodies(val, out, depth + 1)
def _pb_parse_inbox_messages(content: bytes, cmd: int) -> list[dict]:
"""从 get_by_user_init 响应里取出各会话的最新消息。"""
out: list[dict] = []
for fn, wt, val in _pb_parse_fields(content):
if fn != 6 or wt != 2: # Response.body
continue
for bfn, bwt, bval in _pb_parse_fields(val):
if bfn != cmd or bwt != 2: # ResponseBody.<cmd>
continue
_pb_collect_message_bodies(bval, out)
return out
def _is_inbox_control_message(msg: dict) -> bool:
"""收件箱里的会话控制/状态帧（不是用户发的消息）。"""
from .protocol import _is_control_payload
try:
message_type = int(msg.get("message_type") or 0)
except (TypeError, ValueError):
message_type = 0
content_json: Any = None
raw = msg.get("content")
if raw:
try:
content_json = json.loads(raw)
except Exception:
content_json = None
return _is_control_payload(content_json, message_type)
def _pb_parse_inbox_conversations(content: bytes, cmd: int) -> list[dict]:
"""取出 cmd 200 响应里的会话条目。
响应体除了 messages(字段 1) 还带一组会话条目(字段 6)：
f1=conversation_short_id(varint) f4=conversation_id(string)
游标为 0 时这组条目就是账号的完整会话列表，所以「列全部会话」不必再去拉
cmd 203 的 1.5MB 全量快照。
"""
out: list[dict] = []
for fn, wt, val in _pb_parse_fields(content):
if fn != 6 or wt != 2: # Response.body
continue
for bfn, bwt, bval in _pb_parse_fields(val):
if bfn != cmd or bwt != 2:
continue
for cfn, cwt, cval in _pb_parse_fields(bval):
if cfn != 6 or cwt != 2: # repeated conversation entry
continue
short_id = ""
conv_id = ""
for efn, ewt, eval_ in _pb_parse_fields(cval):
if efn == 1 and ewt == 0:
short_id = str(eval_)
elif efn == 4 and ewt == 2:
try:
candidate = eval_.decode("utf-8")
except Exception:
continue
if _CONVERSATION_ID_RE.match(candidate):
conv_id = candidate
if conv_id:
out.append(
{
"conversation_id": conv_id,
"conversation_short_id": short_id,
}
)
return out
def _pb_parse_inbox_page(content: bytes, cmd: int) -> tuple[int, bool]:
"""返回收件箱这一页的 (next_cursor, has_more)。"""
next_cursor = 0
has_more = False
for fn, wt, val in _pb_parse_fields(content):
if fn != 6 or wt != 2:
continue
for bfn, bwt, bval in _pb_parse_fields(val):
if bfn != cmd or bwt != 2:
continue
for cfn, cwt, cval in _pb_parse_fields(bval):
if cfn == 2 and cwt == 0:
next_cursor = int(cval)
elif cfn == 3 and cwt == 0:
has_more = bool(cval)
return next_cursor, has_more
def _pb_response_status(content: bytes) -> tuple[Optional[int], str]:
"""返回 IM protobuf 响应的 (status_code, message)。"""
status: Optional[int] = None
message = ""
try:
for fn, wt, val in _pb_parse_fields(content):
if fn == 3 and wt == 0:
status = int(val)
elif fn == 4 and wt == 2:
message = val.decode("utf-8", errors="replace")
except Exception:
return None, ""
return status, message
def _pb_parse_conversation_messages(content: bytes, cmd: int) -> list[dict]:
"""解析 get_by_conversation 的 protobuf 响应，返回消息列表。"""
out: list[dict] = []
@@ -350,6 +541,13 @@ class DouyinImHttpClient:
# another channel. Ambiguous read timeouts stay false to avoid duplicates.
self.last_send_channel_retryable: bool = False
self.last_request_debug: str = ""
# 会话列表被抖音判定为「请求本身不合法」时置位。换 payload、换 cookie
# 都修不好，上层据此停掉这轮轮询，别每 120 秒白打一次请求。
self.conversation_list_unsupported: bool = False
# 最近一次收件箱响应里的会话条目（只覆盖翻到的那些页）
self._last_inbox_conversations: list[dict] = []
# 翻页预算用尽但抖音还说 has_more：这次拿到的会话列表不完整
self.inbox_truncated: bool = False
self._proxy_url: str = ""
self._source_ip_override = str(source_ip or "").strip()
self._egress_public_ip_override = str(egress_public_ip or "").strip()
@@ -448,16 +646,43 @@ class DouyinImHttpClient:
def _set_error(self, msg: str) -> None:
self.last_error = msg or ""
def _resolve_authoritative_uid(self, auth) -> int:
"""用 query/user 接口核验当前账号真实 UID，并回写 session.my_uid。
def _report_conversation_list_rejected(self, reason: str) -> None:
"""把「会话列表被抖音拒绝」变成可见故障，而不是静默的空收件箱。"""
self._set_error(f"会话列表接口被抖音拒绝：{reason}")
self.conversation_list_unsupported = True
logger.warning(
"Conversation list rejected by IM API (account=%s): %s",
self.account_id,
reason,
)
key = (int(self.account_id or 0), reason)
now = time.monotonic()
last_at = _conversation_reject_logged_at.get(key)
if last_at is not None and now - last_at < _CONVERSATION_REJECT_LOG_INTERVAL:
return
_conversation_reject_logged_at[key] = now
system_logger.record(
"会话列表接口被抖音拒绝，已停用轮询兜底",
detail=(
f"抖音返回：{reason}。该请求本身被判定为不合法，重试也修不好，"
"本轮托管不再重复调用。私信改为完全依赖实时长连接接收；"
"长连接断开期间漏收的消息无法再通过轮询补齐。"
),
level="error",
category="poll",
account_id=self.account_id,
)
采集端从 tea_cache 推断的 my_uid 可能取到访客/对方 id（导致 cmd=609
INVALID_REQUEST、会话列表为 0）。query/user 返回的 user_uid 才是权威值。
核验成功后写回 session 并打标，避免每次发送都请求接口。
def _resolve_authoritative_uid(self, auth) -> int:
"""Resolve a usable UID without overwriting a known IM identity.
Douyin's query/user ``user_uid`` can differ from the UID used by IM.
It is therefore only a last-resort value when the session has no UID;
a profile-verified or collected numeric UID always wins.
"""
sess = self.session
auth.source_ip = self._source_ip
if getattr(sess, "uid_verified", False) and sess.my_uid:
if sess.my_uid:
return int(sess.my_uid)
resolved = None
try:
@@ -466,16 +691,10 @@ class DouyinImHttpClient:
logger.warning(f"query/user 解析 my_uid 失败: {e}")
if resolved and str(resolved).isdigit():
resolved = int(resolved)
old = int(sess.my_uid or 0)
if old and old != resolved:
logger.warning("my_uid 修正(query/user)：采集值 %s -> 权威值 %s", old, resolved)
sess.my_uid = resolved
# 本系统里 device_id 等同账号 uid（采集端常与 my_uid 一起取错，导致会话列表为 0）。
# device_id 为空 / 非数字 / 等于旧的错误 my_uid 时，一并修正为权威 uid。
dev = str(sess.device_id or "")
if (not dev.isdigit()) or (old and dev == str(old)):
if not dev.isdigit():
sess.device_id = str(resolved)
sess.uid_verified = True
return resolved
return int(sess.my_uid or 0)
@@ -734,7 +953,9 @@ class DouyinImHttpClient:
cmd = CMD_GET_MESSAGES_BY_CONVERSATION
try:
request = await asyncio.to_thread(ProtoBuilder.build_normal_request, auth, cmd)
# 同样要用 x_tt_token：带 auth.ticket 时抖音回 OK 但正文恒为空，
# 于是 WS 瘦推送的图片/语音一直补不全真实 content。
request = await asyncio.to_thread(ProtoBuilder.build_read_request, auth, cmd)
conv_req = (
_pb_str(1, conversation_id)
+ _pb_int(2, 1)
@@ -815,83 +1036,207 @@ class DouyinImHttpClient:
pass
return total
async def get_conversations(self, *, enrich_profiles: bool = True) -> list[dict]:
"""拉取会话列表，返回标准化会话"""
payloads = [
{"cursor": 0, "count": 50, "inbox_type": 0},
{"cursor": 0, "limit": 50},
{},
]
conversations = []
seen = set()
async def fetch_inbox_messages(
self,
limit: int = 50,
lookback_seconds: float = INBOX_POLL_LOOKBACK_SECONDS,
max_pages: int = 1,
) -> list[dict]:
"""用 protobuf 拉取收件箱消息，按游标翻页。
for body in payloads:
data = await self._request("POST", "/v1/conversation/list", body)
# Only a transport/parse failure warrants trying GET. An empty
# JSON object/list can be a perfectly valid empty inbox response.
if data is None:
data = await self._request("GET", "/v1/conversation/list", body)
if data is None:
# Payload variants only help with schema compatibility. They
# cannot repair a network outage, so stop after POST + GET
# both fail instead of occupying a scarce global slot for up
# to four more full request timeouts.
logger.warning("Conversation poll transport failed; skipping payload fallbacks")
break
imapi.douyin.com 只接受 protobuf：发 JSON body 会被当成 protobuf 解析，
固定返回 status_code=1 "unexepcted session length"（与 Cookie 无关，
实测不带任何 Cookie 也是同一条错误）。这里用与发送/拉消息同一套 Request
信封，抖音网页版打开私信时用的也是这个 cmd。
status_code = data.get("status_code") if isinstance(data, dict) else None
error_text = ""
if isinstance(data, dict):
error_text = str(
data.get("error_desc")
or data.get("message")
or data.get("error")
or ""
).strip().lower()
explicit_success = status_code in (0, "0")
structured_without_status = (
isinstance(data, (dict, list)) and status_code is None
响应是**分页**的：body 的 f2=next_cursor、f3=has_more，随附的会话条目
只覆盖这一页里出现过的会话。实测 cursor=0 返回 9 个会话且 has_more=1，
翻 6 页后累计 35 个且仍未翻完——所以「一次请求 = 完整会话列表」是错的，
翻不完时必须把 inbox_truncated 置位，别把一页伪装成全部。
"""
from .auth import DouyinAuth
from .proto_builder import ProtoBuilder
cmd = CMD_GET_MESSAGES_BY_USER_INIT
auth = DouyinAuth.from_im_session(self.session)
auth.source_ip = self._source_ip
# 字段 1 是游标（微秒时间戳）。托管轮询只回看一个窗口：这条链路只用来
# 补齐长连接断开期间漏收的消息。lookback_seconds<=0 表示不设游标
# （游标 0 = 从头翻），别算成「now」，那等于只要比此刻更新的消息。
if lookback_seconds <= 0:
cursor = 0
else:
cursor = max(0, int((time.time() - lookback_seconds) * 1_000_000))
messages: list[dict] = []
conversations: list[dict] = []
seen_conversations: set[str] = set()
self.inbox_truncated = False
pages = max(1, int(max_pages))
for page in range(pages):
request = await asyncio.to_thread(
ProtoBuilder.build_read_request, auth, cmd
)
terminal_credential_error = (
status_code not in (None, 0, "0")
and any(
marker in error_text
for marker in (
"empty token",
"invalid token",
"token expired",
"credential expired",
"authentication",
"unauthorized",
"not login",
"not logged",
)
body = _pb_int(1, cursor) + _pb_int(2, int(limit))
payload = request.SerializeToString() + _pb_msg(8, _pb_msg(cmd, body))
resp = await self._post_protobuf(
f"{IMAPI_BASE}/v1/message/get_by_user_init",
auth,
payload,
signed=False,
log_label="inbox",
)
resp.raise_for_status()
status_code, message = _pb_response_status(resp.content)
if status_code is not None and status_code != 0:
self._report_conversation_list_rejected(
message or f"status_code={status_code}"
)
)
return []
if page == 0:
self._adopt_authoritative_uid(resp.content)
messages.extend(_pb_parse_inbox_messages(resp.content, cmd))
for entry in _pb_parse_inbox_conversations(resp.content, cmd):
conv_id = str(entry.get("conversation_id") or "")
if conv_id and conv_id not in seen_conversations:
seen_conversations.add(conv_id)
conversations.append(entry)
normalized = normalize_im_payload(data)
for item in normalized:
name = item.get("sender_name") or ""
key = name or item.get("conversation_id") or ""
if key and key not in seen:
seen.add(key)
conversations.append(item)
# 也从原始结构提取会话级 unread
self._extract_conversation_rows(data, conversations, seen)
# Compatibility payloads are alternatives, not pagination. Stop
# after a successful empty response as well as a non-empty one;
# otherwise every idle account issues three identical endpoint
# calls on every poll. Credential errors cannot be repaired by
# changing only the JSON shape, so do not amplify those either.
if (
conversations
or explicit_success
or structured_without_status
or terminal_credential_error
):
next_cursor, has_more = _pb_parse_inbox_page(resp.content, cmd)
if not has_more:
break
# 游标不前进就停：否则同一页会被无限翻下去。
if not next_cursor or next_cursor == cursor:
break
cursor = next_cursor
if page == pages - 1:
self.inbox_truncated = True
logger.info(
"Inbox paging stopped at the %d-page budget for account %s; "
"%d conversations so far, more remain",
pages,
self.account_id,
len(conversations),
)
self._last_inbox_conversations = conversations
return messages
def _adopt_authoritative_uid(self, content: bytes) -> None:
"""响应字段 13 是抖音认定的本账号 IM uid，用它纠正 session.my_uid。
my_uid 取错时 _is_self_message 拦不住自己发的消息（机器人会自问自答），
resolve_peer_uid 也会把会话对端认成自己。
"""
uid = 0
try:
for fn, wt, val in _pb_parse_fields(content):
if fn == 13 and wt == 0:
uid = int(val)
break
except Exception:
return
if not uid or uid == int(self.session.my_uid or 0):
return
logger.warning(
"Account %s IM uid corrected from %s to %s (imapi response field 13)",
self.account_id,
self.session.my_uid,
uid,
)
self.session.my_uid = uid
self.session.uid_verified = True
async def get_conversations(
self,
*,
enrich_profiles: bool = True,
lookback_seconds: float = INBOX_POLL_LOOKBACK_SECONDS,
max_pages: int = 1,
) -> list[dict]:
"""拉取会话列表，返回标准化会话。
lookback_seconds=0 表示从头翻；max_pages 是翻页预算。抖音不提供「一次
取回全部会话」的接口，翻页预算用完时 inbox_truncated 会被置位——调用方
必须知道自己拿到的可能只是一部分，不能把一页当成完整会话列表。
托管轮询用默认的小窗口 + 单页，只为补齐长连接断开期间漏收的消息。
"""
conversations: list[dict] = []
try:
messages = await self.fetch_inbox_messages(
lookback_seconds=lookback_seconds,
max_pages=max_pages,
)
except Exception as exc:
self._set_error(str(exc))
logger.warning("Conversation poll transport failed: %s", exc)
return []
# 一个会话只保留最新的一条：server_message_id 单调递增。
latest: dict[str, dict] = {}
for msg in messages:
conv_id = str(msg.get("conversation_id") or "")
if not conv_id:
continue
# 会话状态/已读位等控制帧不是用户消息：既不该当成会话预览，
# 更不该被 _handle_incoming 拿去匹配自动回复（WS 侧同样过滤）。
if _is_inbox_control_message(msg):
continue
current = latest.get(conv_id)
if current is None or _as_int(msg.get("server_message_id")) > _as_int(
current.get("server_message_id")
):
latest[conv_id] = msg
# 窗口内没有消息、但账号里确实存在的会话也要出现在列表里，
# 否则「会话列表」会退化成「最近有动静的会话」。
for entry in self._last_inbox_conversations:
conv_id = str(entry.get("conversation_id") or "")
short_id = str(entry.get("conversation_short_id") or "")
if short_id and short_id != "0":
self.session.conv_meta.setdefault(conv_id, {})
self.session.conv_meta[conv_id]["conversation_short_id"] = short_id
if conv_id and conv_id not in latest:
latest[conv_id] = {
"conversation_id": conv_id,
"conversation_short_id": short_id,
"server_message_id": "0",
"content": "",
}
for conv_id, msg in latest.items():
content = str(msg.get("content") or "")
try:
message_type = int(msg.get("message_type") or 0)
except (TypeError, ValueError):
message_type = 0
preview = content
try:
parsed = format_im_message(json.loads(content), message_type)
preview = serialize_message_content(parsed) if parsed else content
except Exception:
pass
short_id = str(msg.get("conversation_short_id") or "")
if short_id and short_id != "0":
# 发送私信需要 short_id；从收件箱顺手补上，省掉一次 create。
self.session.conv_meta.setdefault(conv_id, {})
self.session.conv_meta[conv_id]["conversation_short_id"] = short_id
conversations.append(
{
"conversation_id": conv_id,
"conversation_short_id": short_id,
"sender_name": "",
"sender_avatar": None,
"content": preview,
"raw_content": content,
"message_type": message_type,
"server_message_id": str(msg.get("server_message_id") or ""),
# sender 可能是自己（我方发出的最后一条），不能当 peer；
# 交给 enrich_conversation_item 从 conversation_id 推。
"sender_uid": str(msg.get("sender") or ""),
"unread_count": 0,
}
)
my_uid = int(self.session.my_uid or 0)
enriched: list[dict] = []
@@ -917,51 +1262,6 @@ class DouyinImHttpClient:
logger.info(f"Fetched {len(enriched)} conversations from IM API")
return enriched
def _extract_conversation_rows(self, data: Any, out: list, seen: set, depth: int = 0):
if depth > 10:
return
if isinstance(data, dict):
name = (
data.get("nick_name")
or data.get("nickname")
or (
(data.get("core_info") or {}).get("nick_name")
if isinstance(data.get("core_info"), dict)
else None
)
)
unread = data.get("unread_count") or data.get("unreadCount") or 0
conv_id = data.get("conversation_id") or data.get("conversationId") or ""
preview = ""
last = data.get("last_message") or data.get("latest_message")
if isinstance(last, dict):
preview = last.get("content") or last.get("text") or ""
elif isinstance(last, str):
preview = last
if isinstance(name, str) and name.strip():
name = name.strip()
sender_avatar = _pick_avatar_url(data)
if name not in seen:
try:
unread = int(unread or 0)
except (TypeError, ValueError):
unread = 0
out.append({
"sender_name": name,
"sender_avatar": sender_avatar or None,
"content": str(preview or ""),
"conversation_id": str(conv_id or ""),
"unread_count": unread,
})
seen.add(name)
for v in data.values():
self._extract_conversation_rows(v, out, seen, depth + 1)
elif isinstance(data, list):
for item in data:
self._extract_conversation_rows(item, out, seen, depth + 1)
async def send_text_message(
self,
conversation_id: str,
@@ -1213,7 +1513,7 @@ class DouyinImHttpClient:
self.last_send_channel_retryable = True
detail = (
"抖音安全网关返回 decision=KICK，当前登录/安全会话已被服务端踢下线；"
"请停止托管后用浏览器模式重新登录，并打开一次私信页重新采集凭证"
"系统正在自动重登录，请留意账号卡片上的二维码并扫码"
)
elif decision:
detail = f"抖音安全网关拒绝发送 decision={decision}"
backend/rpa_engine/douyin_im/image_upload.py
+6 -1
View File
@@ -283,8 +283,13 @@ def _fetch_im_upload_sts(session, source_ip: str = "") -> tuple[str, str, str, s
)
auth = DouyinAuth()
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
ua = session.user_agent or DEFAULT_USER_AGENT
auth.perepare_auth(
session.cookie_header(),
session.web_protect_str,
session.keys_str,
user_agent=ua,
)
params = {
"device_platform": "webapp",
backend/rpa_engine/douyin_im/peer_profile.py
+7 -2
View File
@@ -73,9 +73,14 @@ def _requests_proxies() -> dict | None:
def _build_auth(session: DouyinImSession) -> tuple[DouyinAuth, str]:
auth = DouyinAuth()
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
ua = resolve_user_agent(session.user_agent or DEFAULT_USER_AGENT)
auth = DouyinAuth()
auth.perepare_auth(
session.cookie_header(),
session.web_protect_str,
session.keys_str,
user_agent=ua,
)
return auth, ua
backend/rpa_engine/douyin_im/proto_builder.py
+18
View File
@@ -59,6 +59,24 @@ class ProtoBuilder:
request.sdk_cert = normalize_client_cert(auth.client_cert or "")
return request
@staticmethod
def build_read_request(auth, cmd):
"""读接口（收件箱/会话消息）的 Request 信封。
Request.token 必须是 x_tt_token cookie。build_normal_request 填的是
auth.ticket（bd-ticket-guard 票据）：长度合法，服务端照样回
status_code=0 "OK"，但把调用方当成匿名用户，body 恒为空——和「收件箱
里没有消息」完全无法区分。实测同一请求只换 token：
auth.ticket -> 73 字节 0 条；x_tt_token -> 113KB 47 条。
发送接口另有签名，沿用 build_normal_request，不在此处改动。
"""
request = ProtoBuilder.build_normal_request(auth, cmd)
cookies = getattr(auth, "cookie", None) or {}
token = str(cookies.get("x_tt_token") or "").strip()
if token:
request.token = token
return request
@staticmethod
def build_create_conversation_request(auth, toId, myId):
request = ProtoBuilder.build_normal_request(auth, 609)
backend/rpa_engine/douyin_im/protocol.py
+49 -2
View File
@@ -204,9 +204,46 @@ def extract_json_objects(raw: bytes | str) -> list[dict]:
return results
def _looks_like_push_frame(frame) -> bool:
"""判断这段字节确实是 frontier 的 PushFrame 信封。
真实帧一定带 service/method 和 frontier 自己的 headers/traceid；随手一段
二进制偶尔也能被 protobuf 宽松解析成 PushFrame，那种不算。
"""
return bool(
frame.service
or frame.method
or frame.payloadType
or frame.payloadEncoding
or frame.logIdNew
or len(frame.headersList)
)
def _decode_push_frame_payload(frame) -> bytes:
"""取出 PushFrame 内层负载，按 payloadEncoding 解压。
frontier 会用 gzip 压缩 payload；直接把压缩字节喂给 Response.ParseFromString
只会抛异常并被吞掉，整条私信就此丢失。
"""
body = bytes(frame.payload or b"")
if not body:
return b""
encoding = str(frame.payloadEncoding or "").lower()
if encoding in ("gzip", "gz"):
try:
return gzip.decompress(body)
except Exception as exc:
logger.warning("Failed to gunzip frontier frame payload: %s", exc)
return body
return body
def parse_ws_payload(raw: bytes | str) -> list[dict]:
"""解析 WebSocket 二进制帧，返回标准化消息 dict 列表"""
messages = []
frame_payload = b""
is_push_frame = False
# 尝试 Protobuf 解包
if isinstance(raw, bytes):
@@ -214,9 +251,14 @@ def parse_ws_payload(raw: bytes | str) -> list[dict]:
from .static import Live_pb2, Response_pb2
frame = Live_pb2.PushFrame()
frame.ParseFromString(raw)
if frame.payloadType == 'pb':
is_push_frame = _looks_like_push_frame(frame)
frame_payload = _decode_push_frame_payload(frame)
# payloadType 不再作为判据：现网 frontier 帧会带 'pb'、'text/json'
# 或空值，之前只认 'pb' 会把其余帧整帧丢弃。真正的判据是解出来
# 有没有 new_message_notify；解不出就照旧走下面的 JSON/文本兜底。
if frame_payload:
response = Response_pb2.Response()
response.ParseFromString(frame.payload)
response.ParseFromString(frame_payload)
body = response.body
if body.HasField("new_message_notify"):
notify = body.new_message_notify
@@ -306,6 +348,11 @@ def parse_ws_payload(raw: bytes | str) -> list[dict]:
if isinstance(raw, str):
payloads = [raw.encode("utf-8", errors="ignore")]
elif is_push_frame:
# 已确认是 frontier PushFrame：只解析它的内层负载。整帧字节里还有
# seqId / traceid / payloadType 等元数据，拿去做纯文本兜底会把每条
# 「连接建立」等控制帧误当成一条用户私信记录并触发一次自动回复。
payloads = [frame_payload] if frame_payload else []
else:
payloads = [raw]
# 尝试 gzip 解压（frontier 常见）
backend/rpa_engine/douyin_im/service.py
+62 -5
View File
@@ -320,6 +320,7 @@ class DouyinImService:
reply_cooldown_seconds: Optional[int] = None,
cooldown_resolver: Optional[Callable[[], Awaitable[int]]] = None,
refresh_credentials: Optional[Callable[[], Awaitable[bool]]] = None,
send_fallback: Optional[Callable[[str, str], Awaitable[tuple[bool, str]]]] = None,
follow_tick: Optional[Callable[[], Awaitable[None]]] = None,
on_session_invalid: Optional[Callable[[str], Awaitable[None]]] = None,
on_ready: Optional[ReadyFn] = None,
@@ -354,6 +355,11 @@ class DouyinImService:
self._cooldown_resolver = cooldown_resolver
# 由 worker 注入：触发后台重新采集 web_protect/keys（刷新 ts_sign），返回是否刷新成功
self.refresh_credentials = refresh_credentials
# 由 worker 注入的第二套发送方案：当 HTTP 签名发送被安全网关拒绝
# （decision=KICK / 7911 / INVALID_REQUEST）时，用浏览器页面上下文
# 重新发送（真实 JS 生成 a_bogus/bd-ticket-guard，可自愈被踢的会话）。
# 签名: async (conversation_id, content) -> (ok, detail)
self.send_fallback = send_fallback
self._running = False
self._replied_keys: set[str] = set()
self._logged_keys: set[str] = set()
@@ -363,6 +369,9 @@ class DouyinImService:
self._conv_previews: dict[str, str] = {}
self._conv_names: dict[str, str] = {} # uid/conv_id -> nickname
self._conv_meta: dict[str, dict] = {} # conversation_id -> meta
# 抖音判定会话列表请求本身不合法时置位：这轮托管不再重复轮询该接口，
# 实时长连接成为唯一接收通道（已在系统日志里说明）。
self._conversation_list_unsupported = False
self._ws_client: Optional[DouyinImWsClient] = None
self.last_error: str = ""
@@ -1037,6 +1046,11 @@ class DouyinImService:
initial: bool = False,
defer_handlers: bool = False,
) -> list[dict]:
if self._conversation_list_unsupported:
# 抖音已明确拒绝过这个请求本身；重复调用只会每轮浪费一次请求，
# 并把同一条错误反复写进日志。原因已在首次拒绝时记录。
return []
controller = get_traffic_controller()
async with controller.background_slot(
self.account_id,
@@ -1053,6 +1067,14 @@ class DouyinImService:
account_id=self.account_id,
) as http:
conversations = await http.get_conversations(enrich_profiles=False)
if http.conversation_list_unsupported:
self._conversation_list_unsupported = True
logger.warning(
"Account %s disabled conversation reconciliation; "
"the realtime WebSocket is now the only receive path",
self.account_id,
)
return []
# Capture the previous preview before _index_conversations overwrites
# _conv_meta. A conversation-list preview is not inherently a new
@@ -1445,6 +1467,40 @@ class DouyinImService:
if refreshed:
continue
break
# 第二套发送方案（浏览器页面内发送）：
# HTTP 签名发送被安全网关拒绝（KICK/7911/INVALID_REQUEST）时，交给 worker
# 用浏览器页面上下文重发——由抖音页面自带的 security-sdk 在真实环境生成
# a_bogus/bd-ticket-guard，绕开我们 Node execjs 的签名模拟，可自愈被踢会话。
upper_err = (self.last_error or "").upper()
if self.send_fallback and (
"DECISION=KICK" in upper_err
or "STATUS_CODE=7911" in upper_err
or "INVALID_REQUEST" in upper_err
):
try:
fb_ok, fb_detail = await self.send_fallback(conversation_id, content)
except Exception as exc:
logger.warning(f"send_fallback raised for {conversation_id}: {exc}")
fb_ok, fb_detail = False, f"浏览器兜底发送异常：{exc}"
if fb_ok:
self._session_invalid_strikes = 0
self._session_invalid_fired = False # 兜底成功说明登录仍有效，撤销自动下线
system_logger.record(
"浏览器兜底发送成功",
detail=f"会话 {conversation_id}：{fb_detail}",
level="success",
category="send",
account_id=self.account_id,
)
return True, None
system_logger.record(
"浏览器兜底发送失败",
detail=f"会话 {conversation_id}：{fb_detail}",
level="error",
category="send",
account_id=self.account_id,
)
await self._note_session_invalid(self.last_error)
return False, None
@@ -1479,7 +1535,7 @@ class DouyinImService:
system_logger.record(
"IM 登录失效，自动下线",
detail=f"{reason}（{failure_detail}）。"
"请停止托管后用浏览器模式重新登录并打开私信页，再重新启动托管。",
"系统正在自动重登录，请留意账号卡片上的登录二维码并扫码。",
level="error",
category="auth",
account_id=self.account_id,
@@ -1495,17 +1551,18 @@ class DouyinImService:
"""手动发送私信"""
from .conv_util import normalize_conversation_id
from .auth import DouyinAuth
from .dy_util import DEFAULT_USER_AGENT
auth = DouyinAuth()
auth.perepare_auth(
self.session.cookie_header(),
self.session.web_protect_str,
self.session.keys_str,
user_agent=self.session.user_agent or DEFAULT_USER_AGENT,
)
if getattr(self.session, "uid_verified", False) and self.session.my_uid:
my_uid = self.session.my_uid
else:
my_uid = await asyncio.to_thread(lambda: auth.get_uid()) or self.session.my_uid
my_uid = self.session.my_uid
if not my_uid:
my_uid = await asyncio.to_thread(lambda: auth.get_uid()) or 0
if my_uid:
conversation_id = normalize_conversation_id(conversation_id, my_uid)
backend/rpa_engine/douyin_im/session.py
+66 -20
View File
@@ -14,9 +14,21 @@ def is_frontier_ws_url(url: str) -> bool:
真实抓包里 host 可能是 frontier-im.douyin.com，也可能是
frontierNN-normal.zijieapi.com 这类内部别名，二者都要认。
"""
if not url or "token=" not in url:
if not url:
return False
return "frontier-im.douyin.com" in url or ("frontier" in url and "zijieapi.com" in url)
parsed = urlparse(url)
host = (parsed.hostname or "").lower()
query = parse_qs(parsed.query)
fpid = (query.get("fpid") or [""])[0]
legacy = host == "frontier-im.douyin.com" and "token" in query and fpid == "9"
browser_frontier = (
"frontier" in host
and host.endswith("zijieapi.com")
and "access_key" in query
and "device_id" in query
and fpid == "9"
)
return legacy or browser_frontier
@dataclass
@@ -34,7 +46,8 @@ class DouyinImSession:
keys_str: str = ""
web_protect_str: str = ""
my_uid: int = 0
# my_uid 是否已用 query/user 接口核验过（采集端推断的 my_uid 可能取错 tea_cache id）
# my_uid 是否已由账号资料 UID 等可靠来源核验。query/user 的 user_uid
# 不是所有账号的 IM UID，不能据此覆盖已采集/已同步的 my_uid。
uid_verified: bool = False
conv_meta: dict = field(default_factory=dict)
# 方案 A：直接复用浏览器抓到的真实 frontier 连接凭证（绕开我们自己推导 token/access_key 不准的问题）
@@ -96,6 +109,14 @@ class DouyinImSession:
my_uid = _as_uid(extra.get("my_uid")) or _as_uid(data.get("my_uid"))
user_agent = str(extra.get("user_agent") or data.get("user_agent") or "").strip()
# 先整段扫描 localStorage，收集字段（避免遍历顺序导致取值不确定）。
# 关键背景：新版抖音 web 端 __tea_cache_tokens_6383 的 user_unique_id 实际存的是
# web_id（如 7678646545793812008），并非账号 UID；而 web_runtime_security_uid
# 才是账号真实 UID（如 2609567359568155）。混合登录态下若把 tea 的 user_unique_id
# 当 my_uid，会导致 device_id != my_uid，IM 发送被安全网关 KICK。
ls_sec_uid = "" # web_runtime_security_uid（最可靠的账号 UID 来源）
ls_web_id = "" # 第一个 tea 条目的 web_id/user_unique_id
ls_tea_pairs = [] # [(user_unique_id, web_id), ...] 按出现顺序
if not device_id or not web_id or not keys_str or not web_protect_str or not my_uid:
for origin in data.get("origins", []):
for entry in origin.get("localStorage", []):
@@ -107,27 +128,42 @@ class DouyinImSession:
keys_str = value
if name == "security-sdk/s_sdk_sign_data_key/web_protect" and not web_protect_str:
web_protect_str = value
if "tea_cache_tokens" in name and not web_id:
if "tea_cache_tokens" in name:
try:
parsed = json.loads(value)
web_id = str(
parsed.get("web_id")
or parsed.get("user_unique_id")
or ""
)
except Exception:
pass
if name == "web_runtime_security_uid" and not device_id:
if str(value or "").isdigit():
device_id = value
if "tea_cache_tokens" in name and not my_uid:
try:
parsed = json.loads(value)
uid = parsed.get("user_unique_id")
if uid and str(uid).isdigit():
my_uid = int(uid)
if isinstance(parsed, dict):
wid = str(parsed.get("web_id") or "")
uid = str(parsed.get("user_unique_id") or "")
if not ls_web_id:
ls_web_id = wid or uid
ls_tea_pairs.append((uid, wid))
except Exception:
pass
if name == "web_runtime_security_uid":
v = str(value or "")
if v.isdigit() and not ls_sec_uid:
ls_sec_uid = v
# web_id：extra 显式值 > localStorage tea
if not web_id:
web_id = ls_web_id
# my_uid 优先级：extra/顶层 > web_runtime_security_uid（真实账号 UID）>
# tea 的 user_unique_id（仅当与自身 web_id 不同才可信，避免误取 web_id）
if not my_uid and ls_sec_uid:
my_uid = int(ls_sec_uid)
if not my_uid:
for uid, wid in ls_tea_pairs:
if uid.isdigit() and not (wid and uid == wid):
my_uid = int(uid)
break
# device_id 优先级：extra/cookies > web_runtime_security_uid（与账号 UID 绑定）
if not device_id:
if ls_sec_uid:
device_id = ls_sec_uid
elif my_uid:
device_id = str(my_uid)
if not my_uid:
for item in data.get("cookies", []):
@@ -144,6 +180,14 @@ class DouyinImSession:
elif not device_id and web_id:
device_id = web_id
# 最终一致性收敛：protobuf/frontier 的 device_id 优先取 session.device_id
# （resolve_proto_device_id），若凭证里残留旧设备号（如 www 域
# web_runtime_security_uid），发送时 device_id != my_uid 会被安全网关
# 判为设备指纹异常 -> decision=KICK。my_uid 此时已是权威账号 UID，
# 不一致时以 my_uid 收敛 device_id。
if my_uid and device_id and str(device_id) != str(my_uid):
device_id = str(my_uid)
ws_urls = list(extra.get("ws_urls") or [])
# 方案 A：凭证采集工具可携带浏览器抓到的真实 frontier 连接（含 token/sdk_cert/ts_sign）。
@@ -193,6 +237,7 @@ class DouyinImSession:
"keys_str": self.keys_str,
"web_protect_str": self.web_protect_str,
"my_uid": self.my_uid,
"uid_verified": self.uid_verified,
"conv_meta": self.conv_meta,
"sdk_cert": self.sdk_cert,
"frontier_ts_sign": self.frontier_ts_sign,
@@ -212,6 +257,7 @@ class DouyinImSession:
keys_str=str(data.get("keys_str") or ""),
web_protect_str=str(data.get("web_protect_str") or ""),
my_uid=int(data.get("my_uid") or 0),
uid_verified=bool(data.get("uid_verified", False)),
conv_meta=dict(data.get("conv_meta") or {}),
sdk_cert=str(data.get("sdk_cert") or ""),
frontier_ts_sign=str(data.get("frontier_ts_sign") or ""),
backend/rpa_engine/douyin_im/ws_client.py
+80 -6
View File
@@ -1,4 +1,5 @@
import asyncio
import gzip
import logging
import os
import weakref
@@ -14,6 +15,31 @@ logger = logging.getLogger("douyin_im.ws")
MessageHandler = Callable[[dict], Awaitable[None]]
def _safe_frame_metadata(payload: bytes) -> str:
"""Return non-content protobuf metadata for early connection diagnostics."""
try:
from .static import Live_pb2, Response_pb2
frame = Live_pb2.PushFrame()
frame.ParseFromString(payload)
body = bytes(frame.payload)
if str(frame.payloadEncoding or "").lower() == "gzip":
body = gzip.decompress(body)
response = Response_pb2.Response()
response.ParseFromString(body)
fields = [field.name for field, _ in response.body.ListFields()]
message = str(response.message or response.error_desc or "")[:80]
return (
f"service={frame.service} method={frame.method} "
f"encoding={frame.payloadEncoding or 'none'} "
f"type={frame.payloadType or 'none'} payload_bytes={len(body)} "
f"cmd={response.cmd} body={','.join(fields) or 'none'} "
f"status={message or 'ok'}"
)
except Exception as exc:
return f"metadata_unavailable={type(exc).__name__}"
# Both stages are finite. The transport queue gives the receive coroutine a
# small amount of breathing room, while the application queue decouples Pong /
# frame reads from potentially slow database and reply work. Once both fill,
@@ -127,6 +153,8 @@ class DouyinImWsClient:
self._last_connection_lifetime = 0.0
self._message_queue: Optional[asyncio.Queue[dict]] = None
self._dispatcher_task: Optional[asyncio.Task] = None
self._received_frame_count = 0
self._heartbeat_ack_logged = False
async def start(self):
if self._task and not self._task.done():
@@ -331,6 +359,16 @@ class DouyinImWsClient:
headers.append(("Cookie", cookie))
return headers
@staticmethod
def _uses_browser_frontier(url: str) -> bool:
return "zijieapi.com" in url and "access_key=" in url
async def _run_browser_heartbeat(self, websocket) -> None:
"""Mirror Frontier's browser SDK application-level ``hi`` heartbeat."""
while self._running:
await websocket.send("hi")
await asyncio.sleep(30)
async def _run_connection(self, url: str) -> None:
"""Open one connection and dispatch messages sequentially.
@@ -343,6 +381,8 @@ class DouyinImWsClient:
loop = asyncio.get_running_loop()
connected_at: float | None = None
connection: Optional[WebSocketClientProtocol] = None
heartbeat_task: Optional[asyncio.Task] = None
browser_frontier = self._uses_browser_frontier(url)
source_ip = str(getattr(self.session, "egress_source_ip", "") or "").strip()
connect_kwargs = {"local_addr": (source_ip, 0)} if source_ip else {}
try:
@@ -354,7 +394,9 @@ class DouyinImWsClient:
user_agent_header=self.session.user_agent,
compression="deflate",
open_timeout=10,
ping_interval=20,
# The current Douyin browser Frontier SDK uses a text ``hi``
# heartbeat instead of RFC WebSocket ping frames.
ping_interval=None if browser_frontier else 20,
# A handler may legitimately wait up to SQLite's 30s busy
# timeout. Leave enough headroom for queued work so a healthy
# socket isn't mistaken for a dead peer during that stall.
@@ -371,7 +413,10 @@ class DouyinImWsClient:
self._connection = websocket
connected_at = loop.time()
self.connected = True
logger.info("IM WebSocket connected")
logger.info(
"IM WebSocket connected: subprotocol=%s",
getattr(websocket, "subprotocol", None) or "none",
)
self._record_connection_system_event(
"connected",
"实时接收通道已连接",
@@ -379,10 +424,23 @@ class DouyinImWsClient:
level="success",
)
async for raw in websocket:
if not self._running:
break
await self._dispatch(raw)
if browser_frontier:
heartbeat_task = asyncio.create_task(
self._run_browser_heartbeat(websocket),
name=f"im-ws-heartbeat-{self.account_id or 'na'}",
)
try:
async for raw in websocket:
if not self._running:
break
await self._dispatch(raw)
finally:
if heartbeat_task and not heartbeat_task.done():
heartbeat_task.cancel()
try:
await heartbeat_task
except asyncio.CancelledError:
pass
finally:
if connected_at is not None:
self._last_connection_lifetime = max(0.0, loop.time() - connected_at)
@@ -408,6 +466,11 @@ class DouyinImWsClient:
)
async def _dispatch(self, raw):
if raw == "hi":
if not self._heartbeat_ack_logged:
logger.info("IM WebSocket application heartbeat acknowledged")
self._heartbeat_ack_logged = True
return
self._ensure_dispatcher()
queue = self._message_queue
if queue is None:
@@ -417,6 +480,17 @@ class DouyinImWsClient:
else:
payload = raw
items = parse_ws_payload(payload)
self._received_frame_count += 1
if self._received_frame_count <= 3:
metadata = _safe_frame_metadata(payload) if not items else "parsed-message"
logger.info(
"IM WebSocket frame received: seq=%d kind=%s bytes=%d parsed=%d %s",
self._received_frame_count,
"text" if isinstance(raw, str) else "binary",
len(payload),
len(items),
metadata,
)
for item in items:
if not self._running:
return
backend/rpa_engine/playwright_worker.py
+1034 -92
View File
File diff suppressed because it is too large Load Diff
backend/rpa_engine/runtime_config.py
+145 -129
View File
@@ -1,129 +1,145 @@
"""运行时环境配置：住宅代理 + 浏览器显示。
用于解决「部署到云服务器后」两类常见问题：
1. 机房 IP 触发抖音风控（7911）—— 通过 KEFU_DOUYIN_PROXY 让抖音请求走住宅代理。
2. 无图形界面的 Linux 起不来有头浏览器 —— 自动拉起 Xvfb 虚拟显示。
全部通过环境变量控制，无需改代码：
KEFU_DOUYIN_PROXY 抖音 IM HTTP 请求与浏览器登录走的代理，绕开机房 IP 风控。
形如 http://user:pass@host:port 或 socks5://host:port
KEFU_BROWSER_HEADLESS 是否使用无头浏览器（1/true 开启）。默认 false：抖音安全 SDK
对 headless 判定严格，无头易生成无效 ts_sign，反而刷新无效。
"""
import asyncio
import logging
import os
from typing import Optional
from urllib.parse import urlparse
logger = logging.getLogger("rpa_engine.runtime")
_TRUE = {"1", "true", "yes", "on"}
_FALSE = {"0", "false", "no", "off"}
_NO_DISPLAY_HINT = (
"当前是无图形界面的 Linux 服务器，且无法启动虚拟显示来运行有头浏览器。"
"抖音安全 SDK 对 headless 判定严格，扫码登录 / 刷新凭证需要有头 Chromium。请任选其一：\n"
" 1) 安装 Xvfb + pyvirtualdisplay，让程序自动拉起虚拟显示：\n"
" Debian/Ubuntu: apt install -y xvfb && pip install pyvirtualdisplay\n"
" CentOS/Rocky : yum install -y xorg-x11-server-Xvfb && pip install pyvirtualdisplay\n"
" 2) 或用 xvfb-run 启动后端：xvfb-run -a ./start_web.sh\n"
" 3) 或设置 KEFU_BROWSER_HEADLESS=1 强制无头（更易触发抖音风控，不推荐）。"
)
def get_douyin_proxy() -> Optional[str]:
"""读取抖音请求代理 URL（未配置返回 None）。"""
val = (os.getenv("KEFU_DOUYIN_PROXY") or "").strip()
return val or None
def httpx_proxy() -> Optional[str]:
"""供 httpx.AsyncClient(proxy=...) 使用的代理 URL。"""
return get_douyin_proxy()
def requests_proxies() -> Optional[dict]:
"""供 requests.get(proxies=...) 使用的代理字典。"""
url = get_douyin_proxy()
if not url:
return None
return {"http": url, "https": url}
def playwright_proxy() -> Optional[dict]:
"""转成 Playwright launch(proxy=...) 所需结构（未配置或无法解析返回 None）。"""
url = get_douyin_proxy()
if not url:
return None
parsed = urlparse(url)
if not parsed.hostname:
logger.warning("KEFU_DOUYIN_PROXY 格式无法解析，已忽略：%s", url)
return None
server = f"{parsed.scheme or 'http'}://{parsed.hostname}"
if parsed.port:
server += f":{parsed.port}"
proxy: dict[str, str] = {"server": server}
if parsed.username:
proxy["username"] = parsed.username
if parsed.password:
proxy["password"] = parsed.password
return proxy
def resolve_headless(default: bool = False) -> bool:
"""根据 KEFU_BROWSER_HEADLESS 决定是否无头；未设置时用 default。"""
val = (os.getenv("KEFU_BROWSER_HEADLESS") or "").strip().lower()
if val in _TRUE:
return True
if val in _FALSE:
return False
return default
# 进程内仅启动一次的虚拟显示（Xvfb）句柄
_virtual_display = None
_virtual_display_failed = False
def _start_virtual_display_sync() -> Optional[str]:
"""在无 DISPLAY 的 Linux 上启动一次 Xvfb 虚拟显示（阻塞，需放线程执行）。"""
global _virtual_display, _virtual_display_failed
# 仅 Linux 且无 DISPLAY 时才需要虚拟显示；Windows/macOS 有桌面，直接返回。
if os.name != "posix":
return os.environ.get("DISPLAY")
if os.environ.get("DISPLAY"):
return os.environ["DISPLAY"]
if _virtual_display is not None:
return os.environ.get("DISPLAY")
if _virtual_display_failed:
raise RuntimeError(_NO_DISPLAY_HINT)
try:
from pyvirtualdisplay import Display
except ImportError as e:
_virtual_display_failed = True
raise RuntimeError(_NO_DISPLAY_HINT) from e
try:
disp = Display(visible=False, size=(1280, 800))
disp.start() # 设置 os.environ['DISPLAY']
except Exception as e:
_virtual_display_failed = True
raise RuntimeError(_NO_DISPLAY_HINT) from e
_virtual_display = disp
logger.info("已启动 Xvfb 虚拟显示 DISPLAY=%s 供有头浏览器使用", os.environ.get("DISPLAY"))
return os.environ.get("DISPLAY")
async def ensure_browser_display(headless: bool) -> None:
"""有头模式在无 DISPLAY 的 Linux 上自动拉起 Xvfb 虚拟显示。
headless=True 时无需显示，直接返回；启动失败抛出带操作指引的 RuntimeError。
"""
if headless:
return
await asyncio.to_thread(_start_virtual_display_sync)
"""运行时环境配置：住宅代理 + 浏览器显示。
用于解决「部署到云服务器后」两类常见问题：
1. 机房 IP 触发抖音风控（7911）—— 通过 KEFU_DOUYIN_PROXY 让抖音请求走住宅代理。
2. 无图形界面的 Linux 起不来有头浏览器 —— 自动拉起 Xvfb 虚拟显示。
全部通过环境变量控制，无需改代码：
KEFU_DOUYIN_PROXY 抖音 IM HTTP 请求与浏览器登录走的代理，绕开机房 IP 风控。
形如 http://user:pass@host:port 或 socks5://host:port
KEFU_BROWSER_HEADLESS 是否使用无头浏览器（1/true 开启）。默认 false：抖音安全 SDK
对 headless 判定严格，无头易生成无效 ts_sign，反而刷新无效。
"""
import asyncio
import logging
import os
from typing import Optional
from urllib.parse import urlparse
logger = logging.getLogger("rpa_engine.runtime")
_TRUE = {"1", "true", "yes", "on"}
_FALSE = {"0", "false", "no", "off"}
_NO_DISPLAY_HINT = (
"当前是无图形界面的 Linux 服务器，且无法启动虚拟显示来运行有头浏览器。"
"抖音安全 SDK 对 headless 判定严格，扫码登录 / 刷新凭证需要有头 Chromium。请任选其一：\n"
" 1) 安装 Xvfb + pyvirtualdisplay，让程序自动拉起虚拟显示：\n"
" Debian/Ubuntu: apt install -y xvfb && pip install pyvirtualdisplay\n"
" CentOS/Rocky : yum install -y xorg-x11-server-Xvfb && pip install pyvirtualdisplay\n"
" 2) 或用 xvfb-run 启动后端：xvfb-run -a ./start_web.sh\n"
" 3) 或设置 KEFU_BROWSER_HEADLESS=1 强制无头（更易触发抖音风控，不推荐）。"
)
def get_douyin_proxy() -> Optional[str]:
"""读取抖音请求代理 URL（未配置返回 None）。"""
val = (os.getenv("KEFU_DOUYIN_PROXY") or "").strip()
return val or None
def httpx_proxy() -> Optional[str]:
"""供 httpx.AsyncClient(proxy=...) 使用的代理 URL。"""
return get_douyin_proxy()
def requests_proxies() -> Optional[dict]:
"""供 requests.get(proxies=...) 使用的代理字典。"""
url = get_douyin_proxy()
if not url:
return None
return {"http": url, "https": url}
def playwright_proxy() -> Optional[dict]:
"""转成 Playwright launch(proxy=...) 所需结构（未配置或无法解析返回 None）。"""
url = get_douyin_proxy()
if not url:
return None
parsed = urlparse(url)
if not parsed.hostname:
logger.warning("KEFU_DOUYIN_PROXY 格式无法解析，已忽略：%s", url)
return None
server = f"{parsed.scheme or 'http'}://{parsed.hostname}"
if parsed.port:
server += f":{parsed.port}"
proxy: dict[str, str] = {"server": server}
if parsed.username:
proxy["username"] = parsed.username
if parsed.password:
proxy["password"] = parsed.password
return proxy
def resolve_headless(default: bool = False) -> bool:
"""根据 KEFU_BROWSER_HEADLESS 决定是否无头；未设置时用 default。"""
val = (os.getenv("KEFU_BROWSER_HEADLESS") or "").strip().lower()
if val in _TRUE:
return True
if val in _FALSE:
return False
return default
def ui_conversation_page_budget(default: int = 3) -> int:
"""用户点开会话列表时允许翻的收件箱页数（KEFU_UI_CONVERSATION_PAGES）。
抖音收件箱按游标分页，一次请求只给一页（实测每页约 100-500KB）；某账号翻
6 页拿到 35 个会话仍未翻完。所以必须有预算：只拿一页会把「其中一页」当成
完整会话列表，不设上限又可能为一次点击拉下好几 MB。默认 3 页只是折中，
花多少流量换多完整的列表属于业务取舍，用环境变量调整即可。
无论调到多少，翻不完时都会并入本地历史，不会把残缺列表伪装成完整列表。
"""
try:
value = int(os.getenv("KEFU_UI_CONVERSATION_PAGES", str(default)) or default)
except (TypeError, ValueError):
value = default
return max(1, min(20, value))
# 进程内仅启动一次的虚拟显示（Xvfb）句柄
_virtual_display = None
_virtual_display_failed = False
def _start_virtual_display_sync() -> Optional[str]:
"""在无 DISPLAY 的 Linux 上启动一次 Xvfb 虚拟显示（阻塞，需放线程执行）。"""
global _virtual_display, _virtual_display_failed
# 仅 Linux 且无 DISPLAY 时才需要虚拟显示；Windows/macOS 有桌面，直接返回。
if os.name != "posix":
return os.environ.get("DISPLAY")
if os.environ.get("DISPLAY"):
return os.environ["DISPLAY"]
if _virtual_display is not None:
return os.environ.get("DISPLAY")
if _virtual_display_failed:
raise RuntimeError(_NO_DISPLAY_HINT)
try:
from pyvirtualdisplay import Display
except ImportError as e:
_virtual_display_failed = True
raise RuntimeError(_NO_DISPLAY_HINT) from e
try:
disp = Display(visible=False, size=(1280, 800))
disp.start() # 设置 os.environ['DISPLAY']
except Exception as e:
_virtual_display_failed = True
raise RuntimeError(_NO_DISPLAY_HINT) from e
_virtual_display = disp
logger.info("已启动 Xvfb 虚拟显示 DISPLAY=%s 供有头浏览器使用", os.environ.get("DISPLAY"))
return os.environ.get("DISPLAY")
async def ensure_browser_display(headless: bool) -> None:
"""有头模式在无 DISPLAY 的 Linux 上自动拉起 Xvfb 虚拟显示。
headless=True 时无需显示，直接返回；启动失败抛出带操作指引的 RuntimeError。
"""
if headless:
return
await asyncio.to_thread(_start_virtual_display_sync)
backend/tests/test_conversation_poll_bandwidth.py
+55 -51
View File
@@ -30,67 +30,69 @@ class ConversationPollBandwidthTests(unittest.IsolatedAsyncioTestCase):
account_id=9,
)
async def test_terminal_token_error_does_not_probe_other_payloads(self):
async def test_inbox_is_fetched_with_exactly_one_protobuf_request(self):
"""imapi 只认 protobuf；轮询一次就只该发一个请求。"""
client = self._make_client()
client._request = AsyncMock(
return_value={
"status_code": 500,
"error_desc": "empty token",
"body": {},
}
)
client.fetch_inbox_messages = AsyncMock(return_value=[])
client._request = AsyncMock()
self.assertEqual(await client.get_conversations(), [])
client._request.assert_awaited_once()
self.assertEqual(client._request.await_args.args[0], "POST")
client.fetch_inbox_messages.assert_awaited_once()
# 不能再退回 JSON 的 /v1/conversation/list：那个请求恒被抖音拒绝。
client._request.assert_not_awaited()
async def test_successful_empty_response_stops_after_first_payload(self):
async def test_transport_failure_is_recorded_and_returns_empty(self):
client = self._make_client()
client._request = AsyncMock(
return_value={"status_code": 0, "body": {"conversation_list": []}}
)
self.assertEqual(await client.get_conversations(), [])
client._request.assert_awaited_once()
async def test_parameter_error_can_fall_through_to_compatible_payload(self):
client = self._make_client()
client._request = AsyncMock(
side_effect=[
{"status_code": 400, "error_desc": "invalid parameter"},
{"status_code": 0, "body": {"conversation_list": []}},
]
)
self.assertEqual(await client.get_conversations(), [])
self.assertEqual(client._request.await_count, 2)
self.assertTrue(
all(call.args[0] == "POST" for call in client._request.await_args_list)
)
async def test_get_fallback_only_runs_after_transport_failure(self):
client = self._make_client()
client._request = AsyncMock(
side_effect=[None, {"status_code": 0, "body": {}}]
)
self.assertEqual(await client.get_conversations(), [])
self.assertEqual(
[call.args[0] for call in client._request.await_args_list],
["POST", "GET"],
)
async def test_transport_outage_stops_after_one_post_and_get_pair(self):
client = self._make_client()
client._request = AsyncMock(return_value=None)
client.fetch_inbox_messages = AsyncMock(side_effect=RuntimeError("boom"))
with self.assertLogs("douyin_im.http", level="WARNING"):
self.assertEqual(await client.get_conversations(), [])
self.assertEqual(client._request.await_count, 2)
self.assertIn("boom", client.last_error)
async def test_inbox_messages_group_into_one_row_per_conversation(self):
client = self._make_client()
client.fetch_inbox_messages = AsyncMock(
return_value=[
{
"conversation_id": "0:1:10001:20001",
"server_message_id": "700",
"conversation_short_id": "555",
"message_type": 7,
"sender": "20001",
"content": '{"text":"旧"}',
},
{
"conversation_id": "0:1:10001:20001",
"server_message_id": "900",
"conversation_short_id": "555",
"message_type": 7,
"sender": "20001",
"content": '{"text":"新"}',
},
{
"conversation_id": "0:1:10001:20002",
"server_message_id": "800",
"message_type": 7,
"sender": "20002",
"content": '{"text":"另一个"}',
},
]
)
rows = await client.get_conversations(enrich_profiles=False)
by_id = {r["conversation_id"]: r for r in rows}
self.assertEqual(len(rows), 2)
# 同一会话只保留 server_message_id 最大的那条
self.assertEqual(by_id["0:1:10001:20001"]["server_message_id"], "900")
self.assertIn("新", by_id["0:1:10001:20001"]["content"])
# peer_uid 由 conversation_id 推导，不能直接取 sender（可能是自己）
self.assertEqual(by_id["0:1:10001:20002"]["peer_uid"], "20002")
# 顺手缓存 short_id，发送时就不必再 create 一次会话
self.assertEqual(
[call.args[0] for call in client._request.await_args_list],
["POST", "GET"],
client.session.conv_meta["0:1:10001:20001"]["conversation_short_id"],
"555",
)
def test_websocket_reconciliation_is_slow_and_http_fallback_stays_fast(self):
@@ -156,6 +158,7 @@ class ConversationPollBandwidthTests(unittest.IsolatedAsyncioTestCase):
class _HttpClient:
def __init__(self):
self.get_conversations = AsyncMock(return_value=[])
self.conversation_list_unsupported = False
async def __aenter__(self):
return self
@@ -254,6 +257,7 @@ class ConversationPollBandwidthTests(unittest.IsolatedAsyncioTestCase):
class _HttpClient:
def __init__(self):
self.get_conversations = AsyncMock(side_effect=snapshots)
self.conversation_list_unsupported = False
self.enter_count = 0
self.exit_count = 0
backend/tests/test_egress_channels.py
+10
View File
@@ -8,6 +8,8 @@ from pathlib import Path
from unittest.mock import AsyncMock, patch
from sqlalchemy import create_engine, inspect, text
from sqlalchemy.dialects import mysql
from sqlalchemy.schema import CreateTable
BACKEND_DIR = Path(__file__).resolve().parents[1]
@@ -24,6 +26,7 @@ from rpa_engine.egress_channels import (
resolve_send_channels,
)
from models.db_migrate import migrate_accounts_table
from models.models import Account
class EgressChannelTests(unittest.IsolatedAsyncioTestCase):
@@ -98,6 +101,13 @@ class EgressChannelTests(unittest.IsolatedAsyncioTestCase):
class EgressMigrationTests(unittest.TestCase):
def test_mysql_accounts_uses_longtext_for_browser_payloads(self):
ddl = str(CreateTable(Account.__table__).compile(dialect=mysql.dialect()))
self.assertIn("cookie_data LONGTEXT", ddl)
self.assertIn("im_session_data LONGTEXT", ddl)
self.assertIn("qr_code_base64 LONGTEXT", ddl)
def test_old_accounts_table_receives_egress_columns(self):
engine = create_engine("sqlite:///:memory:")
with engine.begin() as connection:
backend/tests/test_im_receive_path.py
+711
View File
@@ -0,0 +1,711 @@
"""接收私信链路的回归测试。
覆盖三个曾让「托管中收不到抖音下发的私信」的缺陷：
1. frontier 长连接地址用了账号 UID 而不是设备号，握手成功却订阅错地址；
2. 浏览器本次登录抓到的真实 frontier 地址被 DB 里的旧地址挤掉；
3. PushFrame 负载是 gzip / payloadType 不是 'pb' 时整帧被丢弃。
另外覆盖会话列表接口被抖音拒绝时不能再伪装成「收件箱为空」。
"""
from __future__ import annotations
import gzip
import json
import os
import sys
import unittest
from datetime import datetime
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
BACKEND_DIR = Path(__file__).resolve().parents[1]
os.environ.setdefault("KEFU_DB_TYPE", "sqlite")
os.environ.setdefault("KEFU_DATABASE_URL", "")
os.environ.setdefault("KEFU_DB_PATH", str(BACKEND_DIR / "kefu.db"))
if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im import frontier as frontier_module
from rpa_engine.douyin_im.auth import DouyinAuth
from rpa_engine.douyin_im.frontier import ensure_frontier_ws
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
from rpa_engine.douyin_im.protocol import parse_ws_payload
from rpa_engine.douyin_im.session import DouyinImSession
from rpa_engine.douyin_im.static import Live_pb2, Response_pb2
from rpa_engine.playwright_worker import DouyinWorker
def _build_push_frame(
*,
conversation_id: str,
sender: int,
content: str,
message_type: int = 7,
server_message_id: int = 7665317099296081465,
encoding: str = "",
payload_type: str = "pb",
) -> bytes:
response = Response_pb2.Response()
notify = response.body.new_message_notify
notify.conversation_id = conversation_id
message = notify.message
message.conversation_id = conversation_id
message.conversation_type = 1
message.server_message_id = server_message_id
message.message_type = message_type
message.sender = sender
message.content = content
body = response.SerializeToString()
if encoding == "gzip":
body = gzip.compress(body)
frame = Live_pb2.PushFrame()
frame.seqId = 1
frame.service = 6
frame.method = 2
frame.payloadEncoding = encoding
frame.payloadType = payload_type
frame.payload = body
return frame.SerializeToString()
class FrontierAddressTests(unittest.TestCase):
"""frontier 按 device_id 寻址，不能用账号 UID 顶替。"""
def _session(self) -> DouyinImSession:
return DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
# query/user 返回的 id：本浏览器的设备注册号
device_id="7678285795559818786",
web_id="7678286623234475535",
my_uid=2609567359568155,
uid_verified=True,
)
def test_built_url_uses_device_id_not_account_uid(self):
session = self._session()
url = ensure_frontier_ws(session)
self.assertIsNotNone(url)
self.assertIn("device_id=7678285795559818786", url)
self.assertNotIn("device_id=2609567359568155", url)
def test_missing_device_id_falls_back_to_query_user_lookup(self):
session = self._session()
session.device_id = ""
session.web_id = ""
with patch.object(
frontier_module,
"fetch_device_id",
return_value="7678285795559818786",
) as fetch:
url = ensure_frontier_ws(session)
fetch.assert_called_once()
self.assertIn("device_id=7678285795559818786", url)
def test_proto_auth_keeps_device_id_for_verified_uid(self):
session = self._session()
auth = DouyinAuth.from_im_session(session)
self.assertEqual(auth.device_id, "7678285795559818786")
class CapturedFrontierUrlTests(unittest.IsolatedAsyncioTestCase):
"""浏览器本次抓到的真实地址必须压过 DB 里的旧地址。"""
async def test_fresh_browser_ws_url_wins_over_cached_url(self):
cached = (
"wss://frontier-im.douyin.com/ws/v2?aid=6383&device_platform=douyin_pc"
"&fpid=9&device_id=7678285795559818786&token=stale&access_key=stale"
)
captured = (
"wss://frontier31-normal.zijieapi.com/ws/v2?aid=6383&fpid=9"
"&device_id=7678285795559818786&access_key=realkey&token=realtoken"
)
saved = DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
ws_urls=[cached],
device_id="7678285795559818786",
my_uid=2609567359568155,
)
row = SimpleNamespace(
im_session_data=json.dumps(saved.to_dict()),
cookie_updated_at=datetime.utcnow(),
uid=None,
profile_updated_at=None,
)
result = MagicMock()
result.one_or_none.return_value = row
db = SimpleNamespace(execute=AsyncMock(return_value=result), close=AsyncMock())
worker = DouyinWorker(account_id=400, login_mode="browser")
worker.get_db = AsyncMock(return_value=db)
worker._load_raw_user_agent = AsyncMock(return_value="test-agent")
session = await worker._build_im_session_from_storage(
{"cookies": [{"name": "sessionid", "value": "6313fec013ec0000000000000000abcd"}]},
{"ws_urls": [captured]},
)
self.assertEqual(session.ws_urls[0], captured)
self.assertEqual(ensure_frontier_ws(session), captured)
class PushFramePayloadTests(unittest.TestCase):
"""PushFrame 负载的编码/类型不能再决定整帧被不被丢弃。"""
def test_plain_protobuf_frame_is_parsed(self):
raw = _build_push_frame(
conversation_id="0:1:869032150442612:2609567359568155",
sender=869032150442612,
content=json.dumps({"text": "你好", "aweType": 700}, ensure_ascii=False),
)
messages = parse_ws_payload(raw)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]["sender_uid"], "869032150442612")
self.assertEqual(
messages[0]["conversation_id"], "0:1:869032150442612:2609567359568155"
)
def test_gzip_encoded_frame_is_parsed(self):
raw = _build_push_frame(
conversation_id="0:1:869032150442612:2609567359568155",
sender=869032150442612,
content=json.dumps({"text": "在吗", "aweType": 700}, ensure_ascii=False),
encoding="gzip",
)
messages = parse_ws_payload(raw)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]["sender_uid"], "869032150442612")
def test_non_pb_payload_type_is_still_parsed(self):
# 现网 frontier 帧会带 payloadType='text/json'；只认 'pb' 会整帧丢弃。
raw = _build_push_frame(
conversation_id="0:1:869032150442612:2609567359568155",
sender=869032150442612,
content=json.dumps({"text": "在吗", "aweType": 700}, ensure_ascii=False),
payload_type="text/json",
)
messages = parse_ws_payload(raw)
self.assertEqual(len(messages), 1)
self.assertEqual(messages[0]["sender_uid"], "869032150442612")
def test_empty_payload_control_frame_yields_no_message(self):
frame = Live_pb2.PushFrame()
frame.service = 6
frame.method = 2
frame.payloadEncoding = "utf-8"
frame.payloadType = "text/json"
self.assertEqual(parse_ws_payload(frame.SerializeToString()), [])
class InboxProtobufTests(unittest.TestCase):
"""imapi 只认 protobuf：解析真实响应形状，而不是 JSON。"""
@staticmethod
def _response(*, cmd=200, status=0, message="OK", messages=()):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_msg, _pb_str
entries = b""
for m in messages:
entries += _pb_msg(
1,
_pb_str(1, m["conversation_id"])
+ _pb_int(3, m["server_message_id"])
+ _pb_int(5, m.get("conversation_short_id", 0))
+ _pb_int(6, m.get("message_type", 7))
+ _pb_int(7, m["sender"])
+ _pb_str(8, m.get("content", "")),
)
return (
_pb_int(1, cmd)
+ _pb_int(3, status)
+ _pb_str(4, message)
+ _pb_msg(6, _pb_msg(cmd, entries))
)
def test_status_is_read_from_the_protobuf_envelope(self):
from rpa_engine.douyin_im.http_client import _pb_response_status
ok = self._response()
self.assertEqual(_pb_response_status(ok), (0, "OK"))
rejected = self._response(status=1, message="unexepcted session length")
self.assertEqual(
_pb_response_status(rejected),
(1, "unexepcted session length"),
)
def test_message_bodies_are_extracted_from_the_inbox_response(self):
from rpa_engine.douyin_im.http_client import _pb_parse_inbox_messages
raw = self._response(
messages=[
{
"conversation_id": "0:1:2609567359568155:869032150442612",
"server_message_id": 7678140298052355621,
"conversation_short_id": 7654765893796266545,
"message_type": 7,
"sender": 869032150442612,
"content": '{"text":"你好"}',
}
]
)
parsed = _pb_parse_inbox_messages(raw, 200)
self.assertEqual(len(parsed), 1)
self.assertEqual(
parsed[0]["conversation_id"],
"0:1:2609567359568155:869032150442612",
)
self.assertEqual(parsed[0]["server_message_id"], "7678140298052355621")
self.assertEqual(parsed[0]["sender"], "869032150442612")
self.assertIn("你好", parsed[0]["content"])
def test_unrelated_protobuf_is_not_mistaken_for_a_message(self):
from rpa_engine.douyin_im.http_client import (
_pb_int, _pb_msg, _pb_parse_inbox_messages, _pb_str,
)
# 一段带字符串字段 1 但不是 conversation_id 的子消息
noise = _pb_msg(6, _pb_msg(200, _pb_msg(1, _pb_str(1, "not-a-conv") + _pb_int(3, 5))))
self.assertEqual(_pb_parse_inbox_messages(noise, 200), [])
def test_empty_inbox_yields_no_messages(self):
from rpa_engine.douyin_im.http_client import _pb_parse_inbox_messages
self.assertEqual(_pb_parse_inbox_messages(self._response(), 200), [])
class InboxCursorAndListTests(unittest.IsolatedAsyncioTestCase):
"""轮询用小窗口，用户点开列表用全量——同一个 cmd，只是游标不同。"""
def _client(self) -> DouyinImHttpClient:
session = DouyinImSession(
cookies={"sessionid": "s", "x_tt_token": "00" + "a" * 353},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
return DouyinImHttpClient(session, account_id=405)
@staticmethod
def _cursor_from_payload(payload: bytes) -> int:
from rpa_engine.douyin_im.http_client import _pb_parse_fields
for fn, wt, val in _pb_parse_fields(payload):
if fn != 8 or wt != 2:
continue
for bfn, bwt, bval in _pb_parse_fields(val):
if bfn != 200 or bwt != 2:
continue
for cfn, cwt, cval in _pb_parse_fields(bval):
if cfn == 1 and cwt == 0:
return int(cval)
return -1
async def _capture_cursor(self, **kwargs) -> int:
from rpa_engine.douyin_im.http_client import _pb_int, _pb_str
client = self._client()
captured: dict = {}
async def fake_post(url, auth, payload, **_kw):
captured["payload"] = payload
return SimpleNamespace(
content=_pb_int(1, 200) + _pb_int(3, 0) + _pb_str(4, "OK"),
raise_for_status=lambda: None,
)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(**kwargs)
return self._cursor_from_payload(captured["payload"])
async def test_poll_window_sends_a_recent_microsecond_cursor(self):
import time as _time
cursor = await self._capture_cursor(lookback_seconds=1800)
now_us = int(_time.time() * 1_000_000)
self.assertGreater(cursor, 0)
# 游标应落在「大约半小时前」，允许几秒误差
self.assertLess(now_us - cursor, int(1810 * 1_000_000))
self.assertGreater(now_us - cursor, int(1790 * 1_000_000))
async def test_zero_lookback_means_no_cursor_not_now(self):
# lookback=0 若被算成 now，就只要「比此刻更新」的消息，永远是空列表。
self.assertEqual(await self._capture_cursor(lookback_seconds=0), 0)
@staticmethod
def _page(*, entries=(), next_cursor=0, has_more=False, cmd=200):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_msg, _pb_str
inner = b""
for short_id, conv_id in entries:
inner += _pb_msg(6, _pb_int(1, short_id) + _pb_str(4, conv_id))
inner += _pb_int(2, next_cursor) + _pb_int(3, 1 if has_more else 0)
return (
_pb_int(1, cmd)
+ _pb_int(3, 0)
+ _pb_str(4, "OK")
+ _pb_msg(6, _pb_msg(cmd, inner))
)
async def test_paging_follows_the_cursor_and_dedupes_conversations(self):
client = self._client()
pages = [
self._page(
entries=[(1, "0:1:10001:20001"), (2, "0:1:10001:20002")],
next_cursor=111,
has_more=True,
),
self._page(
# 第二页重复一个、新增一个
entries=[(2, "0:1:10001:20002"), (3, "0:1:10001:20003")],
next_cursor=222,
has_more=True,
),
]
cursors: list[int] = []
async def fake_post(url, auth, payload, **_kw):
cursors.append(self._cursor_from_payload(payload))
return SimpleNamespace(
content=pages[len(cursors) - 1], raise_for_status=lambda: None
)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(lookback_seconds=0, max_pages=2)
self.assertEqual(cursors, [0, 111])
self.assertEqual(
[c["conversation_id"] for c in client._last_inbox_conversations],
["0:1:10001:20001", "0:1:10001:20002", "0:1:10001:20003"],
)
# 预算用完但抖音还说 has_more：必须承认列表不完整
self.assertTrue(client.inbox_truncated)
async def test_last_page_is_not_reported_as_truncated(self):
client = self._client()
page = self._page(entries=[(1, "0:1:10001:20001")], has_more=False)
async def fake_post(url, auth, payload, **_kw):
return SimpleNamespace(content=page, raise_for_status=lambda: None)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(lookback_seconds=0, max_pages=5)
self.assertFalse(client.inbox_truncated)
async def test_a_stalled_cursor_stops_paging(self):
client = self._client()
# 抖音回 has_more=1 但游标不前进：不能无限翻同一页
page = self._page(
entries=[(1, "0:1:10001:20001")], next_cursor=0, has_more=True
)
calls = {"n": 0}
async def fake_post(url, auth, payload, **_kw):
calls["n"] += 1
return SimpleNamespace(content=page, raise_for_status=lambda: None)
with patch.object(client, "_post_protobuf", fake_post):
await client.fetch_inbox_messages(lookback_seconds=0, max_pages=10)
self.assertEqual(calls["n"], 1)
async def test_conversations_without_recent_messages_still_listed(self):
client = self._client()
client.fetch_inbox_messages = AsyncMock(return_value=[])
client._last_inbox_conversations = [
{"conversation_id": "0:1:10001:20001", "conversation_short_id": "555"},
{"conversation_id": "0:1:10001:20002", "conversation_short_id": "666"},
]
rows = await client.get_conversations(
enrich_profiles=False, lookback_seconds=0
)
self.assertEqual(
{r["conversation_id"] for r in rows},
{"0:1:10001:20001", "0:1:10001:20002"},
)
self.assertEqual(
client.session.conv_meta["0:1:10001:20002"]["conversation_short_id"],
"666",
)
async def test_control_frames_never_become_a_conversation_preview(self):
client = self._client()
client.fetch_inbox_messages = AsyncMock(
return_value=[
{
"conversation_id": "0:1:10001:20001",
"server_message_id": "100",
"message_type": 7,
"sender": "20001",
"content": '{"text":"真实消息"}',
},
{
"conversation_id": "0:1:10001:20001",
"server_message_id": "200",
"message_type": 50001,
"sender": "20001",
"content": '{"command_type":6,"conversation_id":"0:1:10001:20001"}',
},
]
)
rows = await client.get_conversations(enrich_profiles=False)
# 控制帧 server_message_id 更大，但不能顶掉真实消息成为预览，
# 否则 _handle_incoming 会拿它去匹配自动回复。
self.assertEqual(len(rows), 1)
self.assertIn("真实消息", rows[0]["content"])
self.assertEqual(rows[0]["server_message_id"], "100")
class ReadRequestTokenTests(unittest.TestCase):
"""读接口的 Request.token 必须是 x_tt_token。
带 auth.ticket 时抖音照样回 status_code=0 "OK"，但把调用方当匿名用户，
正文恒为空——和「收件箱没有消息」完全无法区分，是最难发现的那类故障。
实测同一请求只换 token：auth.ticket 73 字节 0 条，x_tt_token 113KB 47 条。
"""
def _auth(self):
session = DouyinImSession(
cookies={
"sessionid": "6313fec013ec0000000000000000abcd",
"x_tt_token": "00" + "a" * 353,
},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
return DouyinAuth.from_im_session(session)
def test_read_request_uses_x_tt_token(self):
from rpa_engine.douyin_im.proto_builder import ProtoBuilder
auth = self._auth()
request = ProtoBuilder.build_read_request(auth, 200)
self.assertEqual(request.token, "00" + "a" * 353)
self.assertNotEqual(request.token, auth.ticket)
def test_read_request_keeps_ticket_when_cookie_missing(self):
from rpa_engine.douyin_im.proto_builder import ProtoBuilder
session = DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
auth = DouyinAuth.from_im_session(session)
request = ProtoBuilder.build_read_request(auth, 200)
self.assertEqual(request.token, auth.ticket or "")
def test_send_request_is_left_on_the_normal_envelope(self):
from rpa_engine.douyin_im.proto_builder import ProtoBuilder
# 发送接口另有 bd-ticket-guard 签名且线上可用，不能顺手改掉它的 token。
auth = self._auth()
request = ProtoBuilder.build_normal_request(auth, 100)
self.assertEqual(request.token, auth.ticket or "")
class AuthoritativeUidTests(unittest.TestCase):
"""imapi 响应字段 13 是抖音认定的本账号 IM uid。"""
def test_response_uid_corrects_a_wrong_my_uid(self):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_str
session = DouyinImSession(
cookies={"sessionid": "s"},
my_uid=7678285795559818786, # 误把 frontier 设备号当成了 IM uid
)
client = DouyinImHttpClient(session, account_id=404)
content = (
_pb_int(1, 200)
+ _pb_int(3, 0)
+ _pb_str(4, "OK")
+ _pb_int(13, 2609567359568155)
)
client._adopt_authoritative_uid(content)
self.assertEqual(session.my_uid, 2609567359568155)
self.assertTrue(session.uid_verified)
def test_matching_uid_is_left_alone(self):
from rpa_engine.douyin_im.http_client import _pb_int, _pb_str
session = DouyinImSession(cookies={"sessionid": "s"}, my_uid=2609567359568155)
client = DouyinImHttpClient(session, account_id=404)
content = _pb_int(1, 200) + _pb_int(3, 0) + _pb_str(4, "OK") + _pb_int(
13, 2609567359568155
)
client._adopt_authoritative_uid(content)
self.assertEqual(session.my_uid, 2609567359568155)
self.assertFalse(session.uid_verified)
class ConversationListRejectionTests(unittest.IsolatedAsyncioTestCase):
"""接口被拒不能再伪装成「收件箱为空」。"""
def _client(self) -> DouyinImHttpClient:
session = DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
device_id="7678285795559818786",
my_uid=2609567359568155,
)
return DouyinImHttpClient(session, account_id=401)
async def test_rejected_protobuf_response_is_reported(self):
client = self._client()
raw = InboxProtobufTests._response(
status=1, message="unexepcted session length"
)
post = AsyncMock(
return_value=SimpleNamespace(
content=raw, raise_for_status=lambda: None
)
)
with (
patch.object(client, "_post_protobuf", post),
patch(
"rpa_engine.douyin_im.auth.DouyinAuth.from_im_session",
return_value=SimpleNamespace(source_ip=""),
),
patch(
"rpa_engine.douyin_im.proto_builder.ProtoBuilder.build_normal_request",
return_value=SimpleNamespace(SerializeToString=lambda: b""),
),
patch.object(client, "_report_conversation_list_rejected") as report,
):
self.assertEqual(await client.fetch_inbox_messages(), [])
post.assert_awaited_once()
report.assert_called_once_with("unexepcted session length")
async def test_rejection_marks_the_endpoint_unsupported(self):
client = self._client()
client._report_conversation_list_rejected("unexepcted session length")
self.assertTrue(client.conversation_list_unsupported)
self.assertIn("unexepcted session length", client.last_error)
async def test_empty_but_successful_inbox_is_not_reported_as_failure(self):
client = self._client()
client.fetch_inbox_messages = AsyncMock(return_value=[])
with patch.object(
client, "_report_conversation_list_rejected"
) as report:
self.assertEqual(
await client.get_conversations(enrich_profiles=False), []
)
report.assert_not_called()
self.assertEqual(client.last_error, "")
self.assertFalse(client.conversation_list_unsupported)
class LoggedOutDetectionTests(unittest.IsolatedAsyncioTestCase):
"""抖音回「用户未登录」时必须明确报出来，不能当成资料接口抖动。"""
def test_status_code_8_is_reported_as_logged_out(self):
from rpa_engine import account_profile as ap
auth = SimpleNamespace(cookie={}, msToken="t", get_uid=lambda: "938334054809296")
payloads = [
{"status_code": 0, "user_uid": "938334054809296"},
{"status_code": 8, "status_msg": "用户未登录", "user": None},
{"status_code": 8, "status_msg": "用户未登录", "user": None},
]
responses = [SimpleNamespace(json=lambda v=v: v) for v in payloads]
with (
patch.object(ap, "_build_auth", return_value=(auth, "ua")),
patch.object(ap.requests, "get", side_effect=responses),
patch.object(ap, "generate_a_bogus", return_value="a-bogus"),
patch.object(ap, "generate_webid", return_value="web-id"),
patch.object(ap, "_requests_proxies", return_value=None),
):
detail = ap.fetch_douyin_profile_detail_sync("cookie-json", "ua")
self.assertTrue(detail["logged_out"])
self.assertFalse(detail["fetched"])
self.assertIn("用户未登录", detail["message"])
async def test_hosting_reports_logged_out_once(self):
worker = DouyinWorker(account_id=403, login_mode="im_direct")
with patch(
"rpa_engine.playwright_worker.system_logger.record"
) as record:
await worker._report_douyin_logged_out("抖音返回「用户未登录」")
await worker._report_douyin_logged_out("抖音返回「用户未登录」")
record.assert_called_once()
self.assertEqual(record.call_args.kwargs["level"], "error")
class ReconciliationBackoffTests(unittest.IsolatedAsyncioTestCase):
"""被抖音拒绝过的接口不能每 120 秒再白打一次。"""
def _service(self):
from rpa_engine.douyin_im.service import DouyinImService
return DouyinImService(
session=DouyinImSession(
cookies={"sessionid": "6313fec013ec0000000000000000abcd"},
device_id="7678285795559818786",
my_uid=2609567359568155,
),
match_reply=AsyncMock(return_value=[]),
log_fn=AsyncMock(),
account_id=402,
)
async def test_second_poll_skips_a_rejected_endpoint(self):
service = self._service()
client = MagicMock()
client.get_conversations = AsyncMock(return_value=[])
client.conversation_list_unsupported = True
client.__aenter__ = AsyncMock(return_value=client)
client.__aexit__ = AsyncMock(return_value=False)
with patch(
"rpa_engine.douyin_im.service.DouyinImHttpClient",
return_value=client,
):
self.assertEqual(await service._poll_conversations(), [])
self.assertTrue(service._conversation_list_unsupported)
# 第二轮完全不再构造 HTTP 客户端 / 发请求
self.assertEqual(await service._poll_conversations(), [])
client.get_conversations.assert_awaited_once()
if __name__ == "__main__":
unittest.main()
backend/tests/test_sec_user_id_guard.py
+273 -34
View File
@@ -1,9 +1,11 @@
from __future__ import annotations
import json
import os
import sys
import unittest
from contextlib import asynccontextmanager
from datetime import datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
@@ -18,13 +20,114 @@ if str(BACKEND_DIR) not in sys.path:
from rpa_engine.douyin_im.session import DouyinImSession
from rpa_engine import account_profile as account_profile_module
from rpa_engine.credential import build_im_session_from_storage
from rpa_engine.playwright_worker import DouyinWorker
class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
async def test_im_direct_missing_sec_user_id_exits_before_online_state(self):
async def test_current_profile_uid_overrides_browser_runtime_uid(self):
now = datetime.utcnow()
row = SimpleNamespace(
im_session_data=None,
cookie_updated_at=now,
uid="2609567359568155",
profile_updated_at=now,
)
result = MagicMock()
result.one_or_none.return_value = row
db = SimpleNamespace(
execute=AsyncMock(return_value=result),
close=AsyncMock(),
)
worker = DouyinWorker(account_id=300, login_mode="im_direct")
worker.get_db = AsyncMock(return_value=db)
worker._load_raw_user_agent = AsyncMock(return_value="test-agent")
storage = {
"cookies": [{"name": "sessionid", "value": "test-session"}],
"my_uid": 7678285795559818786,
"origins": [
{
"localStorage": [
{
"name": "web_runtime_security_uid",
"value": "7678285795559818786",
}
]
}
],
}
session = await worker._build_im_session_from_storage(storage)
self.assertEqual(session.my_uid, 2609567359568155)
# device_id 必须与权威 UID 同步（protobuf 发送时 device_id 优先取
# session.device_id，残留的浏览器采集值会导致 device_id != my_uid -> KICK）
self.assertEqual(session.device_id, "2609567359568155")
self.assertTrue(session.uid_verified)
db.close.assert_awaited_once()
async def test_stale_profile_uid_does_not_override_new_cookie(self):
now = datetime.utcnow()
row = SimpleNamespace(
im_session_data=None,
cookie_updated_at=now,
uid="2609567359568155",
profile_updated_at=now - timedelta(seconds=1),
)
result = MagicMock()
result.one_or_none.return_value = row
db = SimpleNamespace(
execute=AsyncMock(return_value=result),
close=AsyncMock(),
)
worker = DouyinWorker(account_id=300, login_mode="im_direct")
worker.get_db = AsyncMock(return_value=db)
worker._load_raw_user_agent = AsyncMock(return_value="test-agent")
session = await worker._build_im_session_from_storage(
{
"cookies": [{"name": "sessionid", "value": "test-session"}],
"my_uid": 7678285795559818786,
}
)
self.assertEqual(session.my_uid, 7678285795559818786)
self.assertFalse(session.uid_verified)
def test_persisted_verified_uid_wins_in_generic_session_builder(self):
saved = DouyinImSession(
cookies={"sessionid": "test-session"},
my_uid=2609567359568155,
device_id="7678285795559818786",
uid_verified=True,
)
session = build_im_session_from_storage(
{
"cookies": [{"name": "sessionid", "value": "test-session"}],
"my_uid": 7678285795559818786,
"origins": [
{
"localStorage": [
{
"name": "web_runtime_security_uid",
"value": "7678285795559818786",
}
]
}
],
},
json.dumps(saved.to_dict()),
)
self.assertEqual(session.my_uid, 2609567359568155)
# device_id 同步为已核验 UID，避免凭证残留设备号导致 device_id != my_uid
self.assertEqual(session.device_id, "2609567359568155")
self.assertTrue(session.uid_verified)
async def test_im_direct_missing_sec_user_id_continues_im_hosting(self):
worker = DouyinWorker(account_id=301, login_mode="im_direct")
worker._require_sec_user_id = AsyncMock(return_value=False)
worker._best_effort_sec_user_id = AsyncMock(return_value="")
worker._load_user_agent = AsyncMock(return_value="test-agent")
im_session = DouyinImSession(
cookies={"sessionid": "test-session"},
@@ -42,37 +145,34 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
{"cookies": [{"name": "sessionid", "value": "test-session"}]}
)
self.assertFalse(started)
self.assertIn("sec_user_id", reason)
worker._require_sec_user_id.assert_awaited_once()
require_call = worker._require_sec_user_id.await_args
self.assertTrue(require_call.kwargs["refresh_if_missing"])
self.assertTrue(require_call.kwargs["refresh_if_stale"])
self.assertTrue(started)
self.assertEqual(reason, "")
worker._best_effort_sec_user_id.assert_awaited_once_with(
refresh_if_missing=True,
refresh_if_stale=True,
)
worker._build_im_session_from_storage.assert_awaited_once()
validate.assert_awaited_once_with(im_session)
worker._persist_im_session.assert_not_awaited()
worker._run_im_direct_service.assert_not_awaited()
worker._persist_im_session.assert_awaited_once()
worker._run_im_direct_service.assert_awaited_once_with(im_session)
async def test_running_guard_precedes_disabled_follow_welcome_setting(self):
async def test_disabled_follow_welcome_does_not_require_identity(self):
worker = DouyinWorker(account_id=302, login_mode="im_direct")
worker.is_running = True
worker._im_service = SimpleNamespace(_running=True)
worker._require_sec_user_id = AsyncMock(return_value=False)
worker._refresh_follow_welcome_config = AsyncMock()
worker._best_effort_sec_user_id = AsyncMock()
worker._refresh_follow_welcome_config = AsyncMock(
return_value=(False, "", "")
)
worker.get_db = AsyncMock()
await worker.follow_welcome_tick()
worker._require_sec_user_id.assert_awaited_once()
require_call = worker._require_sec_user_id.await_args
self.assertFalse(require_call.kwargs.get("refresh_if_missing", False))
# The identity guard must run before the account's follow-welcome flag
# is queried. Otherwise accounts with that feature disabled could stay
# hosted indefinitely without a sec_user_id.
worker._refresh_follow_welcome_config.assert_not_awaited()
worker._refresh_follow_welcome_config.assert_awaited_once_with()
worker._best_effort_sec_user_id.assert_not_awaited()
worker.get_db.assert_not_awaited()
async def test_cached_disabled_follow_setting_still_guards_missing_identity(self):
async def test_cached_disabled_follow_setting_skips_missing_identity(self):
worker = DouyinWorker(account_id=311, login_mode="im_direct")
worker.is_running = True
worker._im_service = SimpleNamespace(_running=True)
@@ -80,12 +180,12 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
worker._refresh_follow_welcome_config = AsyncMock(
return_value=(False, "", "")
)
worker._require_sec_user_id = AsyncMock(return_value="")
worker._best_effort_sec_user_id = AsyncMock(return_value="")
await worker.follow_welcome_tick()
worker._refresh_follow_welcome_config.assert_awaited_once_with()
worker._require_sec_user_id.assert_awaited_once_with("托管运行中")
worker._best_effort_sec_user_id.assert_not_awaited()
async def test_blank_sec_user_id_is_missing_and_stops_hosting(self):
worker = DouyinWorker(account_id=303, login_mode="im_direct")
@@ -297,6 +397,141 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
self.assertIn("托管已自动退出", status_call.kwargs["error_msg"])
record_system_log.assert_called_once()
def test_profile_payload_uid_outranks_query_user_and_cookie_uid(self):
"""资料接口的 UID 必须压过 query/user 的 user_uid 和 cookie 兜底。
实测同一个 Cookie：query/user 返回 user_uid=938334054809296，
而账号资料 UID 是 2609567359568155。以前前者先占位，导致账号卡片
一直显示「用户938334054809296」，反查 sec_user_id 也永远失败。
"""
auth = SimpleNamespace(
cookie={},
msToken="test-ms-token",
get_uid=lambda: "938334054809296",
)
payloads = [
# 1) query/user：只有 user_uid，属于弱兜底
{"status_code": 0, "user_uid": "938334054809296"},
# 2) user/profile/self：权威账号资料
{
"status_code": 0,
"user": {
"uid": "2609567359568155",
"nickname": "凤的心",
"sec_uid": "MS4wLjABAAAA-real-sec-user-id",
"unique_id": "39688250979",
"aweme_count": 50,
"follower_count": 154,
},
},
]
responses = [SimpleNamespace(json=lambda value=value: value) for value in payloads]
with (
patch.object(
account_profile_module, "_build_auth",
return_value=(auth, "test-agent"),
),
patch.object(
account_profile_module.requests, "get", side_effect=responses,
),
patch.object(
account_profile_module, "generate_a_bogus", return_value="a-bogus",
),
patch.object(
account_profile_module, "generate_webid", return_value="web-id",
),
patch.object(
account_profile_module, "_requests_proxies", return_value=None,
),
):
detail = account_profile_module.fetch_douyin_profile_detail_sync(
"cookie-json", "test-agent",
)
self.assertEqual(detail["uid"], "2609567359568155")
self.assertEqual(detail["nickname"], "凤的心")
self.assertTrue(detail["fetched"])
async def test_sync_writes_one_identity_to_both_account_and_profile(self):
"""卡片与详细资料必须来自同一次抓取，不能各抓一次抓出两个身份。"""
detail = {
"uid": "2609567359568155",
"nickname": "凤的心",
"avatar_url": "https://example.test/avatar.png",
"unique_id": "39688250979",
"signature": "",
"sec_user_id": "MS4wLjABAAAA-real-sec-user-id",
"sec_user_id_status": "found",
"video_count": 50,
"follower_count": 154,
"following_count": 162,
"total_favorited": 612,
"favoriting_count": 0,
"fetched": True,
"message": "",
}
account = SimpleNamespace(
id=1,
username="用户938334054809296",
douyin_uid="938334054809296",
avatar_url=None,
user_agent="test-agent",
)
profile = SimpleNamespace(
account_id=1, uid=None, nickname=None, avatar_url=None,
unique_id=None, signature=None, sec_user_id=None,
follower_count=None, following_count=None, total_favorited=None,
favoriting_count=None, video_count=None, synced_at=None,
sync_message=None,
)
profile_result = MagicMock()
profile_result.scalar_one_or_none.return_value = profile
username_result = MagicMock()
username_result.scalar_one_or_none.return_value = None
db = SimpleNamespace(
execute=AsyncMock(side_effect=lambda stmt: (
username_result if "accounts.username" in str(stmt).lower()
or "username" in str(stmt).lower() else profile_result
)),
add=MagicMock(),
commit=AsyncMock(),
refresh=AsyncMock(),
)
with (
patch.object(
account_profile_module,
"fetch_douyin_profile_detail_with_sec_user_id",
new=AsyncMock(return_value=detail),
),
patch.object(
account_profile_module,
"fetch_douyin_user_videos",
new=AsyncMock(return_value={"videos": [], "message": "无作品"}),
),
patch.object(
account_profile_module,
"fetch_douyin_profile",
new=AsyncMock(side_effect=AssertionError("must not re-fetch")),
),
patch.object(
account_profile_module,
"load_account_profile_from_db",
new=AsyncMock(return_value={}),
),
):
await account_profile_module.sync_account_profile_to_db(
db, account, "cookie-json",
)
# 同一个身份同时写进 accounts 行和 account_profile_details 行
self.assertEqual(account.douyin_uid, "2609567359568155")
self.assertEqual(account.username, "凤的心")
self.assertEqual(account.avatar_url, "https://example.test/avatar.png")
self.assertEqual(profile.uid, "2609567359568155")
self.assertEqual(profile.nickname, "凤的心")
async def test_cookie_uid_without_valid_profile_payload_stays_unknown(self):
auth = SimpleNamespace(
cookie={},
@@ -490,7 +725,7 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
),
patch.object(
account_profile_module,
"apply_douyin_profile",
"apply_profile_to_account",
apply_profile,
),
patch.object(
@@ -518,19 +753,19 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
"profile endpoint temporarily unavailable",
)
async def test_browser_login_missing_sec_user_id_closes_browser_and_skips_im(self):
async def test_browser_login_missing_sec_user_id_still_starts_im(self):
worker = DouyinWorker(account_id=305, login_mode="browser")
worker.is_running = True
events: list[str] = []
async def reject_identity(*_args, **_kwargs):
events.append("require-sec-user-id")
return False
return ""
worker._load_user_agent = AsyncMock(return_value="test-agent")
worker._probe_existing_login = AsyncMock(return_value=True)
worker._finalize_login_session = AsyncMock()
worker._require_sec_user_id = AsyncMock(side_effect=reject_identity)
worker._best_effort_sec_user_id = AsyncMock(side_effect=reject_identity)
worker._setup_im_network_listener = AsyncMock()
worker._navigate_to_message_center = AsyncMock()
worker._harvest_im_credentials = AsyncMock()
@@ -595,8 +830,8 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
)
worker._finalize_login_session.assert_awaited_once_with()
worker._require_sec_user_id.assert_awaited_once()
require_call = worker._require_sec_user_id.await_args
worker._best_effort_sec_user_id.assert_awaited_once()
require_call = worker._best_effort_sec_user_id.await_args
self.assertTrue(require_call.kwargs["force_refresh"])
self.assertEqual(
events,
@@ -607,8 +842,10 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
worker._harvest_im_credentials.assert_awaited_once_with(timeout=25)
worker._persist_cookies.assert_awaited_once_with()
worker._build_im_session.assert_awaited_once_with()
worker._persist_im_session.assert_not_awaited()
worker._run_im_direct_service.assert_not_awaited()
# sec_user_id 只服务关注欢迎语；缺它不能阻断私信托管，
# 否则浏览器登录后账号立刻下线、永远不会自动回复。
worker._persist_im_session.assert_awaited_once()
worker._run_im_direct_service.assert_awaited_once_with(im_session)
worker._close_browser_only.assert_awaited_once_with()
async def test_browser_login_with_sec_user_id_continues_to_im(self):
@@ -617,7 +854,7 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
worker._load_user_agent = AsyncMock(return_value="test-agent")
worker._probe_existing_login = AsyncMock(return_value=True)
worker._finalize_login_session = AsyncMock()
worker._require_sec_user_id = AsyncMock(return_value=True)
worker._best_effort_sec_user_id = AsyncMock(return_value="sec-uid-306")
worker._setup_im_network_listener = AsyncMock()
worker._navigate_to_message_center = AsyncMock()
worker._harvest_im_credentials = AsyncMock()
@@ -674,8 +911,10 @@ class SecUserIdGuardTests(unittest.IsolatedAsyncioTestCase):
},
)
worker._require_sec_user_id.assert_awaited_once()
self.assertTrue(worker._require_sec_user_id.await_args.kwargs["force_refresh"])
worker._best_effort_sec_user_id.assert_awaited_once()
self.assertTrue(
worker._best_effort_sec_user_id.await_args.kwargs["force_refresh"]
)
worker._setup_im_network_listener.assert_awaited_once_with()
worker._navigate_to_message_center.assert_awaited_once_with()
worker._harvest_im_credentials.assert_awaited_once_with(timeout=25)
backend/tests/test_send_entry_and_worker_lifecycle.py
+125
View File
@@ -18,9 +18,12 @@ if str(BACKEND_DIR) not in sys.path:
sys.path.insert(0, str(BACKEND_DIR))
from rpa_engine.douyin_im import http_client as http_client_module
from rpa_engine.douyin_im.auth import DouyinAuth
from rpa_engine.douyin_im.frontier import ensure_frontier_ws
from rpa_engine.douyin_im.http_client import DouyinImHttpClient
from rpa_engine.douyin_im.session import DouyinImSession
from rpa_engine.egress_channels import EgressChannel
from rpa_engine import playwright_worker as playwright_worker_module
from rpa_engine.playwright_worker import DouyinWorker
@@ -33,6 +36,75 @@ class SendTextMessageEntryTests(unittest.IsolatedAsyncioTestCase):
)
return DouyinImHttpClient(session, account_id=account_id)
def test_query_user_does_not_replace_existing_im_uid(self):
client = self._make_client()
auth = SimpleNamespace(
source_ip="",
get_uid=MagicMock(return_value=938334054809296),
)
resolved = client._resolve_authoritative_uid(auth)
self.assertEqual(resolved, 10001)
self.assertEqual(client.session.my_uid, 10001)
auth.get_uid.assert_not_called()
def test_verified_uid_does_not_replace_the_runtime_device_id(self):
# device_id 是 query/user 返回的设备注册号，my_uid 是账号 UID。
# 拿 my_uid 顶替 device_id 会让 frontier 订阅到另一个地址：握手照样
# 成功，却永远收不到这个账号的私信。
session = DouyinImSession(
cookies={"sessionid": "test-session"},
my_uid=2609567359568155,
device_id="7678285795559818786",
web_id="7678286623234475535",
uid_verified=True,
)
auth = DouyinAuth.from_im_session(session)
self.assertEqual(auth.device_id, "7678285795559818786")
self.assertEqual(session.device_id, "7678285795559818786")
def test_captured_tokenless_browser_frontier_url_is_kept(self):
url = (
"wss://frontier100-normal.zijieapi.com/ws/v2?aid=6383&"
"device_platform=web&fpid=9&device_id=7678285795559818786&"
"access_key=0123456789abcdef0123456789abcdef&version_code=fws_1.0.0"
)
session = DouyinImSession(
cookies={"sessionid": "test-session"},
ws_urls=[url],
my_uid=2609567359568155,
device_id="7678285795559818786",
)
resolved = ensure_frontier_ws(session)
self.assertEqual(resolved, url)
self.assertEqual(session.ws_urls, [url])
def test_non_im_frontier_product_is_rejected(self):
url = (
"wss://frontier100-normal.zijieapi.com/ws/v2?aid=6383&"
"device_platform=web&fpid=971&device_id=7678285795559818786&"
"access_key=0123456789abcdef0123456789abcdef"
)
session = DouyinImSession(
cookies={"sessionid": "test-session"},
ws_urls=[url],
my_uid=2609567359568155,
device_id="7678285795559818786",
uid_verified=True,
)
resolved = ensure_frontier_ws(session)
self.assertIn("frontier-im.douyin.com", resolved)
self.assertIn("fpid=9", resolved)
self.assertIn("device_id=7678285795559818786", resolved)
self.assertNotIn("fpid=971", resolved)
async def test_public_entry_submits_the_whole_send_to_outbound_queue(self):
client = self._make_client(account_id=73)
submit = AsyncMock(return_value=True)
@@ -194,6 +266,59 @@ class SendTextMessageEntryTests(unittest.IsolatedAsyncioTestCase):
class WorkerLifecycleTests(unittest.IsolatedAsyncioTestCase):
async def test_virtual_display_starts_before_playwright_driver(self):
ensure_display = AsyncMock()
async def start_driver():
ensure_display.assert_awaited_once_with(False)
return "playwright-driver"
manager = SimpleNamespace(start=AsyncMock(side_effect=start_driver))
with (
patch.object(
playwright_worker_module,
"resolve_headless",
return_value=False,
),
patch.object(
playwright_worker_module,
"ensure_browser_display",
ensure_display,
),
patch.object(
playwright_worker_module,
"async_playwright",
return_value=manager,
),
):
driver, headless = await playwright_worker_module._start_playwright_for_browser()
self.assertEqual(driver, "playwright-driver")
self.assertFalse(headless)
manager.start.assert_awaited_once_with()
async def test_visible_login_prompt_overrides_stale_sessionid(self):
worker = DouyinWorker(account_id=917, login_mode="browser")
worker._has_visible_login_prompt = AsyncMock(return_value=True)
worker.check_homepage_login_status = AsyncMock(return_value=True)
worker.check_logged_in_by_cookie = AsyncMock(return_value=True)
logged_in = await worker._verify_login_state()
self.assertFalse(logged_in)
worker.check_homepage_login_status.assert_not_awaited()
worker.check_logged_in_by_cookie.assert_not_awaited()
async def test_homepage_message_entry_is_not_login_evidence(self):
worker = DouyinWorker(account_id=918, login_mode="browser")
worker._has_visible_login_prompt = AsyncMock(return_value=False)
worker.page = SimpleNamespace(query_selector=AsyncMock(return_value=None))
logged_in = await worker.check_homepage_login_status()
self.assertFalse(logged_in)
worker.page.query_selector.assert_awaited_once()
async def test_start_saves_task_and_stop_waits_until_it_is_done(self):
worker = DouyinWorker(account_id=919, login_mode="im_direct")
loop_started = asyncio.Event()
backend/tests/test_worker_scale_controls.py
+8 -7
View File
@@ -109,7 +109,7 @@ class WorkerScaleControlTests(unittest.IsolatedAsyncioTestCase):
)
worker._load_user_agent = AsyncMock(return_value="test-agent")
worker._build_im_session_from_storage = AsyncMock(return_value=session)
worker._require_sec_user_id = AsyncMock(return_value="sec-user")
worker._best_effort_sec_user_id = AsyncMock(return_value="sec-user")
worker._persist_im_session = AsyncMock()
worker._run_im_direct_service = AsyncMock()
@@ -124,7 +124,10 @@ class WorkerScaleControlTests(unittest.IsolatedAsyncioTestCase):
self.assertTrue(started)
self.assertEqual(reason, "")
validate.assert_not_awaited()
worker._require_sec_user_id.assert_awaited_once()
worker._best_effort_sec_user_id.assert_awaited_once_with(
refresh_if_missing=True,
refresh_if_stale=True,
)
worker._run_im_direct_service.assert_awaited_once_with(session)
async def test_disabled_follow_welcome_uses_cached_lightweight_config(self):
@@ -171,20 +174,18 @@ class WorkerScaleControlTests(unittest.IsolatedAsyncioTestCase):
worker.get_db.assert_not_awaited()
worker._require_sec_user_id.assert_not_awaited()
async def test_missing_cached_sec_user_id_stops_hosting(self):
async def test_disabled_follow_welcome_ignores_missing_cached_sec_user_id(self):
worker = DouyinWorker(account_id=505)
worker._im_service = SimpleNamespace(session=object())
worker._follow_config_loaded = True
worker._refresh_follow_welcome_config = AsyncMock(
return_value=(False, "", "")
)
worker._require_sec_user_id = AsyncMock(return_value="")
worker._best_effort_sec_user_id = AsyncMock(return_value="")
await worker.follow_welcome_tick()
worker._require_sec_user_id.assert_awaited_once_with(
"托管运行中"
)
worker._best_effort_sec_user_id.assert_not_awaited()
if __name__ == "__main__":
backend/tests/test_ws_client_scaling.py
+23 -1
View File
@@ -20,7 +20,10 @@ from rpa_engine.douyin_im.session import DouyinImSession
from rpa_engine.douyin_im.ws_client import DouyinImWsClient, _reconnect_delay
TEST_WS_URL = "wss://frontier-im.douyin.com/ws/v2?token=test-token-value"
TEST_WS_URL = (
"wss://frontier-im.douyin.com/ws/v2?fpid=9&device_id=10001&"
"token=test-token-value"
)
class _FakeWebSocket:
@@ -108,6 +111,25 @@ class WebSocketScalingTests(unittest.IsolatedAsyncioTestCase):
self.assertFalse(client.connected)
self.assertIsNone(client._connection)
async def test_browser_frontier_uses_text_heartbeat_and_filters_ack(self):
client = self._make_client()
client._running = True
websocket = AsyncMock()
heartbeat = asyncio.create_task(client._run_browser_heartbeat(websocket))
for _ in range(20):
if websocket.send.await_count:
break
await asyncio.sleep(0)
heartbeat.cancel()
with self.assertRaises(asyncio.CancelledError):
await heartbeat
websocket.send.assert_awaited_once_with("hi")
with patch.object(ws_module, "parse_ws_payload") as parse:
await client._dispatch("hi")
parse.assert_not_called()
async def test_starting_500_clients_does_not_create_os_threads(self):
parked = asyncio.Event()
backend/utils/cookie_store.py
+23
View File
@@ -128,6 +128,10 @@ def _parse_tea_from_ls(origins: list) -> tuple[str, str]:
parsed = json.loads(entry.get("value") or "{}")
except Exception:
continue
# __tea_cache_first_6383 等条目的值是纯数字（如 1），json.loads 返回 int，
# 不是 dict，必须跳过，否则 parsed.get() 抛 AttributeError 导致保存凭证 500。
if not isinstance(parsed, dict):
continue
uid = str(parsed.get("user_unique_id") or "").strip()
wid = str(parsed.get("web_id") or "").strip()
if uid and wid and uid == wid and len(uid) > 12:
@@ -233,6 +237,25 @@ def validate_cookie_json(cookie_data: str) -> dict:
return normalize_storage_state_for_im(data)
def extract_user_agent_from_cookie_data(cookie_data: Optional[str]) -> str:
"""从已保存的 Cookie JSON 提取顶层 user_agent（登录头）。
Playwright storage_state 导出时顶层带 user_agent（采集浏览器当前 UA）；
DYCRED 凭证经 convert_dycred_to_storage_state 后也统一落到 user_agent。
返回完整 UA 字符串；缺失/无效时返回空串，由调用方决定是否回填账号。
"""
if not cookie_data:
return ""
try:
data = json.loads(cookie_data)
except Exception:
return ""
if not isinstance(data, dict):
return ""
ua = str(data.get("user_agent") or data.get("ua") or "").strip()
return ua
def write_cookie_file(account_id: int, cookie_data: str) -> str:
data = validate_cookie_json(cookie_data)
path = get_cookie_path(account_id)