1082 lines
36 KiB
Python
1082 lines
36 KiB
Python
"""从 Cookie 抓取抖音账号资料（昵称 / 头像 / UID）及作品列表。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
from datetime import datetime
|
||
from typing import Any, Optional
|
||
|
||
import requests
|
||
from sqlalchemy import delete, select
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
from models.models import Account, AccountProfileDetail, AccountVideo
|
||
from rpa_engine.douyin_im.auth import DouyinAuth
|
||
from rpa_engine.douyin_im.dy_util import (
|
||
DEFAULT_USER_AGENT,
|
||
generate_a_bogus,
|
||
generate_msToken,
|
||
generate_webid,
|
||
splice_url,
|
||
)
|
||
from rpa_engine.douyin_im.protocol import _pick_avatar_url
|
||
from rpa_engine.douyin_im.session import DouyinImSession
|
||
from rpa_engine.device_profiles import resolve_user_agent
|
||
|
||
logger = logging.getLogger("account_profile")
|
||
|
||
|
||
def _pick_str(data: dict, *keys: str) -> str:
|
||
for key in keys:
|
||
value = data.get(key)
|
||
if value is not None and str(value).strip():
|
||
return str(value).strip()
|
||
return ""
|
||
|
||
|
||
def _pick_int(data: dict, *keys: str) -> Optional[int]:
|
||
for key in keys:
|
||
value = data.get(key)
|
||
if value is None or value == "":
|
||
continue
|
||
try:
|
||
return int(value)
|
||
except (TypeError, ValueError):
|
||
continue
|
||
return None
|
||
|
||
|
||
def _iter_profile_nodes(data: Any):
|
||
if not isinstance(data, dict):
|
||
return
|
||
yield data
|
||
for key in ("user", "user_info", "data", "creator"):
|
||
node = data.get(key)
|
||
if isinstance(node, dict):
|
||
yield node
|
||
|
||
|
||
def _extract_profile_from_payload(data: Any) -> dict[str, str]:
|
||
for node in _iter_profile_nodes(data):
|
||
uid = _pick_str(node, "uid", "user_uid", "user_id", "creator_user_id", "id")
|
||
nickname = _pick_str(
|
||
node,
|
||
"nickname",
|
||
"nick_name",
|
||
"unique_id",
|
||
"display_name",
|
||
"name",
|
||
)
|
||
avatar = _pick_avatar_url(node)
|
||
if uid or nickname or avatar:
|
||
return {
|
||
"uid": uid,
|
||
"nickname": nickname,
|
||
"avatar_url": avatar,
|
||
}
|
||
return {}
|
||
|
||
|
||
def _extract_detail_from_payload(data: Any) -> dict[str, Any]:
|
||
"""从抖音资料接口响应中提取详细统计。"""
|
||
detail: dict[str, Any] = {}
|
||
for node in _iter_profile_nodes(data):
|
||
if not detail.get("uid"):
|
||
uid = _pick_str(node, "uid", "user_uid", "user_id", "creator_user_id", "id")
|
||
if uid:
|
||
detail["uid"] = uid
|
||
if not detail.get("nickname"):
|
||
nickname = _pick_str(
|
||
node, "nickname", "nick_name", "display_name", "name"
|
||
)
|
||
if nickname:
|
||
detail["nickname"] = nickname
|
||
if not detail.get("avatar_url"):
|
||
avatar = _pick_avatar_url(node)
|
||
if avatar:
|
||
detail["avatar_url"] = avatar
|
||
if not detail.get("unique_id"):
|
||
unique_id = _pick_str(node, "unique_id", "short_id", "douyin_id")
|
||
if unique_id:
|
||
detail["unique_id"] = unique_id
|
||
if not detail.get("sec_user_id"):
|
||
sec_user_id = _pick_str(node, "sec_uid", "sec_user_id")
|
||
if sec_user_id:
|
||
detail["sec_user_id"] = sec_user_id
|
||
if not detail.get("signature"):
|
||
signature = _pick_str(node, "signature", "desc", "bio")
|
||
if signature:
|
||
detail["signature"] = signature
|
||
|
||
for field, keys in (
|
||
("video_count", ("aweme_count", "post_count", "video_count", "works_count")),
|
||
("follower_count", ("follower_count", "fans_count")),
|
||
("following_count", ("following_count", "follow_count")),
|
||
("total_favorited", ("total_favorited", "total_favorited_count", "like_count")),
|
||
("favoriting_count", ("favoriting_count", "favorite_count")),
|
||
):
|
||
if detail.get(field) is None:
|
||
val = _pick_int(node, *keys)
|
||
if val is not None:
|
||
detail[field] = val
|
||
return detail
|
||
|
||
|
||
def _pick_url_list(data: Any) -> str:
|
||
if isinstance(data, str) and data.startswith("http"):
|
||
return data
|
||
if isinstance(data, dict):
|
||
urls = data.get("url_list") or data.get("urlList") or []
|
||
if isinstance(urls, list):
|
||
for item in urls:
|
||
if isinstance(item, str) and item.startswith("http"):
|
||
return item
|
||
if isinstance(data, list):
|
||
for item in data:
|
||
if isinstance(item, str) and item.startswith("http"):
|
||
return item
|
||
return ""
|
||
|
||
|
||
def _pick_video_play_url(video: dict[str, Any]) -> str:
|
||
play_addr = video.get("play_addr") or video.get("download_addr") or {}
|
||
urls = play_addr.get("url_list") or []
|
||
if not isinstance(urls, list):
|
||
urls = []
|
||
for item in urls:
|
||
if not isinstance(item, str) or not item.startswith("http"):
|
||
continue
|
||
lower = item.lower()
|
||
if lower.endswith(".mp3") or "/ies-music/" in lower:
|
||
continue
|
||
if "douyinvod.com" in lower or "/video/" in lower or "mime_type=video" in lower:
|
||
return item.replace("playwm", "play")
|
||
for item in urls:
|
||
if isinstance(item, str) and item.startswith("http") and not item.lower().endswith(".mp3"):
|
||
return item.replace("playwm", "play")
|
||
bit_rate = video.get("bit_rate")
|
||
if isinstance(bit_rate, list):
|
||
for entry in bit_rate:
|
||
if not isinstance(entry, dict):
|
||
continue
|
||
play = entry.get("play_addr") or {}
|
||
url = _pick_url_list(play)
|
||
if url and not url.lower().endswith(".mp3"):
|
||
return url.replace("playwm", "play")
|
||
return ""
|
||
|
||
|
||
def _is_published_aweme(item: dict[str, Any]) -> bool:
|
||
"""仅保留已公开发布的作品（排除私密/审核中/已删除）。"""
|
||
if not isinstance(item, dict):
|
||
return False
|
||
if item.get("is_private"):
|
||
return False
|
||
|
||
status = item.get("status")
|
||
if isinstance(status, dict):
|
||
if status.get("is_delete"):
|
||
return False
|
||
if status.get("in_reviewing"):
|
||
return False
|
||
if status.get("is_prohibited"):
|
||
return False
|
||
private_status = status.get("private_status")
|
||
if private_status not in (None, 0, "0"):
|
||
return False
|
||
part_see = status.get("part_see")
|
||
if part_see not in (None, 0, "0", False):
|
||
return False
|
||
review = status.get("review_result")
|
||
if isinstance(review, dict):
|
||
review_status = review.get("review_status")
|
||
if review_status not in (None, 0, "0"):
|
||
return False
|
||
|
||
rate = item.get("rate")
|
||
if rate in (10, 11, 12):
|
||
return False
|
||
return True
|
||
|
||
|
||
def _parse_aweme_item(item: dict[str, Any]) -> dict[str, Any] | None:
|
||
if not isinstance(item, dict):
|
||
return None
|
||
aweme_id = _pick_str(item, "aweme_id", "awemeId", "item_id")
|
||
if not aweme_id:
|
||
return None
|
||
|
||
title = _pick_str(item, "desc", "title", "content")
|
||
video = item.get("video") if isinstance(item.get("video"), dict) else {}
|
||
cover_url = _pick_url_list(video.get("cover") or video.get("origin_cover") or item.get("cover"))
|
||
video_url = _pick_video_play_url(video)
|
||
if not cover_url and isinstance(item.get("images"), list) and item["images"]:
|
||
first_image = item["images"][0]
|
||
if isinstance(first_image, dict):
|
||
cover_url = _pick_url_list(first_image.get("url_list") or first_image)
|
||
|
||
share_url = _pick_str(item, "share_url", "share_link")
|
||
if not share_url:
|
||
share_url = f"https://www.douyin.com/video/{aweme_id}"
|
||
|
||
create_time = item.get("create_time") or item.get("createTime")
|
||
create_dt: datetime | None = None
|
||
if create_time:
|
||
try:
|
||
create_dt = datetime.utcfromtimestamp(int(create_time))
|
||
except (TypeError, ValueError):
|
||
create_dt = None
|
||
|
||
statistics = item.get("statistics") if isinstance(item.get("statistics"), dict) else {}
|
||
if not title:
|
||
title = f"作品 {aweme_id}"
|
||
|
||
aweme_type = item.get("aweme_type")
|
||
if video_url:
|
||
media_type = "video"
|
||
elif aweme_type == 68 or item.get("images"):
|
||
media_type = "image"
|
||
elif aweme_type in (0, 51, 55, 61):
|
||
media_type = "video"
|
||
else:
|
||
media_type = "other"
|
||
|
||
return {
|
||
"aweme_id": aweme_id,
|
||
"title": title,
|
||
"cover_url": cover_url,
|
||
"video_url": video_url,
|
||
"share_url": share_url,
|
||
"create_time": create_dt,
|
||
"media_type": media_type,
|
||
"digg_count": _pick_int(statistics, "digg_count", "like_count"),
|
||
"comment_count": _pick_int(statistics, "comment_count"),
|
||
"play_count": _pick_int(statistics, "play_count", "view_count"),
|
||
}
|
||
|
||
|
||
def _parse_creator_item(item: dict[str, Any]) -> dict[str, Any] | None:
|
||
if not isinstance(item, dict):
|
||
return None
|
||
aweme_id = _pick_str(item, "item_id", "item_id_plain", "aweme_id")
|
||
if not aweme_id:
|
||
return None
|
||
title = _pick_str(item, "title", "desc") or f"作品 {aweme_id}"
|
||
cover_url = _pick_str(item, "cover_image_url", "cover_url")
|
||
share_url = _pick_str(item, "item_link", "share_url") or f"https://www.douyin.com/video/{aweme_id}"
|
||
create_time = item.get("create_time")
|
||
create_dt: datetime | None = None
|
||
if create_time:
|
||
try:
|
||
create_dt = datetime.utcfromtimestamp(int(create_time))
|
||
except (TypeError, ValueError):
|
||
create_dt = None
|
||
return {
|
||
"aweme_id": aweme_id,
|
||
"title": title,
|
||
"cover_url": cover_url or None,
|
||
"video_url": None,
|
||
"share_url": share_url,
|
||
"create_time": create_dt,
|
||
"media_type": "image" if _pick_int(item, "media_type") == 2 else "other",
|
||
"digg_count": _pick_int(item, "digg_count", "like_count"),
|
||
"comment_count": _pick_int(item, "comment_count"),
|
||
"play_count": _pick_int(item, "play_count", "view_count"),
|
||
}
|
||
|
||
|
||
def _extract_creator_item_list(data: Any) -> list[dict[str, Any]]:
|
||
if not isinstance(data, dict):
|
||
return []
|
||
for key in ("item_info_list", "aweme_list", "item_list"):
|
||
node = data.get(key)
|
||
if isinstance(node, list) and node:
|
||
return [x for x in node if isinstance(x, dict)]
|
||
return []
|
||
|
||
|
||
def _extract_aweme_list(data: Any) -> list[dict[str, Any]]:
|
||
if not isinstance(data, dict):
|
||
return []
|
||
for key in ("aweme_list", "awemeList", "item_list", "items", "data"):
|
||
node = data.get(key)
|
||
if isinstance(node, list) and node:
|
||
return [x for x in node if isinstance(x, dict)]
|
||
if isinstance(node, dict):
|
||
nested = node.get("aweme_list") or node.get("item_list")
|
||
if isinstance(nested, list):
|
||
return [x for x in nested if isinstance(x, dict)]
|
||
return []
|
||
|
||
|
||
def _douyin_get_json(
|
||
auth: DouyinAuth,
|
||
ua: str,
|
||
url: str,
|
||
base_params: dict[str, str],
|
||
referer: str = "https://www.douyin.com/",
|
||
) -> dict[str, Any] | None:
|
||
headers = {
|
||
"User-Agent": ua,
|
||
"Referer": referer,
|
||
"Accept": "application/json, text/plain, */*",
|
||
}
|
||
proxies = _requests_proxies()
|
||
try:
|
||
params = dict(base_params)
|
||
query = splice_url(params)
|
||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||
resp = requests.get(
|
||
url,
|
||
params=params,
|
||
headers=headers,
|
||
cookies=auth.cookie,
|
||
verify=False,
|
||
timeout=15,
|
||
proxies=proxies,
|
||
)
|
||
data = resp.json()
|
||
return data if isinstance(data, dict) else None
|
||
except Exception as exc:
|
||
logger.debug(f"douyin get failed for {url}: {exc}")
|
||
return None
|
||
|
||
|
||
def _fetch_aweme_post_page(
|
||
auth: DouyinAuth,
|
||
ua: str,
|
||
sec_user_id: str,
|
||
max_count: int,
|
||
strategy_type: str,
|
||
) -> list[dict[str, Any]]:
|
||
base_params = {
|
||
"device_platform": "webapp",
|
||
"aid": "6383",
|
||
"channel": "channel_pc_web",
|
||
"sec_user_id": sec_user_id,
|
||
"max_cursor": "0",
|
||
"count": str(min(max_count, 35)),
|
||
"publish_video_strategy_type": strategy_type,
|
||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||
"msToken": auth.msToken or generate_msToken(),
|
||
"pc_client_type": "1",
|
||
}
|
||
|
||
videos: list[dict[str, Any]] = []
|
||
max_cursor = 0
|
||
while len(videos) < max_count:
|
||
params = dict(base_params)
|
||
params["max_cursor"] = str(max_cursor)
|
||
params["count"] = str(min(35, max_count - len(videos)))
|
||
data = _douyin_get_json(
|
||
auth,
|
||
ua,
|
||
"https://www.douyin.com/aweme/v1/web/aweme/post/",
|
||
params,
|
||
referer=f"https://www.douyin.com/user/{sec_user_id}",
|
||
)
|
||
if not data:
|
||
break
|
||
|
||
status_code = data.get("status_code")
|
||
if status_code not in (None, 0):
|
||
break
|
||
|
||
batch = _extract_aweme_list(data)
|
||
if not batch:
|
||
break
|
||
|
||
for item in batch:
|
||
if not _is_published_aweme(item):
|
||
continue
|
||
parsed = _parse_aweme_item(item)
|
||
if parsed and parsed["aweme_id"] not in {v["aweme_id"] for v in videos}:
|
||
videos.append(parsed)
|
||
if len(videos) >= max_count:
|
||
break
|
||
|
||
has_more = bool(data.get("has_more") or data.get("hasMore"))
|
||
next_cursor = data.get("max_cursor") or data.get("maxCursor")
|
||
try:
|
||
next_cursor = int(next_cursor or 0)
|
||
except (TypeError, ValueError):
|
||
next_cursor = 0
|
||
|
||
if not has_more or next_cursor == max_cursor:
|
||
break
|
||
max_cursor = next_cursor
|
||
return videos
|
||
|
||
|
||
def fetch_douyin_user_videos_sync(
|
||
cookie_data: str,
|
||
sec_user_id: str,
|
||
user_agent: Optional[str] = None,
|
||
max_count: int = 50,
|
||
) -> dict[str, Any]:
|
||
"""抓取用户已公开发布的作品列表（与抖音主页「作品」一致）。"""
|
||
result: dict[str, Any] = {"videos": [], "fetched": False, "message": ""}
|
||
sec_user_id = (sec_user_id or "").strip()
|
||
try:
|
||
auth, ua = _build_auth(cookie_data, user_agent)
|
||
except Exception as exc:
|
||
result["message"] = f"Cookie 无效: {exc}"
|
||
return result
|
||
|
||
if not sec_user_id:
|
||
result["message"] = "缺少 sec_user_id，无法拉取已发布作品"
|
||
return result
|
||
|
||
videos = _fetch_aweme_post_page(auth, ua, sec_user_id, max_count, "2")
|
||
result["videos"] = videos
|
||
result["fetched"] = bool(videos)
|
||
if not result["fetched"]:
|
||
result["message"] = "未获取到已发布作品，请确认 Cookie 有效且主页有公开作品"
|
||
return result
|
||
|
||
|
||
def _fetch_own_videos_sync(
|
||
auth: DouyinAuth,
|
||
ua: str,
|
||
max_count: int,
|
||
) -> dict[str, Any]:
|
||
"""创作者中心作品列表（适用于当前登录账号）。"""
|
||
result: dict[str, Any] = {"videos": [], "fetched": False}
|
||
endpoints = [
|
||
(
|
||
"https://creator.douyin.com/aweme/v1/creator/item/list/",
|
||
{
|
||
"status": "1",
|
||
"count": str(min(max_count, 35)),
|
||
"max_cursor": "0",
|
||
"aid": "6383",
|
||
"device_platform": "webapp",
|
||
},
|
||
"https://creator.douyin.com/creator-micro/content/manage",
|
||
"creator_item",
|
||
),
|
||
(
|
||
"https://creator.douyin.com/aweme/v1/creator/aweme/list/",
|
||
{
|
||
"status": "1",
|
||
"count": str(min(max_count, 35)),
|
||
"max_cursor": "0",
|
||
"aid": "6383",
|
||
"device_platform": "webapp",
|
||
},
|
||
"https://creator.douyin.com/creator-micro/content/manage",
|
||
"aweme",
|
||
),
|
||
(
|
||
"https://creator.douyin.com/web/api/media/aweme/post/",
|
||
{
|
||
"status": "1",
|
||
"count": str(min(max_count, 35)),
|
||
"max_cursor": "0",
|
||
},
|
||
"https://creator.douyin.com/creator-micro/content/manage",
|
||
"aweme",
|
||
),
|
||
]
|
||
videos: list[dict[str, Any]] = []
|
||
for url, base_params, referer, parser in endpoints:
|
||
data = _douyin_get_json(auth, ua, url, base_params, referer=referer)
|
||
if not data:
|
||
continue
|
||
if parser == "creator_item":
|
||
batch = _extract_creator_item_list(data)
|
||
parse_fn = _parse_creator_item
|
||
else:
|
||
batch = _extract_aweme_list(data)
|
||
parse_fn = _parse_aweme_item
|
||
for item in batch:
|
||
parsed = parse_fn(item)
|
||
if parsed and parsed["aweme_id"] not in {v["aweme_id"] for v in videos}:
|
||
videos.append(parsed)
|
||
if len(videos) >= max_count:
|
||
break
|
||
if videos:
|
||
break
|
||
result["videos"] = videos
|
||
result["fetched"] = bool(videos)
|
||
return result
|
||
|
||
|
||
def _requests_proxies() -> dict | None:
|
||
try:
|
||
from rpa_engine.runtime_config import requests_proxies
|
||
|
||
return requests_proxies()
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _build_auth(cookie_data: str, user_agent: Optional[str] = None) -> tuple[DouyinAuth, str]:
|
||
storage = json.loads(cookie_data)
|
||
session = DouyinImSession.from_storage_state(storage or {})
|
||
ua = resolve_user_agent(user_agent or session.user_agent or DEFAULT_USER_AGENT)
|
||
auth = DouyinAuth()
|
||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||
auth.user_agent = ua
|
||
auth.web_id = session.web_id or session.device_id or None
|
||
return auth, ua
|
||
|
||
|
||
def fetch_douyin_profile_sync(
|
||
cookie_data: str,
|
||
user_agent: Optional[str] = None,
|
||
) -> dict[str, str]:
|
||
detail = fetch_douyin_profile_detail_sync(cookie_data, user_agent)
|
||
return {
|
||
"uid": detail.get("uid") or "",
|
||
"nickname": detail.get("nickname") or "",
|
||
"avatar_url": detail.get("avatar_url") or "",
|
||
}
|
||
|
||
|
||
def fetch_douyin_profile_detail_sync(
|
||
cookie_data: str,
|
||
user_agent: Optional[str] = None,
|
||
) -> dict[str, Any]:
|
||
"""抓取抖音账号详细资料（昵称/头像/UID/作品数/粉丝等）。"""
|
||
result: dict[str, Any] = {
|
||
"uid": "",
|
||
"nickname": "",
|
||
"avatar_url": "",
|
||
"unique_id": "",
|
||
"signature": "",
|
||
"sec_user_id": "",
|
||
"video_count": None,
|
||
"follower_count": None,
|
||
"following_count": None,
|
||
"total_favorited": None,
|
||
"favoriting_count": None,
|
||
"fetched": False,
|
||
"message": "",
|
||
}
|
||
try:
|
||
auth, ua = _build_auth(cookie_data, user_agent)
|
||
except Exception as exc:
|
||
logger.warning(f"build auth for profile failed: {exc}")
|
||
result["message"] = "Cookie 无效，无法解析登录凭证"
|
||
return result
|
||
|
||
uid = auth.get_uid()
|
||
if uid:
|
||
result["uid"] = str(uid)
|
||
|
||
headers = {
|
||
"User-Agent": ua,
|
||
"Referer": "https://www.douyin.com/",
|
||
"Accept": "application/json, text/plain, */*",
|
||
}
|
||
endpoints: list[tuple[str, dict[str, str]]] = [
|
||
(
|
||
"https://www.douyin.com/aweme/v1/web/query/user/",
|
||
{
|
||
"device_platform": "webapp",
|
||
"aid": "6383",
|
||
"channel": "channel_pc_web",
|
||
"publish_video_strategy_type": "2",
|
||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||
"msToken": auth.msToken or generate_msToken(),
|
||
},
|
||
),
|
||
(
|
||
"https://creator.douyin.com/aweme/v1/creator/user/info/",
|
||
{
|
||
"device_platform": "webapp",
|
||
"aid": "6383",
|
||
},
|
||
),
|
||
(
|
||
"https://www.douyin.com/aweme/v1/web/user/profile/self/",
|
||
{
|
||
"device_platform": "webapp",
|
||
"aid": "6383",
|
||
"channel": "channel_pc_web",
|
||
},
|
||
),
|
||
]
|
||
|
||
proxies = _requests_proxies()
|
||
valid_profile_response = False
|
||
for url, base_params in endpoints:
|
||
try:
|
||
params = dict(base_params)
|
||
query = splice_url(params)
|
||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||
resp = requests.get(
|
||
url,
|
||
params=params,
|
||
headers=headers,
|
||
cookies=auth.cookie,
|
||
verify=False,
|
||
timeout=12,
|
||
proxies=proxies,
|
||
)
|
||
data = resp.json()
|
||
if not isinstance(data, dict):
|
||
continue
|
||
|
||
status_code = data.get("status_code")
|
||
if status_code is not None:
|
||
try:
|
||
if int(status_code) != 0:
|
||
continue
|
||
except (TypeError, ValueError):
|
||
continue
|
||
|
||
basic = _extract_profile_from_payload(data)
|
||
stats = _extract_detail_from_payload(data)
|
||
payload_has_profile = bool(
|
||
data.get("user_uid")
|
||
or basic.get("uid")
|
||
or basic.get("nickname")
|
||
or basic.get("avatar_url")
|
||
or stats.get("uid")
|
||
or stats.get("sec_user_id")
|
||
or stats.get("unique_id")
|
||
)
|
||
if not payload_has_profile:
|
||
# 401/风控响应也可能是 JSON；没有任何用户资料节点时
|
||
# 只能视为传输/鉴权未知，不能确认 sec_user_id 缺失。
|
||
continue
|
||
valid_profile_response = True
|
||
|
||
if data.get("user_uid") and not result["uid"]:
|
||
result["uid"] = str(data["user_uid"])
|
||
|
||
if basic.get("uid") and not result["uid"]:
|
||
result["uid"] = basic["uid"]
|
||
if basic.get("nickname") and not result["nickname"]:
|
||
result["nickname"] = basic["nickname"]
|
||
if basic.get("avatar_url") and not result["avatar_url"]:
|
||
result["avatar_url"] = basic["avatar_url"]
|
||
|
||
for key, value in stats.items():
|
||
if value is not None and value != "" and result.get(key) in (None, "", 0):
|
||
result[key] = value
|
||
|
||
if result.get("video_count") is not None:
|
||
result["fetched"] = True
|
||
break
|
||
if result["uid"] and result["nickname"] and result["avatar_url"]:
|
||
result["fetched"] = True
|
||
except Exception as exc:
|
||
logger.debug(f"profile detail fetch failed for {url}: {exc}")
|
||
|
||
if not result["fetched"] and valid_profile_response:
|
||
result["fetched"] = True
|
||
|
||
if not result["fetched"]:
|
||
result["message"] = result["message"] or "未能从抖音获取账号资料，请确认 Cookie 有效"
|
||
result["profile_response_valid"] = valid_profile_response
|
||
return result
|
||
|
||
|
||
async def fetch_douyin_profile(
|
||
cookie_data: str,
|
||
user_agent: Optional[str] = None,
|
||
) -> dict[str, str]:
|
||
return await asyncio.to_thread(fetch_douyin_profile_sync, cookie_data, user_agent)
|
||
|
||
|
||
async def fetch_douyin_profile_detail(
|
||
cookie_data: str,
|
||
user_agent: Optional[str] = None,
|
||
) -> dict[str, Any]:
|
||
return await asyncio.to_thread(fetch_douyin_profile_detail_sync, cookie_data, user_agent)
|
||
|
||
|
||
def _resolve_sec_user_id_by_uid_sync(
|
||
cookie_data: str,
|
||
user_agent: Optional[str],
|
||
uid: str,
|
||
) -> tuple[str, bool]:
|
||
"""Return ``(sec_user_id, request_completed)`` for the UID fallback."""
|
||
try:
|
||
auth, ua = _build_auth(cookie_data, user_agent)
|
||
extra = _douyin_get_json(
|
||
auth,
|
||
ua,
|
||
"https://www.douyin.com/aweme/v1/web/user/profile/other/",
|
||
{
|
||
"device_platform": "webapp",
|
||
"aid": "6383",
|
||
"channel": "channel_pc_web",
|
||
"user_id": str(uid),
|
||
"sec_user_id": "",
|
||
"publish_video_strategy_type": "2",
|
||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||
"msToken": auth.msToken or generate_msToken(),
|
||
},
|
||
)
|
||
except Exception as exc:
|
||
logger.debug(f"resolve sec_user_id failed: {exc}")
|
||
return "", False
|
||
if extra is None:
|
||
return "", False
|
||
status_code = extra.get("status_code")
|
||
if status_code is not None:
|
||
try:
|
||
if int(status_code) != 0:
|
||
return "", False
|
||
except (TypeError, ValueError):
|
||
return "", False
|
||
extracted = _extract_detail_from_payload(extra)
|
||
basic = _extract_profile_from_payload(extra)
|
||
sec_user_id = (extracted.get("sec_user_id") or "").strip()
|
||
if not (
|
||
sec_user_id
|
||
or extracted.get("uid")
|
||
or basic.get("uid")
|
||
or basic.get("nickname")
|
||
or basic.get("avatar_url")
|
||
):
|
||
return "", False
|
||
return sec_user_id, True
|
||
|
||
|
||
async def fetch_douyin_profile_detail_with_sec_user_id(
|
||
cookie_data: str,
|
||
user_agent: Optional[str] = None,
|
||
) -> dict[str, Any]:
|
||
"""Fetch profile data and run the existing UID fallback for sec_user_id.
|
||
|
||
``sec_user_id_status`` distinguishes a confirmed missing field from a
|
||
transient request failure so hosting is never stopped merely because the
|
||
profile endpoint was temporarily unavailable.
|
||
"""
|
||
detail = await fetch_douyin_profile_detail(cookie_data, user_agent)
|
||
sec_user_id = str(detail.get("sec_user_id") or "").strip()
|
||
if sec_user_id:
|
||
detail["sec_user_id"] = sec_user_id
|
||
detail["sec_user_id_status"] = "found"
|
||
return detail
|
||
|
||
if not detail.get("fetched"):
|
||
detail["sec_user_id_status"] = "unknown"
|
||
return detail
|
||
|
||
uid = str(detail.get("uid") or "").strip()
|
||
if uid:
|
||
sec_user_id, request_completed = await asyncio.to_thread(
|
||
_resolve_sec_user_id_by_uid_sync,
|
||
cookie_data,
|
||
user_agent,
|
||
uid,
|
||
)
|
||
if sec_user_id:
|
||
detail["sec_user_id"] = sec_user_id
|
||
detail["sec_user_id_status"] = "found"
|
||
else:
|
||
detail["sec_user_id_status"] = (
|
||
"missing" if request_completed else "unknown"
|
||
)
|
||
return detail
|
||
|
||
detail["sec_user_id_status"] = "missing"
|
||
return detail
|
||
|
||
|
||
async def fetch_douyin_user_videos(
|
||
cookie_data: str,
|
||
sec_user_id: str,
|
||
user_agent: Optional[str] = None,
|
||
max_count: int = 50,
|
||
) -> dict[str, Any]:
|
||
return await asyncio.to_thread(
|
||
fetch_douyin_user_videos_sync,
|
||
cookie_data,
|
||
sec_user_id,
|
||
user_agent,
|
||
max_count,
|
||
)
|
||
|
||
|
||
def _douyin_profile_url(sec_user_id: Optional[str]) -> Optional[str]:
|
||
sec = (sec_user_id or "").strip()
|
||
if not sec:
|
||
return None
|
||
return f"https://www.douyin.com/user/{sec}"
|
||
|
||
|
||
def _profile_detail_to_dict(
|
||
profile: AccountProfileDetail | None,
|
||
account: Account,
|
||
videos: list[AccountVideo] | None = None,
|
||
) -> dict[str, Any]:
|
||
video_rows = videos or []
|
||
cached_count = len(video_rows)
|
||
|
||
def _row_media_type(row: AccountVideo) -> str:
|
||
if row.media_type:
|
||
return row.media_type
|
||
return "video" if row.video_url else "image"
|
||
|
||
video_work_count = sum(1 for v in video_rows if _row_media_type(v) == "video")
|
||
image_work_count = sum(1 for v in video_rows if _row_media_type(v) == "image")
|
||
playable_count = video_work_count
|
||
if profile:
|
||
api_count = profile.video_count
|
||
if cached_count:
|
||
display_count = cached_count
|
||
else:
|
||
display_count = api_count
|
||
return {
|
||
"account_id": account.id,
|
||
"uid": profile.uid or account.douyin_uid,
|
||
"nickname": profile.nickname or account.username,
|
||
"avatar_url": profile.avatar_url or account.avatar_url,
|
||
"unique_id": profile.unique_id,
|
||
"signature": profile.signature,
|
||
"sec_user_id": profile.sec_user_id,
|
||
"profile_url": _douyin_profile_url(profile.sec_user_id),
|
||
"video_count": display_count,
|
||
"video_count_douyin": api_count,
|
||
"cached_work_count": cached_count,
|
||
"playable_video_count": playable_count,
|
||
"video_work_count": video_work_count,
|
||
"image_work_count": image_work_count,
|
||
"follower_count": profile.follower_count,
|
||
"following_count": profile.following_count,
|
||
"total_favorited": profile.total_favorited,
|
||
"favoriting_count": profile.favoriting_count,
|
||
"fetched": bool(profile.synced_at),
|
||
"message": profile.sync_message,
|
||
"synced_at": profile.synced_at.isoformat() if profile.synced_at else None,
|
||
"profile_aweme_count": api_count,
|
||
"videos": [
|
||
{
|
||
"id": v.id,
|
||
"aweme_id": v.aweme_id,
|
||
"title": v.title or "",
|
||
"cover_url": v.cover_url,
|
||
"video_url": v.video_url,
|
||
"share_url": v.share_url,
|
||
"create_time": v.create_time.isoformat() if v.create_time else None,
|
||
"digg_count": v.digg_count,
|
||
"comment_count": v.comment_count,
|
||
"play_count": v.play_count,
|
||
"media_type": v.media_type,
|
||
}
|
||
for v in video_rows
|
||
],
|
||
}
|
||
return {
|
||
"account_id": account.id,
|
||
"uid": account.douyin_uid,
|
||
"nickname": account.username,
|
||
"avatar_url": account.avatar_url,
|
||
"unique_id": None,
|
||
"signature": None,
|
||
"sec_user_id": None,
|
||
"profile_url": None,
|
||
"video_count": None,
|
||
"video_count_douyin": None,
|
||
"cached_work_count": 0,
|
||
"playable_video_count": 0,
|
||
"video_work_count": 0,
|
||
"image_work_count": 0,
|
||
"follower_count": None,
|
||
"following_count": None,
|
||
"total_favorited": None,
|
||
"favoriting_count": None,
|
||
"fetched": False,
|
||
"message": "暂无本地资料，请点击刷新从抖音同步",
|
||
"synced_at": None,
|
||
"videos": [],
|
||
}
|
||
|
||
|
||
async def load_account_profile_from_db(
|
||
db: AsyncSession,
|
||
account: Account,
|
||
) -> dict[str, Any]:
|
||
profile = (
|
||
await db.execute(
|
||
select(AccountProfileDetail).where(AccountProfileDetail.account_id == account.id)
|
||
)
|
||
).scalar_one_or_none()
|
||
videos: list[AccountVideo] = []
|
||
if profile:
|
||
videos = list(
|
||
(
|
||
await db.execute(
|
||
select(AccountVideo)
|
||
.where(AccountVideo.account_id == account.id)
|
||
.order_by(AccountVideo.sort_order.asc(), AccountVideo.id.asc())
|
||
)
|
||
).scalars().all()
|
||
)
|
||
return _profile_detail_to_dict(profile, account, videos)
|
||
|
||
|
||
async def sync_account_profile_to_db(
|
||
db: AsyncSession,
|
||
account: Account,
|
||
cookie_data: str,
|
||
max_videos: int = 50,
|
||
) -> dict[str, Any]:
|
||
"""从抖音拉取资料与作品并写入本地数据库。"""
|
||
detail = await fetch_douyin_profile_detail_with_sec_user_id(
|
||
cookie_data,
|
||
account.user_agent,
|
||
)
|
||
sec_user_id = (detail.get("sec_user_id") or "").strip()
|
||
|
||
if detail.get("sec_user_id_status") == "unknown":
|
||
message = (
|
||
detail.get("message")
|
||
or "暂时无法核验 sec_user_id，已保留原有账号资料，请稍后重试"
|
||
)
|
||
profile = (
|
||
await db.execute(
|
||
select(AccountProfileDetail).where(
|
||
AccountProfileDetail.account_id == account.id
|
||
)
|
||
)
|
||
).scalar_one_or_none()
|
||
if profile:
|
||
profile.sync_message = message
|
||
await db.commit()
|
||
cached = await load_account_profile_from_db(db, account)
|
||
cached["message"] = message
|
||
return cached
|
||
|
||
if detail.get("fetched"):
|
||
try:
|
||
await apply_douyin_profile(db, account, cookie_data)
|
||
except Exception as exc:
|
||
logger.warning(f"apply douyin profile failed: {exc}")
|
||
|
||
video_result = await fetch_douyin_user_videos(
|
||
cookie_data,
|
||
sec_user_id,
|
||
account.user_agent,
|
||
max_count=max_videos,
|
||
)
|
||
|
||
now = datetime.utcnow()
|
||
profile = (
|
||
await db.execute(
|
||
select(AccountProfileDetail).where(AccountProfileDetail.account_id == account.id)
|
||
)
|
||
).scalar_one_or_none()
|
||
if not profile:
|
||
profile = AccountProfileDetail(account_id=account.id)
|
||
db.add(profile)
|
||
|
||
profile.uid = detail.get("uid") or account.douyin_uid
|
||
profile.nickname = detail.get("nickname") or account.username
|
||
profile.avatar_url = detail.get("avatar_url") or account.avatar_url
|
||
profile.unique_id = detail.get("unique_id") or None
|
||
profile.signature = detail.get("signature") or None
|
||
sec_user_id_status = detail.get("sec_user_id_status")
|
||
if sec_user_id:
|
||
profile.sec_user_id = sec_user_id
|
||
elif sec_user_id_status == "missing":
|
||
# Only an authoritative successful response may clear the identity.
|
||
# Network/parse failures keep the last known value so a profile refresh
|
||
# cannot accidentally force a running account offline.
|
||
profile.sec_user_id = None
|
||
profile.follower_count = detail.get("follower_count")
|
||
profile.following_count = detail.get("following_count")
|
||
profile.total_favorited = detail.get("total_favorited")
|
||
profile.favoriting_count = detail.get("favoriting_count")
|
||
profile.synced_at = now
|
||
|
||
messages: list[str] = []
|
||
if detail.get("message"):
|
||
messages.append(str(detail["message"]))
|
||
|
||
fetched_videos = video_result.get("videos") or []
|
||
await db.execute(delete(AccountVideo).where(AccountVideo.account_id == account.id))
|
||
for idx, item in enumerate(fetched_videos):
|
||
db.add(
|
||
AccountVideo(
|
||
account_id=account.id,
|
||
aweme_id=item["aweme_id"],
|
||
title=item.get("title"),
|
||
cover_url=item.get("cover_url"),
|
||
video_url=item.get("video_url"),
|
||
share_url=item.get("share_url"),
|
||
create_time=item.get("create_time"),
|
||
digg_count=item.get("digg_count"),
|
||
comment_count=item.get("comment_count"),
|
||
play_count=item.get("play_count"),
|
||
media_type=item.get("media_type"),
|
||
sort_order=idx,
|
||
synced_at=now,
|
||
)
|
||
)
|
||
|
||
profile.video_count = len(fetched_videos)
|
||
if fetched_videos:
|
||
profile.sync_message = "；".join(messages) if messages else None
|
||
else:
|
||
empty_msg = str(video_result.get("message") or "主页暂无已公开发布作品")
|
||
if messages:
|
||
messages.append(empty_msg)
|
||
else:
|
||
messages = [empty_msg]
|
||
profile.sync_message = "；".join(messages)
|
||
|
||
await db.commit()
|
||
await db.refresh(account)
|
||
return await load_account_profile_from_db(db, account)
|
||
|
||
|
||
async def _pick_unique_username(
|
||
db: AsyncSession,
|
||
account: Account,
|
||
nickname: str,
|
||
uid: str,
|
||
) -> str:
|
||
candidates: list[str] = []
|
||
if nickname and uid:
|
||
candidates.extend([nickname, f"{nickname}_{uid}"])
|
||
elif nickname:
|
||
candidates.append(nickname)
|
||
elif uid:
|
||
candidates.append(f"用户{uid}")
|
||
candidates.append(f"{(nickname or '账号')}_{account.id}")
|
||
|
||
for candidate in candidates:
|
||
name = candidate[:100]
|
||
stmt = select(Account.id).where(Account.username == name, Account.id != account.id)
|
||
conflict = (await db.execute(stmt)).scalar_one_or_none()
|
||
if not conflict:
|
||
return name
|
||
return f"账号_{account.id}"
|
||
|
||
|
||
async def apply_douyin_profile(
|
||
db: AsyncSession,
|
||
account: Account,
|
||
cookie_data: str,
|
||
) -> dict[str, str]:
|
||
"""抓取并写入账号资料，返回抓取结果。"""
|
||
profile = await fetch_douyin_profile(cookie_data, account.user_agent)
|
||
uid = (profile.get("uid") or "").strip()
|
||
nickname = (profile.get("nickname") or "").strip()
|
||
avatar = (profile.get("avatar_url") or "").strip()
|
||
|
||
if uid:
|
||
account.douyin_uid = uid
|
||
if avatar:
|
||
account.avatar_url = avatar
|
||
if nickname or uid:
|
||
account.username = await _pick_unique_username(db, account, nickname, uid)
|
||
|
||
return profile
|