更新
This commit is contained in:
Your Name
@@ -0,0 +1,959 @@
|
||||
"""从 Cookie 抓取抖音账号资料（昵称 / 头像 / UID）及作品列表。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
import requests
|
||||
from sqlalchemy import delete, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from models.models import Account, AccountProfileDetail, AccountVideo
|
||||
from rpa_engine.douyin_im.auth import DouyinAuth
|
||||
from rpa_engine.douyin_im.dy_util import (
|
||||
DEFAULT_USER_AGENT,
|
||||
generate_a_bogus,
|
||||
generate_msToken,
|
||||
generate_webid,
|
||||
splice_url,
|
||||
)
|
||||
from rpa_engine.douyin_im.protocol import _pick_avatar_url
|
||||
from rpa_engine.douyin_im.session import DouyinImSession
|
||||
from rpa_engine.device_profiles import resolve_user_agent
|
||||
|
||||
logger = logging.getLogger("account_profile")
|
||||
|
||||
|
||||
def _pick_str(data: dict, *keys: str) -> str:
|
||||
for key in keys:
|
||||
value = data.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return str(value).strip()
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_int(data: dict, *keys: str) -> Optional[int]:
|
||||
for key in keys:
|
||||
value = data.get(key)
|
||||
if value is None or value == "":
|
||||
continue
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _iter_profile_nodes(data: Any):
|
||||
if not isinstance(data, dict):
|
||||
return
|
||||
yield data
|
||||
for key in ("user", "user_info", "data", "creator"):
|
||||
node = data.get(key)
|
||||
if isinstance(node, dict):
|
||||
yield node
|
||||
|
||||
|
||||
def _extract_profile_from_payload(data: Any) -> dict[str, str]:
|
||||
for node in _iter_profile_nodes(data):
|
||||
uid = _pick_str(node, "uid", "user_uid", "user_id", "creator_user_id", "id")
|
||||
nickname = _pick_str(
|
||||
node,
|
||||
"nickname",
|
||||
"nick_name",
|
||||
"unique_id",
|
||||
"display_name",
|
||||
"name",
|
||||
)
|
||||
avatar = _pick_avatar_url(node)
|
||||
if uid or nickname or avatar:
|
||||
return {
|
||||
"uid": uid,
|
||||
"nickname": nickname,
|
||||
"avatar_url": avatar,
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
def _extract_detail_from_payload(data: Any) -> dict[str, Any]:
|
||||
"""从抖音资料接口响应中提取详细统计。"""
|
||||
detail: dict[str, Any] = {}
|
||||
for node in _iter_profile_nodes(data):
|
||||
if not detail.get("uid"):
|
||||
uid = _pick_str(node, "uid", "user_uid", "user_id", "creator_user_id", "id")
|
||||
if uid:
|
||||
detail["uid"] = uid
|
||||
if not detail.get("nickname"):
|
||||
nickname = _pick_str(
|
||||
node, "nickname", "nick_name", "display_name", "name"
|
||||
)
|
||||
if nickname:
|
||||
detail["nickname"] = nickname
|
||||
if not detail.get("avatar_url"):
|
||||
avatar = _pick_avatar_url(node)
|
||||
if avatar:
|
||||
detail["avatar_url"] = avatar
|
||||
if not detail.get("unique_id"):
|
||||
unique_id = _pick_str(node, "unique_id", "short_id", "douyin_id")
|
||||
if unique_id:
|
||||
detail["unique_id"] = unique_id
|
||||
if not detail.get("sec_user_id"):
|
||||
sec_user_id = _pick_str(node, "sec_uid", "sec_user_id")
|
||||
if sec_user_id:
|
||||
detail["sec_user_id"] = sec_user_id
|
||||
if not detail.get("signature"):
|
||||
signature = _pick_str(node, "signature", "desc", "bio")
|
||||
if signature:
|
||||
detail["signature"] = signature
|
||||
|
||||
for field, keys in (
|
||||
("video_count", ("aweme_count", "post_count", "video_count", "works_count")),
|
||||
("follower_count", ("follower_count", "fans_count")),
|
||||
("following_count", ("following_count", "follow_count")),
|
||||
("total_favorited", ("total_favorited", "total_favorited_count", "like_count")),
|
||||
("favoriting_count", ("favoriting_count", "favorite_count")),
|
||||
):
|
||||
if detail.get(field) is None:
|
||||
val = _pick_int(node, *keys)
|
||||
if val is not None:
|
||||
detail[field] = val
|
||||
return detail
|
||||
|
||||
|
||||
def _pick_url_list(data: Any) -> str:
|
||||
if isinstance(data, str) and data.startswith("http"):
|
||||
return data
|
||||
if isinstance(data, dict):
|
||||
urls = data.get("url_list") or data.get("urlList") or []
|
||||
if isinstance(urls, list):
|
||||
for item in urls:
|
||||
if isinstance(item, str) and item.startswith("http"):
|
||||
return item
|
||||
if isinstance(data, list):
|
||||
for item in data:
|
||||
if isinstance(item, str) and item.startswith("http"):
|
||||
return item
|
||||
return ""
|
||||
|
||||
|
||||
def _pick_video_play_url(video: dict[str, Any]) -> str:
|
||||
play_addr = video.get("play_addr") or video.get("download_addr") or {}
|
||||
urls = play_addr.get("url_list") or []
|
||||
if not isinstance(urls, list):
|
||||
urls = []
|
||||
for item in urls:
|
||||
if not isinstance(item, str) or not item.startswith("http"):
|
||||
continue
|
||||
lower = item.lower()
|
||||
if lower.endswith(".mp3") or "/ies-music/" in lower:
|
||||
continue
|
||||
if "douyinvod.com" in lower or "/video/" in lower or "mime_type=video" in lower:
|
||||
return item.replace("playwm", "play")
|
||||
for item in urls:
|
||||
if isinstance(item, str) and item.startswith("http") and not item.lower().endswith(".mp3"):
|
||||
return item.replace("playwm", "play")
|
||||
bit_rate = video.get("bit_rate")
|
||||
if isinstance(bit_rate, list):
|
||||
for entry in bit_rate:
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
play = entry.get("play_addr") or {}
|
||||
url = _pick_url_list(play)
|
||||
if url and not url.lower().endswith(".mp3"):
|
||||
return url.replace("playwm", "play")
|
||||
return ""
|
||||
|
||||
|
||||
def _is_published_aweme(item: dict[str, Any]) -> bool:
|
||||
"""仅保留已公开发布的作品（排除私密/审核中/已删除）。"""
|
||||
if not isinstance(item, dict):
|
||||
return False
|
||||
if item.get("is_private"):
|
||||
return False
|
||||
|
||||
status = item.get("status")
|
||||
if isinstance(status, dict):
|
||||
if status.get("is_delete"):
|
||||
return False
|
||||
if status.get("in_reviewing"):
|
||||
return False
|
||||
if status.get("is_prohibited"):
|
||||
return False
|
||||
private_status = status.get("private_status")
|
||||
if private_status not in (None, 0, "0"):
|
||||
return False
|
||||
part_see = status.get("part_see")
|
||||
if part_see not in (None, 0, "0", False):
|
||||
return False
|
||||
review = status.get("review_result")
|
||||
if isinstance(review, dict):
|
||||
review_status = review.get("review_status")
|
||||
if review_status not in (None, 0, "0"):
|
||||
return False
|
||||
|
||||
rate = item.get("rate")
|
||||
if rate in (10, 11, 12):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _parse_aweme_item(item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
aweme_id = _pick_str(item, "aweme_id", "awemeId", "item_id")
|
||||
if not aweme_id:
|
||||
return None
|
||||
|
||||
title = _pick_str(item, "desc", "title", "content")
|
||||
video = item.get("video") if isinstance(item.get("video"), dict) else {}
|
||||
cover_url = _pick_url_list(video.get("cover") or video.get("origin_cover") or item.get("cover"))
|
||||
video_url = _pick_video_play_url(video)
|
||||
if not cover_url and isinstance(item.get("images"), list) and item["images"]:
|
||||
first_image = item["images"][0]
|
||||
if isinstance(first_image, dict):
|
||||
cover_url = _pick_url_list(first_image.get("url_list") or first_image)
|
||||
|
||||
share_url = _pick_str(item, "share_url", "share_link")
|
||||
if not share_url:
|
||||
share_url = f"https://www.douyin.com/video/{aweme_id}"
|
||||
|
||||
create_time = item.get("create_time") or item.get("createTime")
|
||||
create_dt: datetime | None = None
|
||||
if create_time:
|
||||
try:
|
||||
create_dt = datetime.utcfromtimestamp(int(create_time))
|
||||
except (TypeError, ValueError):
|
||||
create_dt = None
|
||||
|
||||
statistics = item.get("statistics") if isinstance(item.get("statistics"), dict) else {}
|
||||
if not title:
|
||||
title = f"作品 {aweme_id}"
|
||||
|
||||
aweme_type = item.get("aweme_type")
|
||||
if video_url:
|
||||
media_type = "video"
|
||||
elif aweme_type == 68 or item.get("images"):
|
||||
media_type = "image"
|
||||
elif aweme_type in (0, 51, 55, 61):
|
||||
media_type = "video"
|
||||
else:
|
||||
media_type = "other"
|
||||
|
||||
return {
|
||||
"aweme_id": aweme_id,
|
||||
"title": title,
|
||||
"cover_url": cover_url,
|
||||
"video_url": video_url,
|
||||
"share_url": share_url,
|
||||
"create_time": create_dt,
|
||||
"media_type": media_type,
|
||||
"digg_count": _pick_int(statistics, "digg_count", "like_count"),
|
||||
"comment_count": _pick_int(statistics, "comment_count"),
|
||||
"play_count": _pick_int(statistics, "play_count", "view_count"),
|
||||
}
|
||||
|
||||
|
||||
def _parse_creator_item(item: dict[str, Any]) -> dict[str, Any] | None:
|
||||
if not isinstance(item, dict):
|
||||
return None
|
||||
aweme_id = _pick_str(item, "item_id", "item_id_plain", "aweme_id")
|
||||
if not aweme_id:
|
||||
return None
|
||||
title = _pick_str(item, "title", "desc") or f"作品 {aweme_id}"
|
||||
cover_url = _pick_str(item, "cover_image_url", "cover_url")
|
||||
share_url = _pick_str(item, "item_link", "share_url") or f"https://www.douyin.com/video/{aweme_id}"
|
||||
create_time = item.get("create_time")
|
||||
create_dt: datetime | None = None
|
||||
if create_time:
|
||||
try:
|
||||
create_dt = datetime.utcfromtimestamp(int(create_time))
|
||||
except (TypeError, ValueError):
|
||||
create_dt = None
|
||||
return {
|
||||
"aweme_id": aweme_id,
|
||||
"title": title,
|
||||
"cover_url": cover_url or None,
|
||||
"video_url": None,
|
||||
"share_url": share_url,
|
||||
"create_time": create_dt,
|
||||
"media_type": "image" if _pick_int(item, "media_type") == 2 else "other",
|
||||
"digg_count": _pick_int(item, "digg_count", "like_count"),
|
||||
"comment_count": _pick_int(item, "comment_count"),
|
||||
"play_count": _pick_int(item, "play_count", "view_count"),
|
||||
}
|
||||
|
||||
|
||||
def _extract_creator_item_list(data: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
for key in ("item_info_list", "aweme_list", "item_list"):
|
||||
node = data.get(key)
|
||||
if isinstance(node, list) and node:
|
||||
return [x for x in node if isinstance(x, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _extract_aweme_list(data: Any) -> list[dict[str, Any]]:
|
||||
if not isinstance(data, dict):
|
||||
return []
|
||||
for key in ("aweme_list", "awemeList", "item_list", "items", "data"):
|
||||
node = data.get(key)
|
||||
if isinstance(node, list) and node:
|
||||
return [x for x in node if isinstance(x, dict)]
|
||||
if isinstance(node, dict):
|
||||
nested = node.get("aweme_list") or node.get("item_list")
|
||||
if isinstance(nested, list):
|
||||
return [x for x in nested if isinstance(x, dict)]
|
||||
return []
|
||||
|
||||
|
||||
def _douyin_get_json(
|
||||
auth: DouyinAuth,
|
||||
ua: str,
|
||||
url: str,
|
||||
base_params: dict[str, str],
|
||||
referer: str = "https://www.douyin.com/",
|
||||
) -> dict[str, Any] | None:
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": referer,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
proxies = _requests_proxies()
|
||||
try:
|
||||
params = dict(base_params)
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
resp = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
verify=False,
|
||||
timeout=15,
|
||||
proxies=proxies,
|
||||
)
|
||||
data = resp.json()
|
||||
return data if isinstance(data, dict) else None
|
||||
except Exception as exc:
|
||||
logger.debug(f"douyin get failed for {url}: {exc}")
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_aweme_post_page(
|
||||
auth: DouyinAuth,
|
||||
ua: str,
|
||||
sec_user_id: str,
|
||||
max_count: int,
|
||||
strategy_type: str,
|
||||
) -> list[dict[str, Any]]:
|
||||
base_params = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"sec_user_id": sec_user_id,
|
||||
"max_cursor": "0",
|
||||
"count": str(min(max_count, 35)),
|
||||
"publish_video_strategy_type": strategy_type,
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
"pc_client_type": "1",
|
||||
}
|
||||
|
||||
videos: list[dict[str, Any]] = []
|
||||
max_cursor = 0
|
||||
while len(videos) < max_count:
|
||||
params = dict(base_params)
|
||||
params["max_cursor"] = str(max_cursor)
|
||||
params["count"] = str(min(35, max_count - len(videos)))
|
||||
data = _douyin_get_json(
|
||||
auth,
|
||||
ua,
|
||||
"https://www.douyin.com/aweme/v1/web/aweme/post/",
|
||||
params,
|
||||
referer=f"https://www.douyin.com/user/{sec_user_id}",
|
||||
)
|
||||
if not data:
|
||||
break
|
||||
|
||||
status_code = data.get("status_code")
|
||||
if status_code not in (None, 0):
|
||||
break
|
||||
|
||||
batch = _extract_aweme_list(data)
|
||||
if not batch:
|
||||
break
|
||||
|
||||
for item in batch:
|
||||
if not _is_published_aweme(item):
|
||||
continue
|
||||
parsed = _parse_aweme_item(item)
|
||||
if parsed and parsed["aweme_id"] not in {v["aweme_id"] for v in videos}:
|
||||
videos.append(parsed)
|
||||
if len(videos) >= max_count:
|
||||
break
|
||||
|
||||
has_more = bool(data.get("has_more") or data.get("hasMore"))
|
||||
next_cursor = data.get("max_cursor") or data.get("maxCursor")
|
||||
try:
|
||||
next_cursor = int(next_cursor or 0)
|
||||
except (TypeError, ValueError):
|
||||
next_cursor = 0
|
||||
|
||||
if not has_more or next_cursor == max_cursor:
|
||||
break
|
||||
max_cursor = next_cursor
|
||||
return videos
|
||||
|
||||
|
||||
def fetch_douyin_user_videos_sync(
|
||||
cookie_data: str,
|
||||
sec_user_id: str,
|
||||
user_agent: Optional[str] = None,
|
||||
max_count: int = 50,
|
||||
) -> dict[str, Any]:
|
||||
"""抓取用户已公开发布的作品列表（与抖音主页「作品」一致）。"""
|
||||
result: dict[str, Any] = {"videos": [], "fetched": False, "message": ""}
|
||||
sec_user_id = (sec_user_id or "").strip()
|
||||
try:
|
||||
auth, ua = _build_auth(cookie_data, user_agent)
|
||||
except Exception as exc:
|
||||
result["message"] = f"Cookie 无效: {exc}"
|
||||
return result
|
||||
|
||||
if not sec_user_id:
|
||||
result["message"] = "缺少 sec_user_id，无法拉取已发布作品"
|
||||
return result
|
||||
|
||||
videos = _fetch_aweme_post_page(auth, ua, sec_user_id, max_count, "2")
|
||||
result["videos"] = videos
|
||||
result["fetched"] = bool(videos)
|
||||
if not result["fetched"]:
|
||||
result["message"] = "未获取到已发布作品，请确认 Cookie 有效且主页有公开作品"
|
||||
return result
|
||||
|
||||
|
||||
def _fetch_own_videos_sync(
|
||||
auth: DouyinAuth,
|
||||
ua: str,
|
||||
max_count: int,
|
||||
) -> dict[str, Any]:
|
||||
"""创作者中心作品列表（适用于当前登录账号）。"""
|
||||
result: dict[str, Any] = {"videos": [], "fetched": False}
|
||||
endpoints = [
|
||||
(
|
||||
"https://creator.douyin.com/aweme/v1/creator/item/list/",
|
||||
{
|
||||
"status": "1",
|
||||
"count": str(min(max_count, 35)),
|
||||
"max_cursor": "0",
|
||||
"aid": "6383",
|
||||
"device_platform": "webapp",
|
||||
},
|
||||
"https://creator.douyin.com/creator-micro/content/manage",
|
||||
"creator_item",
|
||||
),
|
||||
(
|
||||
"https://creator.douyin.com/aweme/v1/creator/aweme/list/",
|
||||
{
|
||||
"status": "1",
|
||||
"count": str(min(max_count, 35)),
|
||||
"max_cursor": "0",
|
||||
"aid": "6383",
|
||||
"device_platform": "webapp",
|
||||
},
|
||||
"https://creator.douyin.com/creator-micro/content/manage",
|
||||
"aweme",
|
||||
),
|
||||
(
|
||||
"https://creator.douyin.com/web/api/media/aweme/post/",
|
||||
{
|
||||
"status": "1",
|
||||
"count": str(min(max_count, 35)),
|
||||
"max_cursor": "0",
|
||||
},
|
||||
"https://creator.douyin.com/creator-micro/content/manage",
|
||||
"aweme",
|
||||
),
|
||||
]
|
||||
videos: list[dict[str, Any]] = []
|
||||
for url, base_params, referer, parser in endpoints:
|
||||
data = _douyin_get_json(auth, ua, url, base_params, referer=referer)
|
||||
if not data:
|
||||
continue
|
||||
if parser == "creator_item":
|
||||
batch = _extract_creator_item_list(data)
|
||||
parse_fn = _parse_creator_item
|
||||
else:
|
||||
batch = _extract_aweme_list(data)
|
||||
parse_fn = _parse_aweme_item
|
||||
for item in batch:
|
||||
parsed = parse_fn(item)
|
||||
if parsed and parsed["aweme_id"] not in {v["aweme_id"] for v in videos}:
|
||||
videos.append(parsed)
|
||||
if len(videos) >= max_count:
|
||||
break
|
||||
if videos:
|
||||
break
|
||||
result["videos"] = videos
|
||||
result["fetched"] = bool(videos)
|
||||
return result
|
||||
|
||||
|
||||
def _requests_proxies() -> dict | None:
|
||||
try:
|
||||
from rpa_engine.runtime_config import requests_proxies
|
||||
|
||||
return requests_proxies()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _build_auth(cookie_data: str, user_agent: Optional[str] = None) -> tuple[DouyinAuth, str]:
|
||||
storage = json.loads(cookie_data)
|
||||
session = DouyinImSession.from_storage_state(storage or {})
|
||||
ua = resolve_user_agent(user_agent or session.user_agent or DEFAULT_USER_AGENT)
|
||||
auth = DouyinAuth()
|
||||
auth.perepare_auth(session.cookie_header(), session.web_protect_str, session.keys_str)
|
||||
auth.user_agent = ua
|
||||
auth.web_id = session.web_id or session.device_id or None
|
||||
return auth, ua
|
||||
|
||||
|
||||
def fetch_douyin_profile_sync(
|
||||
cookie_data: str,
|
||||
user_agent: Optional[str] = None,
|
||||
) -> dict[str, str]:
|
||||
detail = fetch_douyin_profile_detail_sync(cookie_data, user_agent)
|
||||
return {
|
||||
"uid": detail.get("uid") or "",
|
||||
"nickname": detail.get("nickname") or "",
|
||||
"avatar_url": detail.get("avatar_url") or "",
|
||||
}
|
||||
|
||||
|
||||
def fetch_douyin_profile_detail_sync(
|
||||
cookie_data: str,
|
||||
user_agent: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""抓取抖音账号详细资料（昵称/头像/UID/作品数/粉丝等）。"""
|
||||
result: dict[str, Any] = {
|
||||
"uid": "",
|
||||
"nickname": "",
|
||||
"avatar_url": "",
|
||||
"unique_id": "",
|
||||
"signature": "",
|
||||
"sec_user_id": "",
|
||||
"video_count": None,
|
||||
"follower_count": None,
|
||||
"following_count": None,
|
||||
"total_favorited": None,
|
||||
"favoriting_count": None,
|
||||
"fetched": False,
|
||||
"message": "",
|
||||
}
|
||||
try:
|
||||
auth, ua = _build_auth(cookie_data, user_agent)
|
||||
except Exception as exc:
|
||||
logger.warning(f"build auth for profile failed: {exc}")
|
||||
result["message"] = "Cookie 无效，无法解析登录凭证"
|
||||
return result
|
||||
|
||||
uid = auth.get_uid()
|
||||
if uid:
|
||||
result["uid"] = str(uid)
|
||||
|
||||
headers = {
|
||||
"User-Agent": ua,
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
}
|
||||
endpoints: list[tuple[str, dict[str, str]]] = [
|
||||
(
|
||||
"https://www.douyin.com/aweme/v1/web/query/user/",
|
||||
{
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"publish_video_strategy_type": "2",
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
},
|
||||
),
|
||||
(
|
||||
"https://creator.douyin.com/aweme/v1/creator/user/info/",
|
||||
{
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
},
|
||||
),
|
||||
(
|
||||
"https://www.douyin.com/aweme/v1/web/user/profile/self/",
|
||||
{
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
},
|
||||
),
|
||||
]
|
||||
|
||||
proxies = _requests_proxies()
|
||||
for url, base_params in endpoints:
|
||||
try:
|
||||
params = dict(base_params)
|
||||
query = splice_url(params)
|
||||
params["a_bogus"] = generate_a_bogus(query, user_agent=ua)
|
||||
resp = requests.get(
|
||||
url,
|
||||
params=params,
|
||||
headers=headers,
|
||||
cookies=auth.cookie,
|
||||
verify=False,
|
||||
timeout=12,
|
||||
proxies=proxies,
|
||||
)
|
||||
data = resp.json()
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
|
||||
if data.get("user_uid") and not result["uid"]:
|
||||
result["uid"] = str(data["user_uid"])
|
||||
|
||||
basic = _extract_profile_from_payload(data)
|
||||
if basic.get("uid") and not result["uid"]:
|
||||
result["uid"] = basic["uid"]
|
||||
if basic.get("nickname") and not result["nickname"]:
|
||||
result["nickname"] = basic["nickname"]
|
||||
if basic.get("avatar_url") and not result["avatar_url"]:
|
||||
result["avatar_url"] = basic["avatar_url"]
|
||||
|
||||
stats = _extract_detail_from_payload(data)
|
||||
for key, value in stats.items():
|
||||
if value is not None and value != "" and result.get(key) in (None, "", 0):
|
||||
result[key] = value
|
||||
|
||||
if result.get("video_count") is not None:
|
||||
result["fetched"] = True
|
||||
break
|
||||
if result["uid"] and result["nickname"] and result["avatar_url"]:
|
||||
result["fetched"] = True
|
||||
except Exception as exc:
|
||||
logger.debug(f"profile detail fetch failed for {url}: {exc}")
|
||||
|
||||
if not result["fetched"] and (result["uid"] or result["nickname"]):
|
||||
result["fetched"] = True
|
||||
|
||||
if not result["fetched"]:
|
||||
result["message"] = result["message"] or "未能从抖音获取账号资料，请确认 Cookie 有效"
|
||||
return result
|
||||
|
||||
|
||||
async def fetch_douyin_profile(
|
||||
cookie_data: str,
|
||||
user_agent: Optional[str] = None,
|
||||
) -> dict[str, str]:
|
||||
return await asyncio.to_thread(fetch_douyin_profile_sync, cookie_data, user_agent)
|
||||
|
||||
|
||||
async def fetch_douyin_profile_detail(
|
||||
cookie_data: str,
|
||||
user_agent: Optional[str] = None,
|
||||
) -> dict[str, Any]:
|
||||
return await asyncio.to_thread(fetch_douyin_profile_detail_sync, cookie_data, user_agent)
|
||||
|
||||
|
||||
async def fetch_douyin_user_videos(
|
||||
cookie_data: str,
|
||||
sec_user_id: str,
|
||||
user_agent: Optional[str] = None,
|
||||
max_count: int = 50,
|
||||
) -> dict[str, Any]:
|
||||
return await asyncio.to_thread(
|
||||
fetch_douyin_user_videos_sync,
|
||||
cookie_data,
|
||||
sec_user_id,
|
||||
user_agent,
|
||||
max_count,
|
||||
)
|
||||
|
||||
|
||||
def _douyin_profile_url(sec_user_id: Optional[str]) -> Optional[str]:
|
||||
sec = (sec_user_id or "").strip()
|
||||
if not sec:
|
||||
return None
|
||||
return f"https://www.douyin.com/user/{sec}"
|
||||
|
||||
|
||||
def _profile_detail_to_dict(
|
||||
profile: AccountProfileDetail | None,
|
||||
account: Account,
|
||||
videos: list[AccountVideo] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
video_rows = videos or []
|
||||
cached_count = len(video_rows)
|
||||
|
||||
def _row_media_type(row: AccountVideo) -> str:
|
||||
if row.media_type:
|
||||
return row.media_type
|
||||
return "video" if row.video_url else "image"
|
||||
|
||||
video_work_count = sum(1 for v in video_rows if _row_media_type(v) == "video")
|
||||
image_work_count = sum(1 for v in video_rows if _row_media_type(v) == "image")
|
||||
playable_count = video_work_count
|
||||
if profile:
|
||||
api_count = profile.video_count
|
||||
if cached_count:
|
||||
display_count = cached_count
|
||||
else:
|
||||
display_count = api_count
|
||||
return {
|
||||
"account_id": account.id,
|
||||
"uid": profile.uid or account.douyin_uid,
|
||||
"nickname": profile.nickname or account.username,
|
||||
"avatar_url": profile.avatar_url or account.avatar_url,
|
||||
"unique_id": profile.unique_id,
|
||||
"signature": profile.signature,
|
||||
"sec_user_id": profile.sec_user_id,
|
||||
"profile_url": _douyin_profile_url(profile.sec_user_id),
|
||||
"video_count": display_count,
|
||||
"video_count_douyin": api_count,
|
||||
"cached_work_count": cached_count,
|
||||
"playable_video_count": playable_count,
|
||||
"video_work_count": video_work_count,
|
||||
"image_work_count": image_work_count,
|
||||
"follower_count": profile.follower_count,
|
||||
"following_count": profile.following_count,
|
||||
"total_favorited": profile.total_favorited,
|
||||
"favoriting_count": profile.favoriting_count,
|
||||
"fetched": bool(profile.synced_at),
|
||||
"message": profile.sync_message,
|
||||
"synced_at": profile.synced_at.isoformat() if profile.synced_at else None,
|
||||
"profile_aweme_count": api_count,
|
||||
"videos": [
|
||||
{
|
||||
"id": v.id,
|
||||
"aweme_id": v.aweme_id,
|
||||
"title": v.title or "",
|
||||
"cover_url": v.cover_url,
|
||||
"video_url": v.video_url,
|
||||
"share_url": v.share_url,
|
||||
"create_time": v.create_time.isoformat() if v.create_time else None,
|
||||
"digg_count": v.digg_count,
|
||||
"comment_count": v.comment_count,
|
||||
"play_count": v.play_count,
|
||||
"media_type": v.media_type,
|
||||
}
|
||||
for v in video_rows
|
||||
],
|
||||
}
|
||||
return {
|
||||
"account_id": account.id,
|
||||
"uid": account.douyin_uid,
|
||||
"nickname": account.username,
|
||||
"avatar_url": account.avatar_url,
|
||||
"unique_id": None,
|
||||
"signature": None,
|
||||
"sec_user_id": None,
|
||||
"profile_url": None,
|
||||
"video_count": None,
|
||||
"video_count_douyin": None,
|
||||
"cached_work_count": 0,
|
||||
"playable_video_count": 0,
|
||||
"video_work_count": 0,
|
||||
"image_work_count": 0,
|
||||
"follower_count": None,
|
||||
"following_count": None,
|
||||
"total_favorited": None,
|
||||
"favoriting_count": None,
|
||||
"fetched": False,
|
||||
"message": "暂无本地资料，请点击刷新从抖音同步",
|
||||
"synced_at": None,
|
||||
"videos": [],
|
||||
}
|
||||
|
||||
|
||||
async def load_account_profile_from_db(
|
||||
db: AsyncSession,
|
||||
account: Account,
|
||||
) -> dict[str, Any]:
|
||||
profile = (
|
||||
await db.execute(
|
||||
select(AccountProfileDetail).where(AccountProfileDetail.account_id == account.id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
videos: list[AccountVideo] = []
|
||||
if profile:
|
||||
videos = list(
|
||||
(
|
||||
await db.execute(
|
||||
select(AccountVideo)
|
||||
.where(AccountVideo.account_id == account.id)
|
||||
.order_by(AccountVideo.sort_order.asc(), AccountVideo.id.asc())
|
||||
)
|
||||
).scalars().all()
|
||||
)
|
||||
return _profile_detail_to_dict(profile, account, videos)
|
||||
|
||||
|
||||
async def sync_account_profile_to_db(
|
||||
db: AsyncSession,
|
||||
account: Account,
|
||||
cookie_data: str,
|
||||
max_videos: int = 50,
|
||||
) -> dict[str, Any]:
|
||||
"""从抖音拉取资料与作品并写入本地数据库。"""
|
||||
detail = await fetch_douyin_profile_detail(cookie_data, account.user_agent)
|
||||
sec_user_id = (detail.get("sec_user_id") or "").strip()
|
||||
|
||||
if detail.get("fetched"):
|
||||
try:
|
||||
await apply_douyin_profile(db, account, cookie_data)
|
||||
except Exception as exc:
|
||||
logger.warning(f"apply douyin profile failed: {exc}")
|
||||
|
||||
if not sec_user_id and detail.get("uid"):
|
||||
try:
|
||||
auth, ua = _build_auth(cookie_data, account.user_agent)
|
||||
extra = _douyin_get_json(
|
||||
auth,
|
||||
ua,
|
||||
"https://www.douyin.com/aweme/v1/web/user/profile/other/",
|
||||
{
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"user_id": str(detail["uid"]),
|
||||
"sec_user_id": "",
|
||||
"publish_video_strategy_type": "2",
|
||||
"verifyFp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"fp": auth.cookie.get("s_v_web_id", "") if auth.cookie else "",
|
||||
"webid": generate_webid(auth, "https://www.douyin.com/"),
|
||||
"msToken": auth.msToken or generate_msToken(),
|
||||
},
|
||||
)
|
||||
if extra:
|
||||
sec_user_id = (_extract_detail_from_payload(extra).get("sec_user_id") or "").strip()
|
||||
except Exception as exc:
|
||||
logger.debug(f"resolve sec_user_id failed: {exc}")
|
||||
|
||||
video_result = await fetch_douyin_user_videos(
|
||||
cookie_data,
|
||||
sec_user_id,
|
||||
account.user_agent,
|
||||
max_count=max_videos,
|
||||
)
|
||||
|
||||
now = datetime.utcnow()
|
||||
profile = (
|
||||
await db.execute(
|
||||
select(AccountProfileDetail).where(AccountProfileDetail.account_id == account.id)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not profile:
|
||||
profile = AccountProfileDetail(account_id=account.id)
|
||||
db.add(profile)
|
||||
|
||||
profile.uid = detail.get("uid") or account.douyin_uid
|
||||
profile.nickname = detail.get("nickname") or account.username
|
||||
profile.avatar_url = detail.get("avatar_url") or account.avatar_url
|
||||
profile.unique_id = detail.get("unique_id") or None
|
||||
profile.signature = detail.get("signature") or None
|
||||
profile.sec_user_id = sec_user_id or None
|
||||
profile.follower_count = detail.get("follower_count")
|
||||
profile.following_count = detail.get("following_count")
|
||||
profile.total_favorited = detail.get("total_favorited")
|
||||
profile.favoriting_count = detail.get("favoriting_count")
|
||||
profile.synced_at = now
|
||||
|
||||
messages: list[str] = []
|
||||
if detail.get("message"):
|
||||
messages.append(str(detail["message"]))
|
||||
|
||||
fetched_videos = video_result.get("videos") or []
|
||||
await db.execute(delete(AccountVideo).where(AccountVideo.account_id == account.id))
|
||||
for idx, item in enumerate(fetched_videos):
|
||||
db.add(
|
||||
AccountVideo(
|
||||
account_id=account.id,
|
||||
aweme_id=item["aweme_id"],
|
||||
title=item.get("title"),
|
||||
cover_url=item.get("cover_url"),
|
||||
video_url=item.get("video_url"),
|
||||
share_url=item.get("share_url"),
|
||||
create_time=item.get("create_time"),
|
||||
digg_count=item.get("digg_count"),
|
||||
comment_count=item.get("comment_count"),
|
||||
play_count=item.get("play_count"),
|
||||
media_type=item.get("media_type"),
|
||||
sort_order=idx,
|
||||
synced_at=now,
|
||||
)
|
||||
)
|
||||
|
||||
profile.video_count = len(fetched_videos)
|
||||
if fetched_videos:
|
||||
profile.sync_message = "；".join(messages) if messages else None
|
||||
else:
|
||||
empty_msg = str(video_result.get("message") or "主页暂无已公开发布作品")
|
||||
if messages:
|
||||
messages.append(empty_msg)
|
||||
else:
|
||||
messages = [empty_msg]
|
||||
profile.sync_message = "；".join(messages)
|
||||
|
||||
await db.commit()
|
||||
await db.refresh(account)
|
||||
return await load_account_profile_from_db(db, account)
|
||||
|
||||
|
||||
async def _pick_unique_username(
|
||||
db: AsyncSession,
|
||||
account: Account,
|
||||
nickname: str,
|
||||
uid: str,
|
||||
) -> str:
|
||||
candidates: list[str] = []
|
||||
if nickname and uid:
|
||||
candidates.extend([nickname, f"{nickname}_{uid}"])
|
||||
elif nickname:
|
||||
candidates.append(nickname)
|
||||
elif uid:
|
||||
candidates.append(f"用户{uid}")
|
||||
candidates.append(f"{(nickname or '账号')}_{account.id}")
|
||||
|
||||
for candidate in candidates:
|
||||
name = candidate[:100]
|
||||
stmt = select(Account.id).where(Account.username == name, Account.id != account.id)
|
||||
conflict = (await db.execute(stmt)).scalar_one_or_none()
|
||||
if not conflict:
|
||||
return name
|
||||
return f"账号_{account.id}"
|
||||
|
||||
|
||||
async def apply_douyin_profile(
|
||||
db: AsyncSession,
|
||||
account: Account,
|
||||
cookie_data: str,
|
||||
) -> dict[str, str]:
|
||||
"""抓取并写入账号资料，返回抓取结果。"""
|
||||
profile = await fetch_douyin_profile(cookie_data, account.user_agent)
|
||||
uid = (profile.get("uid") or "").strip()
|
||||
nickname = (profile.get("nickname") or "").strip()
|
||||
avatar = (profile.get("avatar_url") or "").strip()
|
||||
|
||||
if uid:
|
||||
account.douyin_uid = uid
|
||||
if avatar:
|
||||
account.avatar_url = avatar
|
||||
if nickname or uid:
|
||||
account.username = await _pick_unique_username(db, account, nickname, uid)
|
||||
|
||||
return profile
|
||||
Reference in New Issue
Block a user