469 lines
18 KiB
Python
469 lines
18 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""归档消息中短文本 protobuf 的保守解析器。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import re
|
||
from typing import Any
|
||
|
||
|
||
_HEX_RE = re.compile(r"^[0-9a-fA-F]+$")
|
||
_TEXT_TYPES = {"text", "文本"}
|
||
_MAX_HEX_CHARS = 256 * 1024
|
||
|
||
|
||
def _read_varint(data: bytes, offset: int) -> tuple[int, int] | None:
|
||
value = 0
|
||
shift = 0
|
||
for index in range(10):
|
||
if offset >= len(data):
|
||
return None
|
||
current = data[offset]
|
||
offset += 1
|
||
value |= (current & 0x7F) << shift
|
||
if current < 0x80:
|
||
return value, offset
|
||
shift += 7
|
||
return None
|
||
|
||
|
||
def _readable_utf8(data: bytes) -> str:
|
||
if not data:
|
||
return ""
|
||
try:
|
||
text = data.decode("utf-8")
|
||
except UnicodeDecodeError:
|
||
return ""
|
||
if "\x00" in text or "\ufffd" in text:
|
||
return ""
|
||
if not all(character.isprintable() or character in "\r\n\t" for character in text):
|
||
return ""
|
||
return text.strip()
|
||
|
||
|
||
def _protobuf_texts(
|
||
data: bytes, *, depth: int = 0, max_depth: int = 8
|
||
) -> list[tuple[int, str]]:
|
||
if not data or depth > max_depth:
|
||
return []
|
||
offset = 0
|
||
result: list[tuple[int, str]] = []
|
||
parsed_fields = 0
|
||
while offset < len(data):
|
||
key_result = _read_varint(data, offset)
|
||
if key_result is None:
|
||
return []
|
||
key, offset = key_result
|
||
field_number = key >> 3
|
||
wire_type = key & 7
|
||
if field_number <= 0:
|
||
return []
|
||
parsed_fields += 1
|
||
if wire_type == 0:
|
||
value_result = _read_varint(data, offset)
|
||
if value_result is None:
|
||
return []
|
||
_, offset = value_result
|
||
elif wire_type == 1:
|
||
offset += 8
|
||
elif wire_type == 2:
|
||
length_result = _read_varint(data, offset)
|
||
if length_result is None:
|
||
return []
|
||
length, offset = length_result
|
||
end = offset + length
|
||
if length < 0 or end > len(data):
|
||
return []
|
||
value = data[offset:end]
|
||
offset = end
|
||
readable = _readable_utf8(value)
|
||
if readable:
|
||
result.append((depth, readable))
|
||
result.extend(_protobuf_texts(value, depth=depth + 1, max_depth=max_depth))
|
||
elif wire_type == 5:
|
||
offset += 4
|
||
else:
|
||
return []
|
||
if offset > len(data):
|
||
return []
|
||
return result if parsed_fields else []
|
||
|
||
|
||
def decode_hex_protobuf_text(value: Any, message_type: Any) -> str:
|
||
"""从十六进制 protobuf 中提取文本，只对明确的文本消息生效。"""
|
||
|
||
if str(message_type or "").strip().casefold() not in _TEXT_TYPES:
|
||
return ""
|
||
raw = str(value or "").strip()
|
||
if not raw or len(raw) > _MAX_HEX_CHARS:
|
||
return ""
|
||
decoded_parts: list[str] = []
|
||
for raw_part in raw.split("|"):
|
||
part = re.sub(r"\s+", "", raw_part)
|
||
if not part or len(part) % 2 or not _HEX_RE.fullmatch(part):
|
||
return ""
|
||
try:
|
||
data = bytes.fromhex(part)
|
||
except ValueError:
|
||
return ""
|
||
candidates = _protobuf_texts(data)
|
||
if not candidates:
|
||
return ""
|
||
deepest = max(depth for depth, _ in candidates)
|
||
texts = [text for depth, text in candidates if depth == deepest]
|
||
for text in texts:
|
||
if text and text not in decoded_parts:
|
||
decoded_parts.append(text)
|
||
return " | ".join(decoded_parts)
|
||
|
||
|
||
def _protobuf_fields(data: bytes) -> dict[int, list[tuple[int, Any]]]:
|
||
fields: dict[int, list[tuple[int, Any]]] = {}
|
||
offset = 0
|
||
while offset < len(data):
|
||
key_result = _read_varint(data, offset)
|
||
if key_result is None:
|
||
return {}
|
||
key, offset = key_result
|
||
field_number = key >> 3
|
||
wire_type = key & 7
|
||
if field_number <= 0:
|
||
return {}
|
||
if wire_type == 0:
|
||
value_result = _read_varint(data, offset)
|
||
if value_result is None:
|
||
return {}
|
||
value, offset = value_result
|
||
elif wire_type == 1:
|
||
end = offset + 8
|
||
if end > len(data):
|
||
return {}
|
||
value = data[offset:end]
|
||
offset = end
|
||
elif wire_type == 2:
|
||
length_result = _read_varint(data, offset)
|
||
if length_result is None:
|
||
return {}
|
||
length, offset = length_result
|
||
end = offset + length
|
||
if end > len(data):
|
||
return {}
|
||
value = data[offset:end]
|
||
offset = end
|
||
elif wire_type == 5:
|
||
end = offset + 4
|
||
if end > len(data):
|
||
return {}
|
||
value = data[offset:end]
|
||
offset = end
|
||
else:
|
||
return {}
|
||
fields.setdefault(field_number, []).append((wire_type, value))
|
||
return fields
|
||
|
||
|
||
def parse_file_message_metadata(value: Any, content_type: Any = None) -> dict[str, Any]:
|
||
"""解析企业微信 content_type=20 的文件卡片，不暴露远端令牌。"""
|
||
|
||
try:
|
||
if int(content_type) != 20:
|
||
return {}
|
||
except (TypeError, ValueError):
|
||
return {}
|
||
if isinstance(value, memoryview):
|
||
value = value.tobytes()
|
||
if not isinstance(value, bytes) or not value:
|
||
return {}
|
||
fields = _protobuf_fields(value)
|
||
filename_values = fields.get(2) or []
|
||
size_values = fields.get(4) or []
|
||
checksum_values = fields.get(10) or []
|
||
if not filename_values or not size_values:
|
||
return {}
|
||
filename = _readable_utf8(filename_values[0][1])
|
||
filename = re.split(r"[\\/]", filename)[-1].strip()[:512]
|
||
if not filename or "." not in filename or any(ord(char) < 32 for char in filename):
|
||
return {}
|
||
size_bytes = int(size_values[0][1]) if size_values[0][0] == 0 else 0
|
||
if size_bytes <= 0:
|
||
return {}
|
||
checksum = ""
|
||
if checksum_values and checksum_values[0][0] == 2:
|
||
checksum = _readable_utf8(checksum_values[0][1]).upper()
|
||
if not re.fullmatch(r"[0-9A-F]{32,64}", checksum):
|
||
checksum = ""
|
||
remote_values = fields.get(1) or []
|
||
reference_hash = ""
|
||
if remote_values and remote_values[0][0] == 2:
|
||
reference_hash = hashlib.sha256(remote_values[0][1]).hexdigest()
|
||
return {
|
||
"original_filename": filename,
|
||
"size_bytes": size_bytes,
|
||
"checksum": checksum,
|
||
"media_type": "file",
|
||
"source_reference_sha256": reference_hash,
|
||
}
|
||
|
||
|
||
def file_message_content(metadata: dict[str, Any], *, cached: bool) -> str:
|
||
filename = str(metadata.get("original_filename") or "未命名文件")
|
||
size = int(metadata.get("size_bytes") or 0)
|
||
if size >= 1024 * 1024 * 1024:
|
||
size_label = f"{size / (1024 * 1024 * 1024):.2f} GB"
|
||
elif size >= 1024 * 1024:
|
||
size_label = f"{size / (1024 * 1024):.2f} MB"
|
||
elif size >= 1024:
|
||
size_label = f"{size / 1024:.1f} KB"
|
||
else:
|
||
size_label = f"{size} B"
|
||
availability = "" if cached else "，源文件未缓存"
|
||
return f"[文件] {filename}（{size_label}{availability}）"
|
||
|
||
|
||
def _protobuf_text_field(
|
||
fields: dict[int, list[tuple[int, Any]]], field_number: int
|
||
) -> str:
|
||
values = fields.get(field_number) or []
|
||
if not values or values[0][0] != 2:
|
||
return ""
|
||
return _readable_utf8(values[0][1])
|
||
|
||
|
||
def parse_mini_program_metadata(
|
||
value: Any, content_type: Any = None
|
||
) -> dict[str, Any]:
|
||
"""解析企业微信 content_type=78 的小程序卡片。"""
|
||
|
||
try:
|
||
if int(content_type) != 78:
|
||
return {}
|
||
except (TypeError, ValueError):
|
||
return {}
|
||
if isinstance(value, memoryview):
|
||
value = value.tobytes()
|
||
if not isinstance(value, bytes) or not value:
|
||
return {}
|
||
top_fields = _protobuf_fields(value)
|
||
candidate: dict[int, list[tuple[int, Any]]] = {}
|
||
for values in top_fields.values():
|
||
for wire_type, nested_value in values:
|
||
if wire_type != 2 or not isinstance(nested_value, bytes):
|
||
continue
|
||
nested = _protobuf_fields(nested_value)
|
||
app_id = _protobuf_text_field(nested, 2)
|
||
page_path = _protobuf_text_field(nested, 3)
|
||
if app_id.startswith("wx") and page_path:
|
||
candidate = nested
|
||
break
|
||
if candidate:
|
||
break
|
||
if not candidate:
|
||
return {}
|
||
title = (
|
||
_protobuf_text_field(candidate, 7)
|
||
or _protobuf_text_field(candidate, 8)
|
||
or _protobuf_text_field(top_fields, 3)
|
||
)
|
||
app_name = _protobuf_text_field(candidate, 10)
|
||
page_path = _protobuf_text_field(candidate, 3)
|
||
if not title or not page_path:
|
||
return {}
|
||
return {
|
||
"title": title[:512],
|
||
"app_name": app_name[:512],
|
||
"app_id": _protobuf_text_field(candidate, 2)[:128],
|
||
"username": _protobuf_text_field(candidate, 1)[:255],
|
||
"page_path": page_path[:2048],
|
||
"cover_url": (
|
||
_protobuf_text_field(candidate, 6)
|
||
or _protobuf_text_field(top_fields, 2)
|
||
)[:4096],
|
||
}
|
||
|
||
|
||
def mini_program_content(metadata: dict[str, Any]) -> str:
|
||
lines = [f"[小程序] {str(metadata.get('title') or '未命名小程序')}"]
|
||
if metadata.get("app_name"):
|
||
lines.append(f"应用：{metadata['app_name']}")
|
||
if metadata.get("app_id"):
|
||
lines.append(f"AppID：{metadata['app_id']}")
|
||
if metadata.get("page_path"):
|
||
lines.append(f"页面：{metadata['page_path']}")
|
||
return "\n".join(lines)
|
||
|
||
|
||
# Database pages have already been decrypted before these bounded message parsers run.
|
||
# Opaque media references are not text ciphertext and must never be model input.
|
||
_MESSAGE_BYTES_LIMIT = 128 * 1024
|
||
_TEXT_CONTENT_TYPES = {0, 1, 2}
|
||
_SYSTEM_CONTENT_TYPES = {11, 38, 76, 77, 79, 80, 81, 82, 83, 101, 132,
|
||
1002, 1011, 1012, 1017, 1022, 1025, 1043, 1988}
|
||
_OPAQUE_MEDIA_REFERENCE = re.compile(r"\*[0-9]+\*[A-Za-z0-9+/=_-]{40,}")
|
||
_MEDIA_PATH = re.compile(r"(?:[A-Za-z]:[\\/]|/(?:storage|data|sdcard|private|var)/)")
|
||
|
||
|
||
def _message_bytes(value: Any) -> bytes | None:
|
||
if isinstance(value, str):
|
||
if len(value) > _MESSAGE_BYTES_LIMIT:
|
||
return None
|
||
value = value.encode("utf-8", errors="strict")
|
||
elif isinstance(value, (bytearray, memoryview)):
|
||
if (value.nbytes if isinstance(value, memoryview) else len(value)) > _MESSAGE_BYTES_LIMIT:
|
||
return None
|
||
value = bytes(value)
|
||
if not isinstance(value, bytes) or len(value) > _MESSAGE_BYTES_LIMIT:
|
||
return None
|
||
return value
|
||
|
||
|
||
def _message_utf8(value: bytes, *, strip: bool = True) -> str:
|
||
try:
|
||
text = value.decode("utf-8")
|
||
except UnicodeDecodeError:
|
||
return ""
|
||
# ZWJ/ZWNJ are meaningful in emoji and written languages, not protobuf noise.
|
||
if "\ufffd" in text or not all(c.isprintable() or c in "\r\n\t\u200c\u200d" for c in text):
|
||
return ""
|
||
return text.strip() if strip else text
|
||
|
||
|
||
def _single_bytes_field(fields: dict[int, list[tuple[int, Any]]], number: int) -> bytes | None:
|
||
values = fields.get(number) or []
|
||
if len(values) == 1 and values[0][0] == 2 and isinstance(values[0][1], bytes):
|
||
return values[0][1]
|
||
return None
|
||
|
||
|
||
def _text_message_payload(raw: bytes) -> str:
|
||
"""Only known text envelopes, including short/repeated message segments.
|
||
|
||
Native message_table: repeated 1 -> {1: flags, 2 -> {1: UTF-8}}.
|
||
Older exports: {1: UTF-8}. No word-length/Chinese-character heuristics.
|
||
"""
|
||
top = _protobuf_fields(raw)
|
||
if set(top) != {1} or len(top[1]) > 256:
|
||
return ""
|
||
parts = []
|
||
for wire, body in top[1]:
|
||
if wire != 2 or not isinstance(body, bytes):
|
||
return ""
|
||
envelope = _protobuf_fields(body)
|
||
if set(envelope) == {1, 2} and len(envelope[1]) == 1 and envelope[1][0][0] == 0:
|
||
segment = _single_bytes_field(envelope, 2)
|
||
if segment is None:
|
||
return ""
|
||
fields = _protobuf_fields(segment)
|
||
if set(fields) != {1}:
|
||
return ""
|
||
text_bytes = _single_bytes_field(fields, 1)
|
||
text = _message_utf8(text_bytes, strip=False) if text_bytes is not None else ""
|
||
else:
|
||
text = _message_utf8(body, strip=False)
|
||
if not text:
|
||
return ""
|
||
parts.append(text)
|
||
return "".join(parts).strip()
|
||
|
||
|
||
def _safe_summary_text(value: Any) -> str:
|
||
text = str(value or "").strip()
|
||
if (not text or _OPAQUE_MEDIA_REFERENCE.search(text) or _MEDIA_PATH.search(text)
|
||
or re.search(r"https?://", text, re.IGNORECASE)
|
||
or text.startswith(("{", "["))
|
||
or re.fullmatch(r"[A-Za-z0-9+/=_-]{32,}", text)):
|
||
return ""
|
||
return text[:2000]
|
||
|
||
|
||
def _summary_field(fields: dict[int, list[tuple[int, Any]]], number: int) -> str:
|
||
raw = _single_bytes_field(fields, number)
|
||
return _safe_summary_text(_message_utf8(raw)) if raw is not None else ""
|
||
|
||
|
||
def parse_message_content(value: Any, content_type: Any, *, extra_content: Any = None) -> dict[str, str]:
|
||
"""Decode an already-decrypted database message before backend submission.
|
||
|
||
decoded = text body; summary = typed non-text description; unsupported =
|
||
unreadable text/unknown structure. Never guess compression/encryption,
|
||
return hex, or recursively search arbitrary binary for readable fragments.
|
||
extra_content contains device/session metadata, not a substitute body.
|
||
VoiceMessageReader supplies transcription after this parser.
|
||
"""
|
||
del extra_content
|
||
try:
|
||
kind_number = int(content_type)
|
||
except (TypeError, ValueError):
|
||
kind_number = -1
|
||
named = str(content_type or "").strip().casefold()
|
||
if named in {"text", "文本"}:
|
||
kind_number = 2
|
||
try:
|
||
raw = _message_bytes(value)
|
||
except UnicodeEncodeError:
|
||
raw = None
|
||
|
||
def result(text: str, status: str, kind: str) -> dict[str, str]:
|
||
return {"text": text, "status": status, "kind": kind}
|
||
|
||
if kind_number in _TEXT_CONTENT_TYPES:
|
||
if raw is None:
|
||
return result("[文本暂无法解析]", "unsupported", "text")
|
||
text = _text_message_payload(raw) or _message_utf8(raw)
|
||
if text and not _OPAQUE_MEDIA_REFERENCE.search(text):
|
||
return result(text, "decoded", "text")
|
||
return result("[文本暂无法解析]", "unsupported", "text")
|
||
if kind_number in {4, 16} or named in {"voice", "语音"}:
|
||
return result("[语音待转文字]", "summary", "voice")
|
||
if kind_number in {3, 14, 123} or named in {"image", "图片", "截图"}:
|
||
return result("[图片]", "summary", "image")
|
||
if kind_number == 5 or named in {"emoji", "表情"}:
|
||
return result("[表情]", "summary", "emoji")
|
||
if kind_number in {8, 17} or named in {"video", "视频", "视频号"}:
|
||
return result("[视频]", "summary", "video")
|
||
|
||
fields = _protobuf_fields(raw) if raw else {}
|
||
if kind_number == 20:
|
||
metadata = parse_file_message_metadata(raw, kind_number) if raw else {}
|
||
if metadata and _safe_summary_text(metadata.get("original_filename")):
|
||
# Metadata alone is not proof that the file itself was downloaded.
|
||
return result(file_message_content(metadata, cached=False), "summary", "file")
|
||
return result("[文件]", "summary", "file")
|
||
if kind_number in {7, 23, 42, 55, 65, 111} or named in {"file", "文件"}:
|
||
return result("[文件]", "summary", "file")
|
||
if kind_number == 78:
|
||
metadata = parse_mini_program_metadata(raw, kind_number) if raw else {}
|
||
title = _safe_summary_text(metadata.get("title"))
|
||
app_name = _safe_summary_text(metadata.get("app_name"))
|
||
text = "[小程序]" + (" " + title if title else "")
|
||
if app_name:
|
||
text += "\n应用：" + app_name
|
||
return result(text, "summary", "mini_program")
|
||
if kind_number == 529:
|
||
# Native moments: 4.5 is the caption, whereas 4.6.2 is a media token.
|
||
nested_raw = _single_bytes_field(fields, 4)
|
||
nested = _protobuf_fields(nested_raw) if nested_raw else {}
|
||
caption = _summary_field(nested, 5) or _summary_field(fields, 2)
|
||
return result("[朋友圈]" + (" " + caption if caption else ""), "summary", "moments")
|
||
if kind_number in {6, 31} or named in {"link", "链接", "图文链接"}:
|
||
nested_raw = _single_bytes_field(fields, 1) if kind_number == 31 else None
|
||
title = _summary_field(_protobuf_fields(nested_raw), 1) if nested_raw else ""
|
||
return result("[链接]" + (" " + title if title else ""), "summary", "link")
|
||
if kind_number in _SYSTEM_CONTENT_TYPES:
|
||
text = _safe_summary_text(_message_utf8(raw)) if raw else ""
|
||
if not text and kind_number == 38:
|
||
text = "\n".join(filter(None, (_summary_field(fields, 1), _summary_field(fields, 2))))
|
||
return result(text or "[系统消息]", "summary", "system")
|
||
if kind_number in {503, 561, 565, 573, 64, 70}:
|
||
title = _summary_field(fields, 1) if kind_number in {561, 573} else ""
|
||
return result("[应用消息]" + (" " + title if title else ""), "summary", "card")
|
||
labels = {9: "位置", 10: "名片", 12: "引用消息", 13: "红包", 15: "转账",
|
||
21: "日程", 22: "接龙", 26: "位置共享", 29: "视频通话", 40: "待办",
|
||
46: "话题", 47: "图文", 49: "待办", 51: "收藏", 53: "订阅通知",
|
||
56: "群公告", 57: "投票", 59: "回执", 61: "文件编辑", 62: "文件评论",
|
||
63: "合并转发", 66: "工作台", 67: "位置共享", 68: "视频通话",
|
||
69: "音频通话", 73: "微信好友", 74: "回复"}
|
||
if kind_number in labels:
|
||
return result("[" + labels[kind_number] + "]", "summary", "card")
|
||
return result("[暂不支持的消息类型]", "unsupported", "unknown")
|