# -*- coding: utf-8 -*- """归档消息中短文本 protobuf 的保守解析器。""" from __future__ import annotations import hashlib import re from typing import Any _HEX_RE = re.compile(r"^[0-9a-fA-F]+$") _TEXT_TYPES = {"text", "文本"} _MAX_HEX_CHARS = 256 * 1024 def _read_varint(data: bytes, offset: int) -> tuple[int, int] | None: value = 0 shift = 0 for index in range(10): if offset >= len(data): return None current = data[offset] offset += 1 value |= (current & 0x7F) << shift if current < 0x80: return value, offset shift += 7 return None def _readable_utf8(data: bytes) -> str: if not data: return "" try: text = data.decode("utf-8") except UnicodeDecodeError: return "" if "\x00" in text or "\ufffd" in text: return "" if not all(character.isprintable() or character in "\r\n\t" for character in text): return "" return text.strip() def _protobuf_texts( data: bytes, *, depth: int = 0, max_depth: int = 8 ) -> list[tuple[int, str]]: if not data or depth > max_depth: return [] offset = 0 result: list[tuple[int, str]] = [] parsed_fields = 0 while offset < len(data): key_result = _read_varint(data, offset) if key_result is None: return [] key, offset = key_result field_number = key >> 3 wire_type = key & 7 if field_number <= 0: return [] parsed_fields += 1 if wire_type == 0: value_result = _read_varint(data, offset) if value_result is None: return [] _, offset = value_result elif wire_type == 1: offset += 8 elif wire_type == 2: length_result = _read_varint(data, offset) if length_result is None: return [] length, offset = length_result end = offset + length if length < 0 or end > len(data): return [] value = data[offset:end] offset = end readable = _readable_utf8(value) if readable: result.append((depth, readable)) result.extend(_protobuf_texts(value, depth=depth + 1, max_depth=max_depth)) elif wire_type == 5: offset += 4 else: return [] if offset > len(data): return [] return result if parsed_fields else [] def decode_hex_protobuf_text(value: Any, message_type: Any) -> str: """从十六进制 protobuf 中提取文本,只对明确的文本消息生效。""" if str(message_type or "").strip().casefold() not in _TEXT_TYPES: return "" raw = str(value or "").strip() if not raw or len(raw) > _MAX_HEX_CHARS: return "" decoded_parts: list[str] = [] for raw_part in raw.split("|"): part = re.sub(r"\s+", "", raw_part) if not part or len(part) % 2 or not _HEX_RE.fullmatch(part): return "" try: data = bytes.fromhex(part) except ValueError: return "" candidates = _protobuf_texts(data) if not candidates: return "" deepest = max(depth for depth, _ in candidates) texts = [text for depth, text in candidates if depth == deepest] for text in texts: if text and text not in decoded_parts: decoded_parts.append(text) return " | ".join(decoded_parts) def _protobuf_fields(data: bytes) -> dict[int, list[tuple[int, Any]]]: fields: dict[int, list[tuple[int, Any]]] = {} offset = 0 while offset < len(data): key_result = _read_varint(data, offset) if key_result is None: return {} key, offset = key_result field_number = key >> 3 wire_type = key & 7 if field_number <= 0: return {} if wire_type == 0: value_result = _read_varint(data, offset) if value_result is None: return {} value, offset = value_result elif wire_type == 1: end = offset + 8 if end > len(data): return {} value = data[offset:end] offset = end elif wire_type == 2: length_result = _read_varint(data, offset) if length_result is None: return {} length, offset = length_result end = offset + length if end > len(data): return {} value = data[offset:end] offset = end elif wire_type == 5: end = offset + 4 if end > len(data): return {} value = data[offset:end] offset = end else: return {} fields.setdefault(field_number, []).append((wire_type, value)) return fields def parse_file_message_metadata(value: Any, content_type: Any = None) -> dict[str, Any]: """解析企业微信 content_type=20 的文件卡片,不暴露远端令牌。""" try: if int(content_type) != 20: return {} except (TypeError, ValueError): return {} if isinstance(value, memoryview): value = value.tobytes() if not isinstance(value, bytes) or not value: return {} fields = _protobuf_fields(value) filename_values = fields.get(2) or [] size_values = fields.get(4) or [] checksum_values = fields.get(10) or [] if not filename_values or not size_values: return {} filename = _readable_utf8(filename_values[0][1]) filename = re.split(r"[\\/]", filename)[-1].strip()[:512] if not filename or "." not in filename or any(ord(char) < 32 for char in filename): return {} size_bytes = int(size_values[0][1]) if size_values[0][0] == 0 else 0 if size_bytes <= 0: return {} checksum = "" if checksum_values and checksum_values[0][0] == 2: checksum = _readable_utf8(checksum_values[0][1]).upper() if not re.fullmatch(r"[0-9A-F]{32,64}", checksum): checksum = "" remote_values = fields.get(1) or [] reference_hash = "" if remote_values and remote_values[0][0] == 2: reference_hash = hashlib.sha256(remote_values[0][1]).hexdigest() return { "original_filename": filename, "size_bytes": size_bytes, "checksum": checksum, "media_type": "file", "source_reference_sha256": reference_hash, } def file_message_content(metadata: dict[str, Any], *, cached: bool) -> str: filename = str(metadata.get("original_filename") or "未命名文件") size = int(metadata.get("size_bytes") or 0) if size >= 1024 * 1024 * 1024: size_label = f"{size / (1024 * 1024 * 1024):.2f} GB" elif size >= 1024 * 1024: size_label = f"{size / (1024 * 1024):.2f} MB" elif size >= 1024: size_label = f"{size / 1024:.1f} KB" else: size_label = f"{size} B" availability = "" if cached else ",源文件未缓存" return f"[文件] {filename}({size_label}{availability})" def _protobuf_text_field( fields: dict[int, list[tuple[int, Any]]], field_number: int ) -> str: values = fields.get(field_number) or [] if not values or values[0][0] != 2: return "" return _readable_utf8(values[0][1]) def parse_mini_program_metadata( value: Any, content_type: Any = None ) -> dict[str, Any]: """解析企业微信 content_type=78 的小程序卡片。""" try: if int(content_type) != 78: return {} except (TypeError, ValueError): return {} if isinstance(value, memoryview): value = value.tobytes() if not isinstance(value, bytes) or not value: return {} top_fields = _protobuf_fields(value) candidate: dict[int, list[tuple[int, Any]]] = {} for values in top_fields.values(): for wire_type, nested_value in values: if wire_type != 2 or not isinstance(nested_value, bytes): continue nested = _protobuf_fields(nested_value) app_id = _protobuf_text_field(nested, 2) page_path = _protobuf_text_field(nested, 3) if app_id.startswith("wx") and page_path: candidate = nested break if candidate: break if not candidate: return {} title = ( _protobuf_text_field(candidate, 7) or _protobuf_text_field(candidate, 8) or _protobuf_text_field(top_fields, 3) ) app_name = _protobuf_text_field(candidate, 10) page_path = _protobuf_text_field(candidate, 3) if not title or not page_path: return {} return { "title": title[:512], "app_name": app_name[:512], "app_id": _protobuf_text_field(candidate, 2)[:128], "username": _protobuf_text_field(candidate, 1)[:255], "page_path": page_path[:2048], "cover_url": ( _protobuf_text_field(candidate, 6) or _protobuf_text_field(top_fields, 2) )[:4096], } def mini_program_content(metadata: dict[str, Any]) -> str: lines = [f"[小程序] {str(metadata.get('title') or '未命名小程序')}"] if metadata.get("app_name"): lines.append(f"应用:{metadata['app_name']}") if metadata.get("app_id"): lines.append(f"AppID:{metadata['app_id']}") if metadata.get("page_path"): lines.append(f"页面:{metadata['page_path']}") return "\n".join(lines) # Database pages have already been decrypted before these bounded message parsers run. # Opaque media references are not text ciphertext and must never be model input. _MESSAGE_BYTES_LIMIT = 128 * 1024 _TEXT_CONTENT_TYPES = {0, 1, 2} _SYSTEM_CONTENT_TYPES = {11, 38, 76, 77, 79, 80, 81, 82, 83, 101, 132, 1002, 1011, 1012, 1017, 1022, 1025, 1043, 1988} _OPAQUE_MEDIA_REFERENCE = re.compile(r"\*[0-9]+\*[A-Za-z0-9+/=_-]{40,}") _MEDIA_PATH = re.compile(r"(?:[A-Za-z]:[\\/]|/(?:storage|data|sdcard|private|var)/)") def _message_bytes(value: Any) -> bytes | None: if isinstance(value, str): if len(value) > _MESSAGE_BYTES_LIMIT: return None value = value.encode("utf-8", errors="strict") elif isinstance(value, (bytearray, memoryview)): if (value.nbytes if isinstance(value, memoryview) else len(value)) > _MESSAGE_BYTES_LIMIT: return None value = bytes(value) if not isinstance(value, bytes) or len(value) > _MESSAGE_BYTES_LIMIT: return None return value def _message_utf8(value: bytes, *, strip: bool = True) -> str: try: text = value.decode("utf-8") except UnicodeDecodeError: return "" # ZWJ/ZWNJ are meaningful in emoji and written languages, not protobuf noise. if "\ufffd" in text or not all(c.isprintable() or c in "\r\n\t\u200c\u200d" for c in text): return "" return text.strip() if strip else text def _single_bytes_field(fields: dict[int, list[tuple[int, Any]]], number: int) -> bytes | None: values = fields.get(number) or [] if len(values) == 1 and values[0][0] == 2 and isinstance(values[0][1], bytes): return values[0][1] return None def _text_message_payload(raw: bytes) -> str: """Only known text envelopes, including short/repeated message segments. Native message_table: repeated 1 -> {1: flags, 2 -> {1: UTF-8}}. Older exports: {1: UTF-8}. No word-length/Chinese-character heuristics. """ top = _protobuf_fields(raw) if set(top) != {1} or len(top[1]) > 256: return "" parts = [] for wire, body in top[1]: if wire != 2 or not isinstance(body, bytes): return "" envelope = _protobuf_fields(body) if set(envelope) == {1, 2} and len(envelope[1]) == 1 and envelope[1][0][0] == 0: segment = _single_bytes_field(envelope, 2) if segment is None: return "" fields = _protobuf_fields(segment) if set(fields) != {1}: return "" text_bytes = _single_bytes_field(fields, 1) text = _message_utf8(text_bytes, strip=False) if text_bytes is not None else "" else: text = _message_utf8(body, strip=False) if not text: return "" parts.append(text) return "".join(parts).strip() def _safe_summary_text(value: Any) -> str: text = str(value or "").strip() if (not text or _OPAQUE_MEDIA_REFERENCE.search(text) or _MEDIA_PATH.search(text) or re.search(r"https?://", text, re.IGNORECASE) or text.startswith(("{", "[")) or re.fullmatch(r"[A-Za-z0-9+/=_-]{32,}", text)): return "" return text[:2000] def _summary_field(fields: dict[int, list[tuple[int, Any]]], number: int) -> str: raw = _single_bytes_field(fields, number) return _safe_summary_text(_message_utf8(raw)) if raw is not None else "" def parse_message_content(value: Any, content_type: Any, *, extra_content: Any = None) -> dict[str, str]: """Decode an already-decrypted database message before backend submission. decoded = text body; summary = typed non-text description; unsupported = unreadable text/unknown structure. Never guess compression/encryption, return hex, or recursively search arbitrary binary for readable fragments. extra_content contains device/session metadata, not a substitute body. VoiceMessageReader supplies transcription after this parser. """ del extra_content try: kind_number = int(content_type) except (TypeError, ValueError): kind_number = -1 named = str(content_type or "").strip().casefold() if named in {"text", "文本"}: kind_number = 2 try: raw = _message_bytes(value) except UnicodeEncodeError: raw = None def result(text: str, status: str, kind: str) -> dict[str, str]: return {"text": text, "status": status, "kind": kind} if kind_number in _TEXT_CONTENT_TYPES: if raw is None: return result("[文本暂无法解析]", "unsupported", "text") text = _text_message_payload(raw) or _message_utf8(raw) if text and not _OPAQUE_MEDIA_REFERENCE.search(text): return result(text, "decoded", "text") return result("[文本暂无法解析]", "unsupported", "text") if kind_number in {4, 16} or named in {"voice", "语音"}: return result("[语音待转文字]", "summary", "voice") if kind_number in {3, 14, 123} or named in {"image", "图片", "截图"}: return result("[图片]", "summary", "image") if kind_number == 5 or named in {"emoji", "表情"}: return result("[表情]", "summary", "emoji") if kind_number in {8, 17} or named in {"video", "视频", "视频号"}: return result("[视频]", "summary", "video") fields = _protobuf_fields(raw) if raw else {} if kind_number == 20: metadata = parse_file_message_metadata(raw, kind_number) if raw else {} if metadata and _safe_summary_text(metadata.get("original_filename")): # Metadata alone is not proof that the file itself was downloaded. return result(file_message_content(metadata, cached=False), "summary", "file") return result("[文件]", "summary", "file") if kind_number in {7, 23, 42, 55, 65, 111} or named in {"file", "文件"}: return result("[文件]", "summary", "file") if kind_number == 78: metadata = parse_mini_program_metadata(raw, kind_number) if raw else {} title = _safe_summary_text(metadata.get("title")) app_name = _safe_summary_text(metadata.get("app_name")) text = "[小程序]" + (" " + title if title else "") if app_name: text += "\n应用:" + app_name return result(text, "summary", "mini_program") if kind_number == 529: # Native moments: 4.5 is the caption, whereas 4.6.2 is a media token. nested_raw = _single_bytes_field(fields, 4) nested = _protobuf_fields(nested_raw) if nested_raw else {} caption = _summary_field(nested, 5) or _summary_field(fields, 2) return result("[朋友圈]" + (" " + caption if caption else ""), "summary", "moments") if kind_number in {6, 31} or named in {"link", "链接", "图文链接"}: nested_raw = _single_bytes_field(fields, 1) if kind_number == 31 else None title = _summary_field(_protobuf_fields(nested_raw), 1) if nested_raw else "" return result("[链接]" + (" " + title if title else ""), "summary", "link") if kind_number in _SYSTEM_CONTENT_TYPES: text = _safe_summary_text(_message_utf8(raw)) if raw else "" if not text and kind_number == 38: text = "\n".join(filter(None, (_summary_field(fields, 1), _summary_field(fields, 2)))) return result(text or "[系统消息]", "summary", "system") if kind_number in {503, 561, 565, 573, 64, 70}: title = _summary_field(fields, 1) if kind_number in {561, 573} else "" return result("[应用消息]" + (" " + title if title else ""), "summary", "card") labels = {9: "位置", 10: "名片", 12: "引用消息", 13: "红包", 15: "转账", 21: "日程", 22: "接龙", 26: "位置共享", 29: "视频通话", 40: "待办", 46: "话题", 47: "图文", 49: "待办", 51: "收藏", 53: "订阅通知", 56: "群公告", 57: "投票", 59: "回执", 61: "文件编辑", 62: "文件评论", 63: "合并转发", 66: "工作台", 67: "位置共享", 68: "视频通话", 69: "音频通话", 73: "微信好友", 74: "回复"} if kind_number in labels: return result("[" + labels[kind_number] + "]", "summary", "card") return result("[暂不支持的消息类型]", "unsupported", "unknown")