1954 lines
78 KiB
Python
1954 lines
78 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""桌面软件启动后的企业微信聊天记录自动归档桥接器。
|
|
|
|
主进程只负责启动一个隐藏的后台子进程，避免解密、扫描素材和上传 COS 阻塞 Qt。
|
|
子进程在隔离的模块空间里加载现有导出器，按服务端检查点读取消息，通过归档 API
|
|
批量写库。图片、语音、视频和文件先直传 COS，完成校验后再把 media_id 绑定消息。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from dataclasses import dataclass, replace
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
import requests
|
|
|
|
import backend_client
|
|
from archive_content_parser import (
|
|
parse_message_content,
|
|
decode_hex_protobuf_text,
|
|
file_message_content,
|
|
mini_program_content,
|
|
parse_file_message_metadata,
|
|
parse_mini_program_metadata,
|
|
)
|
|
from archive_conversation_filter import should_exclude_conversation
|
|
from runtime_paths import application_data_dir, is_frozen, resource_path
|
|
|
|
EXACT_EXPORTER_DIR = "2026-08-19-18-27-34"
|
|
DEFAULT_BATCH_SIZE = 500
|
|
DEFAULT_INTERVAL_MINUTES = 60
|
|
MIN_INTERVAL_MINUTES = 1
|
|
MAX_INTERVAL_MINUTES = 7 * 24 * 60
|
|
CONFIG_REFRESH_SECONDS = 30.0
|
|
MAX_RAW_BINARY_BYTES = 8 * 1024
|
|
SOURCE_TABLE = "message_table"
|
|
CONTENT_PARSER_VERSION = "desktop-plaintext-v1"
|
|
CONTENT_REPAIR_BATCHES = 2
|
|
CONTENT_REPAIR_BATCH_SIZE = 500
|
|
_START_LOCK = threading.Lock()
|
|
_START_THREAD: threading.Thread | None = None
|
|
_STOP_EVENT = threading.Event()
|
|
|
|
|
|
class ArchiveBackupError(RuntimeError):
|
|
"""自动归档无法安全继续。"""
|
|
|
|
|
|
def _truthy(value: Any, default: bool = True) -> bool:
|
|
text = str(value if value is not None else "").strip().lower()
|
|
if not text:
|
|
return default
|
|
return text not in {"0", "false", "no", "off", "disabled"}
|
|
|
|
|
|
def _discover_exporter_root() -> Path | None:
|
|
configured = os.environ.get("WECOM_ARCHIVE_EXPORTER_DIR", "").strip()
|
|
if configured:
|
|
candidate = Path(configured).expanduser().resolve()
|
|
return candidate if (candidate / "wxwork_export_final.py").is_file() else None
|
|
|
|
if is_frozen():
|
|
bundled = resource_path("archive_exporter")
|
|
return bundled if (bundled / "wxwork_export_final.py").is_file() else None
|
|
|
|
project_parent = Path(__file__).resolve().parent.parent
|
|
exact = project_parent / EXACT_EXPORTER_DIR
|
|
if (exact / "wxwork_export_final.py").is_file():
|
|
return exact
|
|
candidates = sorted(
|
|
(
|
|
item
|
|
for item in project_parent.iterdir()
|
|
if item.is_dir() and (item / "wxwork_export_final.py").is_file()
|
|
),
|
|
key=lambda item: item.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
return candidates[0] if candidates else None
|
|
|
|
|
|
def _normalized_path(value: str | os.PathLike[str]) -> Path | None:
|
|
try:
|
|
return Path(value).expanduser().resolve()
|
|
except (OSError, RuntimeError, TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _source_root_evidence(source_root: Path) -> tuple[float, int] | None:
|
|
"""验证 WXWork 根目录，返回最近数据库时间和可归档账号数。"""
|
|
|
|
if not source_root.is_dir():
|
|
return None
|
|
latest_mtime = 0.0
|
|
account_count = 0
|
|
try:
|
|
children = list(source_root.iterdir())
|
|
except OSError:
|
|
return None
|
|
for account_dir in children:
|
|
if not account_dir.is_dir():
|
|
continue
|
|
message_db = account_dir / "Data" / "message.db"
|
|
if not message_db.is_file():
|
|
continue
|
|
account_count += 1
|
|
try:
|
|
latest_mtime = max(latest_mtime, message_db.stat().st_mtime)
|
|
except OSError:
|
|
pass
|
|
if not account_count:
|
|
return None
|
|
return latest_mtime, account_count
|
|
|
|
|
|
def _windows_documents_dirs() -> list[Path]:
|
|
"""读取 Windows 当前用户的文档目录，包括被 OneDrive 重定向的情况。"""
|
|
|
|
values: list[Path] = []
|
|
if os.name != "nt":
|
|
return values
|
|
try:
|
|
import winreg
|
|
|
|
key_names = (
|
|
r"Software\Microsoft\Windows\CurrentVersion\Explorer\User Shell Folders",
|
|
r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders",
|
|
)
|
|
for key_name in key_names:
|
|
try:
|
|
with winreg.OpenKey(winreg.HKEY_CURRENT_USER, key_name) as key:
|
|
raw, _kind = winreg.QueryValueEx(key, "Personal")
|
|
except OSError:
|
|
continue
|
|
expanded = os.path.expandvars(str(raw or "").strip())
|
|
candidate = _normalized_path(expanded) if expanded else None
|
|
if candidate is not None:
|
|
values.append(candidate)
|
|
except (ImportError, OSError):
|
|
pass
|
|
return values
|
|
|
|
|
|
def _candidate_source_roots(preferred: Iterable[Path] = ()) -> list[Path]:
|
|
"""只扫描常见用户目录，不进行耗时且不可控的全盘搜索。"""
|
|
|
|
candidates: list[Path] = list(preferred)
|
|
candidates.append(Path.home() / "Documents" / "WXWork")
|
|
candidates.extend(path / "WXWork" for path in _windows_documents_dirs())
|
|
|
|
for variable in ("OneDrive", "OneDriveCommercial", "OneDriveConsumer"):
|
|
one_drive = os.environ.get(variable, "").strip()
|
|
if one_drive:
|
|
candidates.extend(
|
|
Path(one_drive) / folder / "WXWork"
|
|
for folder in ("Documents", "文档")
|
|
)
|
|
|
|
if os.name == "nt":
|
|
users_root = Path(os.environ.get("SystemDrive", "C:") + os.sep) / "Users"
|
|
try:
|
|
user_dirs = sorted(item for item in users_root.iterdir() if item.is_dir())
|
|
except OSError:
|
|
user_dirs = []
|
|
candidates.extend(item / "Documents" / "WXWork" for item in user_dirs)
|
|
|
|
result: list[Path] = []
|
|
seen: set[str] = set()
|
|
for value in candidates:
|
|
candidate = _normalized_path(value)
|
|
if candidate is None:
|
|
continue
|
|
identity = os.path.normcase(str(candidate))
|
|
if identity in seen:
|
|
continue
|
|
seen.add(identity)
|
|
result.append(candidate)
|
|
return result
|
|
|
|
|
|
def _exporter_state_path(exporter_root: Path, filename: str) -> Path:
|
|
if is_frozen() and exporter_root.resolve() == resource_path("archive_exporter").resolve():
|
|
# Persist settings outside the installation directory so updates keep them.
|
|
return application_data_dir() / filename
|
|
return exporter_root / filename
|
|
|
|
|
|
def _persist_source_root(exporter_root: Path, source_root: Path) -> None:
|
|
"""保存自动发现结果，同时保留导出器的其他 GUI 配置。"""
|
|
|
|
config_path = _exporter_state_path(exporter_root, "wxwork_gui_config.json")
|
|
try:
|
|
saved = json.loads(config_path.read_text(encoding="utf-8-sig"))
|
|
except (OSError, TypeError, ValueError):
|
|
saved = {}
|
|
if not isinstance(saved, dict):
|
|
saved = {}
|
|
selected = str(source_root)
|
|
if str(saved.get("db_dir") or "").strip() == selected:
|
|
return
|
|
saved["db_dir"] = selected
|
|
_atomic_json(config_path, saved)
|
|
|
|
|
|
def _configured_source_root(
|
|
exporter_root: Path, *, preferred: Iterable[Path] = ()
|
|
) -> Path:
|
|
"""校验保存路径；失效时自动发现并持久化最近使用的有效目录。"""
|
|
|
|
configured_paths: list[Path] = list(preferred)
|
|
environment_path = os.environ.get("WECOM_ARCHIVE_SOURCE_DIR", "").strip()
|
|
if environment_path:
|
|
candidate = _normalized_path(environment_path)
|
|
if candidate is not None:
|
|
configured_paths.append(candidate)
|
|
|
|
config_path = _exporter_state_path(exporter_root, "wxwork_gui_config.json")
|
|
try:
|
|
saved = json.loads(config_path.read_text(encoding="utf-8-sig"))
|
|
selected = str(saved.get("db_dir") or "").strip()
|
|
candidate = _normalized_path(selected) if selected else None
|
|
if candidate is not None:
|
|
configured_paths.append(candidate)
|
|
except (OSError, TypeError, ValueError):
|
|
pass
|
|
|
|
valid: list[tuple[float, int, int, Path]] = []
|
|
for order, candidate in enumerate(_candidate_source_roots(configured_paths)):
|
|
evidence = _source_root_evidence(candidate)
|
|
if evidence is not None:
|
|
latest_mtime, account_count = evidence
|
|
valid.append((latest_mtime, account_count, -order, candidate))
|
|
if valid:
|
|
source_root = max(valid, key=lambda item: item[:3])[3]
|
|
try:
|
|
_persist_source_root(exporter_root, source_root)
|
|
except OSError:
|
|
# 自动备份不应因 GUI 配置文件暂时不可写而停止。
|
|
pass
|
|
return source_root
|
|
|
|
if configured_paths:
|
|
fallback = _normalized_path(configured_paths[0])
|
|
if fallback is not None:
|
|
return fallback
|
|
return (Path.home() / "Documents" / "WXWork").resolve()
|
|
|
|
|
|
def _with_port(value: str, port: int) -> str:
|
|
parsed = urlparse(str(value or "").strip())
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
return ""
|
|
host = parsed.hostname
|
|
if ":" in host and not host.startswith("["):
|
|
host = f"[{host}]"
|
|
return urlunparse(
|
|
(parsed.scheme, f"{host}:{port}", "", "", "", "")
|
|
).rstrip("/")
|
|
|
|
|
|
def _api_candidates(explicit: str = "") -> list[str]:
|
|
"""安装版沿用登录云端；源码模式保留显式开发地址。"""
|
|
if is_frozen():
|
|
return [backend_client.resolve_backend_url()]
|
|
configured = explicit or os.environ.get("WECOM_ARCHIVE_API_URL", "").strip()
|
|
if configured:
|
|
return [backend_client.normalize_server_url(configured)]
|
|
try:
|
|
saved = backend_client.load_settings().get("server_url")
|
|
except Exception:
|
|
saved = None
|
|
server_url = backend_client.resolve_backend_url(saved)
|
|
values = [server_url]
|
|
if urlparse(server_url).hostname in {"127.0.0.1", "localhost", "::1"}:
|
|
values.append(_with_port(server_url, 8766))
|
|
return list(dict.fromkeys(value for value in values if value))
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AutoBackupConfig:
|
|
exporter_root: Path
|
|
source_root: Path
|
|
work_root: Path
|
|
api_urls: tuple[str, ...]
|
|
corp_scope_id: str = "local-wecom-corp"
|
|
batch_size: int = DEFAULT_BATCH_SIZE
|
|
|
|
@classmethod
|
|
def discover(cls, *, api_url: str = "") -> "AutoBackupConfig":
|
|
exporter_root = _discover_exporter_root()
|
|
if exporter_root is None:
|
|
raise ArchiveBackupError("未找到企业微信聊天导出程序目录")
|
|
source_root = _configured_source_root(exporter_root)
|
|
work_root = application_data_dir() / "archive_auto_backup"
|
|
batch_size = max(
|
|
1,
|
|
min(
|
|
5000,
|
|
int(os.environ.get("WECOM_ARCHIVE_BATCH_SIZE", DEFAULT_BATCH_SIZE)),
|
|
),
|
|
)
|
|
corp_scope = str(
|
|
os.environ.get("WECOM_ARCHIVE_CORP_SCOPE_ID", "local-wecom-corp")
|
|
).strip() or "local-wecom-corp"
|
|
return cls(
|
|
exporter_root=exporter_root,
|
|
source_root=source_root,
|
|
work_root=work_root,
|
|
api_urls=tuple(_api_candidates(api_url)),
|
|
corp_scope_id=corp_scope,
|
|
batch_size=batch_size,
|
|
)
|
|
|
|
|
|
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
temporary.write_text(
|
|
json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
)
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def _process_running(pid: int) -> bool:
|
|
if pid <= 0:
|
|
return False
|
|
if pid == os.getpid():
|
|
return True
|
|
if os.name == "nt":
|
|
try:
|
|
import ctypes
|
|
|
|
handle = ctypes.windll.kernel32.OpenProcess(0x1000, False, pid)
|
|
if not handle:
|
|
return False
|
|
try:
|
|
exit_code = ctypes.c_ulong()
|
|
if not ctypes.windll.kernel32.GetExitCodeProcess(
|
|
handle, ctypes.byref(exit_code)
|
|
):
|
|
return False
|
|
return exit_code.value == 259
|
|
finally:
|
|
ctypes.windll.kernel32.CloseHandle(handle)
|
|
except Exception:
|
|
return False
|
|
try:
|
|
os.kill(pid, 0)
|
|
return True
|
|
except PermissionError:
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
class _ProcessLock:
|
|
def __init__(self, path: Path):
|
|
self.path = path
|
|
self.acquired = False
|
|
|
|
def __enter__(self) -> "_ProcessLock":
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
for _attempt in range(2):
|
|
try:
|
|
descriptor = os.open(
|
|
self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY
|
|
)
|
|
except FileExistsError:
|
|
try:
|
|
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
|
owner = int(raw.get("pid") or 0)
|
|
except (OSError, TypeError, ValueError):
|
|
owner = 0
|
|
if owner and _process_running(owner):
|
|
raise ArchiveBackupError("另一个自动归档任务正在运行")
|
|
try:
|
|
self.path.unlink()
|
|
except OSError as exc:
|
|
raise ArchiveBackupError("无法清理失效的自动归档锁") from exc
|
|
continue
|
|
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
json.dump({"pid": os.getpid(), "started_at": time.time()}, handle)
|
|
self.acquired = True
|
|
return self
|
|
raise ArchiveBackupError("无法取得自动归档任务锁")
|
|
|
|
def __exit__(self, _kind, _value, _traceback) -> None:
|
|
if self.acquired:
|
|
try:
|
|
self.path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
class ArchiveApiClient:
|
|
"""使用已登录账号调用归档接口；消息批量入库，素材直传 COS。"""
|
|
|
|
def __init__(self, base_url: str, access_token: str, *, timeout: float = 30.0):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.timeout = timeout
|
|
self.archive_scope = ""
|
|
self.session = requests.Session()
|
|
self.session.headers.update(
|
|
{
|
|
"Accept": "application/json",
|
|
"User-Agent": "WeCom-RPA-Archive/1.0",
|
|
"Authorization": f"Bearer {access_token}",
|
|
}
|
|
)
|
|
self._uploaded: dict[str, str] = {}
|
|
self._uploaded_lock = threading.Lock()
|
|
self._thread_sessions = threading.local()
|
|
self._upload_slots = threading.BoundedSemaphore(
|
|
max(2, min(16, int(os.environ.get("WECOM_ARCHIVE_UPLOAD_SLOTS", "8"))))
|
|
)
|
|
|
|
@classmethod
|
|
def connect(cls, candidates: Iterable[str]) -> "ArchiveApiClient":
|
|
access_token = backend_client.desktop_access_token()
|
|
if not access_token:
|
|
raise ArchiveBackupError("请先登录桌面账号")
|
|
last_error = ""
|
|
for candidate in candidates:
|
|
client = cls(candidate, access_token)
|
|
settings = backend_client.load_settings()
|
|
account = settings.get("desktop_account") or {}
|
|
client.archive_scope = str(account.get("id") or settings.get("zyt_account") or "")
|
|
try:
|
|
response = client.session.get(
|
|
client.base_url + "/api/v2/health", timeout=2.5
|
|
)
|
|
if response.status_code == 200:
|
|
return client
|
|
last_error = f"{candidate} 返回 HTTP {response.status_code}"
|
|
except requests.RequestException as exc:
|
|
last_error = f"{candidate}: {exc}"
|
|
client.close()
|
|
raise ArchiveBackupError(f"归档 API 未启动或不可用：{last_error or '没有候选地址'}")
|
|
|
|
def close(self) -> None:
|
|
self.session.close()
|
|
|
|
def _thread_session(self) -> requests.Session:
|
|
current = getattr(self._thread_sessions, "session", None)
|
|
if current is None:
|
|
current = requests.Session()
|
|
current.headers.update(self.session.headers)
|
|
self._thread_sessions.session = current
|
|
return current
|
|
|
|
@staticmethod
|
|
def _response_error(response: requests.Response) -> str:
|
|
try:
|
|
payload = response.json()
|
|
return str(
|
|
payload.get("detail")
|
|
or payload.get("error")
|
|
or payload.get("message")
|
|
or f"HTTP {response.status_code}"
|
|
)
|
|
except (TypeError, ValueError):
|
|
return f"HTTP {response.status_code}"
|
|
|
|
def _json(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
*,
|
|
payload: dict[str, Any] | None = None,
|
|
params: dict[str, Any] | None = None,
|
|
timeout: float | None = None,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
response = self._thread_session().request(
|
|
method,
|
|
self.base_url + path,
|
|
json=payload,
|
|
params=params,
|
|
timeout=timeout or self.timeout,
|
|
)
|
|
except requests.RequestException as exc:
|
|
raise ArchiveBackupError(f"调用归档 API 失败：{exc}") from exc
|
|
if not response.ok:
|
|
raise ArchiveBackupError(
|
|
f"归档 API 拒绝请求：{self._response_error(response)}"
|
|
)
|
|
try:
|
|
result = response.json()
|
|
except ValueError as exc:
|
|
raise ArchiveBackupError("归档 API 返回了无效 JSON") from exc
|
|
if not isinstance(result, dict):
|
|
raise ArchiveBackupError("归档 API 返回结构不正确")
|
|
return result
|
|
|
|
def checkpoint(self, account: str) -> dict[str, Any]:
|
|
result = self._json(
|
|
"GET",
|
|
"/api/v2/archive/desktop/checkpoint",
|
|
params={"external_account_id": account, "source_table": SOURCE_TABLE},
|
|
)
|
|
checkpoint = result.get("checkpoint")
|
|
return checkpoint if isinstance(checkpoint, dict) else {}
|
|
|
|
def advance_checkpoint(
|
|
self,
|
|
account: str,
|
|
checkpoint: dict[str, Any],
|
|
*,
|
|
display_name: str = "",
|
|
corp_scope_id: str = "",
|
|
) -> dict[str, Any]:
|
|
result = self._json(
|
|
"POST",
|
|
"/api/v2/archive/desktop/checkpoint",
|
|
payload={
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": display_name or account,
|
|
"corp_scope_id": corp_scope_id,
|
|
},
|
|
"source_table": SOURCE_TABLE,
|
|
"checkpoint": checkpoint,
|
|
},
|
|
)
|
|
value = result.get("checkpoint")
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
def pending_attachments(self, account: str, limit: int = 500) -> list[str]:
|
|
result = self._json(
|
|
"GET",
|
|
"/api/v2/archive/desktop/pending-attachments",
|
|
params={"external_account_id": account, "limit": limit},
|
|
)
|
|
values = result.get("source_message_ids")
|
|
if not isinstance(values, list):
|
|
return []
|
|
return [str(value) for value in values if str(value or "").strip()]
|
|
|
|
def import_messages(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
return self._json(
|
|
"POST", "/api/v2/archive/desktop/imports/messages", payload=payload,
|
|
timeout=120,
|
|
)
|
|
|
|
def sync_metadata(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
return self._json(
|
|
"POST",
|
|
"/api/v2/archive/desktop/imports/metadata",
|
|
payload=payload,
|
|
timeout=120,
|
|
)
|
|
|
|
def upload_media(self, path: Path) -> str:
|
|
path = path.resolve()
|
|
digest = _sha256_file(path)
|
|
with self._uploaded_lock:
|
|
cached = self._uploaded.get(digest)
|
|
if cached:
|
|
return cached
|
|
mime_type = _mime_type(path)
|
|
prepared = self._json(
|
|
"POST",
|
|
"/api/v2/archive/desktop/media/prepare",
|
|
payload={
|
|
"sha256": digest,
|
|
"size_bytes": path.stat().st_size,
|
|
"mime_type": mime_type,
|
|
"original_filename": path.name,
|
|
},
|
|
timeout=60,
|
|
)
|
|
media = prepared.get("media") if isinstance(prepared.get("media"), dict) else {}
|
|
media_id = str(media.get("id") or "")
|
|
if not media_id:
|
|
raise ArchiveBackupError("COS 预上传接口没有返回素材 ID")
|
|
if not bool(prepared.get("reused")):
|
|
if prepared.get("upload_mode") == "multipart":
|
|
self._upload_multipart(path, media_id, prepared.get("multipart"))
|
|
else:
|
|
self._upload_single(path, media_id, prepared)
|
|
with self._uploaded_lock:
|
|
self._uploaded[digest] = media_id
|
|
return media_id
|
|
|
|
def _upload_single(
|
|
self, path: Path, media_id: str, prepared: dict[str, Any]
|
|
) -> None:
|
|
upload_url = str(prepared.get("upload_url") or "")
|
|
headers = prepared.get("required_headers")
|
|
if not upload_url or not isinstance(headers, dict):
|
|
raise ArchiveBackupError("COS 预上传接口没有返回完整上传信息")
|
|
try:
|
|
with self._upload_slots:
|
|
with path.open("rb") as handle:
|
|
upload = requests.put(
|
|
upload_url,
|
|
data=handle,
|
|
headers={str(key): str(value) for key, value in headers.items()},
|
|
timeout=(15, 600),
|
|
)
|
|
upload.raise_for_status()
|
|
except (OSError, requests.RequestException) as exc:
|
|
raise ArchiveBackupError(f"素材上传 COS 失败：{path.name}: {exc}") from exc
|
|
self._json(
|
|
"POST", f"/api/v2/archive/desktop/media/{media_id}/complete", timeout=120
|
|
)
|
|
|
|
def _upload_multipart(
|
|
self, path: Path, media_id: str, multipart: Any
|
|
) -> None:
|
|
if not isinstance(multipart, dict):
|
|
raise ArchiveBackupError("COS 分块上传信息不完整")
|
|
upload_id = str(multipart.get("upload_id") or "")
|
|
part_size = int(multipart.get("part_size") or 0)
|
|
raw_parts = multipart.get("parts")
|
|
if not upload_id or part_size <= 0 or not isinstance(raw_parts, list) or not raw_parts:
|
|
raise ArchiveBackupError("COS 分块上传信息不完整")
|
|
expected_by_number = {
|
|
int(item.get("part_number") or 0): item for item in raw_parts
|
|
}
|
|
|
|
def upload_part(item: dict[str, Any]) -> dict[str, Any]:
|
|
number = int(item.get("part_number") or 0)
|
|
size = int(item.get("size_bytes") or 0)
|
|
url = str(item.get("upload_url") or "")
|
|
if number <= 0 or size <= 0 or not url:
|
|
raise ArchiveBackupError("COS 分块参数不正确")
|
|
try:
|
|
with path.open("rb") as handle:
|
|
handle.seek((number - 1) * part_size)
|
|
data = handle.read(size)
|
|
if len(data) != size:
|
|
raise OSError("读取的文件分块长度不完整")
|
|
except OSError as exc:
|
|
raise ArchiveBackupError(
|
|
f"素材分块读取失败：{path.name} 第 {number} 块: {exc}"
|
|
) from exc
|
|
last_error: Exception | None = None
|
|
response: requests.Response | None = None
|
|
for attempt in range(1, 4):
|
|
try:
|
|
with self._upload_slots:
|
|
response = requests.put(url, data=data, timeout=(15, 180))
|
|
response.raise_for_status()
|
|
last_error = None
|
|
break
|
|
except requests.RequestException as exc:
|
|
last_error = exc
|
|
if attempt < 3:
|
|
time.sleep(attempt * 1.5)
|
|
if response is None or last_error is not None:
|
|
raise ArchiveBackupError(
|
|
f"素材分块上传 COS 失败：{path.name} 第 {number} 块: {last_error}"
|
|
) from last_error
|
|
etag = str(response.headers.get("ETag") or "").strip().strip('"')
|
|
if not etag:
|
|
raise ArchiveBackupError(f"COS 没有返回第 {number} 块的 ETag")
|
|
return {"part_number": number, "etag": etag}
|
|
|
|
completed: list[dict[str, Any]] = []
|
|
completed_numbers: set[int] = set()
|
|
for item in multipart.get("completed_parts") or []:
|
|
number = int(item.get("part_number") or 0)
|
|
etag = str(item.get("etag") or "").strip().strip('"')
|
|
recorded_size = int(item.get("size_bytes") or 0)
|
|
expected_item = expected_by_number.get(number)
|
|
if (
|
|
expected_item is not None
|
|
and etag
|
|
and recorded_size == int(expected_item.get("size_bytes") or 0)
|
|
):
|
|
completed.append({"part_number": number, "etag": etag})
|
|
completed_numbers.add(number)
|
|
missing_parts = [
|
|
dict(item)
|
|
for item in raw_parts
|
|
if int(item.get("part_number") or 0) not in completed_numbers
|
|
]
|
|
if missing_parts:
|
|
with ThreadPoolExecutor(
|
|
max_workers=min(6, len(missing_parts)),
|
|
thread_name_prefix="archive-cos-part",
|
|
) as pool:
|
|
futures = [pool.submit(upload_part, item) for item in missing_parts]
|
|
for future in as_completed(futures):
|
|
completed.append(future.result())
|
|
completed.sort(key=lambda item: item["part_number"])
|
|
self._json(
|
|
"POST",
|
|
f"/api/v2/archive/desktop/media/{media_id}/multipart-complete",
|
|
payload={"upload_id": upload_id, "parts": completed},
|
|
timeout=180,
|
|
)
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _mime_type(path: Path) -> str:
|
|
special = {
|
|
".silk": "audio/silk",
|
|
".amr": "audio/amr",
|
|
".m4a": "audio/mp4",
|
|
".webp": "image/webp",
|
|
}
|
|
return special.get(path.suffix.lower()) or mimetypes.guess_type(path.name)[0] or (
|
|
"application/octet-stream"
|
|
)
|
|
|
|
|
|
def _safe_raw(value: Any) -> Any:
|
|
if isinstance(value, memoryview):
|
|
value = value.tobytes()
|
|
if isinstance(value, bytes):
|
|
digest = hashlib.sha256(value).hexdigest()
|
|
return {"encoding": "external-binary", "size": len(value), "sha256": digest}
|
|
if value is None or isinstance(value, (bool, float, int, str)):
|
|
return value
|
|
return str(value)
|
|
|
|
|
|
def _raw_message_metadata(row: dict[str, Any]) -> dict[str, Any]:
|
|
"""Upload provenance, never protocol message bodies or binary blobs."""
|
|
result = {}
|
|
for key, value in row.items():
|
|
if key == "__archive_rowid":
|
|
continue
|
|
if ("content" in key.casefold() and key != "content_type") or key.casefold() in {"body", "payload", "msg_data"}:
|
|
data = value.tobytes() if isinstance(value, memoryview) else value
|
|
if not isinstance(data, bytes):
|
|
data = str(data or "").encode("utf-8")
|
|
result[key] = {"encoding": "local-body-digest", "size": len(data),
|
|
"sha256": hashlib.sha256(data).hexdigest()}
|
|
else:
|
|
result[key] = _safe_raw(value)
|
|
return result
|
|
|
|
|
|
def _valid_source_id(value: Any) -> str:
|
|
text = str(value if value is not None else "").strip()
|
|
return "" if text.lower() in {"", "0", "-1", "none", "null"} else text
|
|
|
|
|
|
def _table_columns(connection: sqlite3.Connection, table: str) -> list[str]:
|
|
return [
|
|
str(row[1])
|
|
for row in connection.execute(f'PRAGMA table_info("{table}")').fetchall()
|
|
]
|
|
|
|
|
|
def _tables(connection: sqlite3.Connection) -> set[str]:
|
|
return {
|
|
str(row[0])
|
|
for row in connection.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
).fetchall()
|
|
}
|
|
|
|
|
|
def _metadata(
|
|
decrypted: list[tuple[str, str, str]], exporter: Any
|
|
) -> tuple[dict[tuple[str, str], str], dict[tuple[str, str], str]]:
|
|
users: dict[tuple[str, str], str] = {}
|
|
conversations: dict[tuple[str, str], str] = {}
|
|
for database_path, database_name, account in decrypted:
|
|
if database_name not in {"user.db", "session.db"}:
|
|
continue
|
|
connection = exporter.connect_sqlite(database_path)
|
|
try:
|
|
available = _tables(connection)
|
|
if database_name == "user.db" and "user_table" in available:
|
|
columns = set(_table_columns(connection, "user_table"))
|
|
wanted = [
|
|
name for name in ("id", "name", "real_name", "account")
|
|
if name in columns
|
|
]
|
|
if "id" in wanted:
|
|
cursor = connection.execute(
|
|
f"SELECT {','.join(wanted)} FROM user_table"
|
|
)
|
|
values_list = cursor.fetchall()
|
|
cursor.close()
|
|
for values in values_list:
|
|
item = dict(zip(wanted, values))
|
|
user_id = str(item.get("id") or "")
|
|
label = str(
|
|
item.get("name")
|
|
or item.get("real_name")
|
|
or item.get("account")
|
|
or user_id
|
|
)
|
|
if user_id:
|
|
users[(account, user_id)] = label
|
|
if database_name == "session.db" and "conversation_table" in available:
|
|
columns = set(_table_columns(connection, "conversation_table"))
|
|
wanted = [
|
|
name
|
|
for name in ("id", "name", "roomname_remark", "session_id")
|
|
if name in columns
|
|
]
|
|
if "id" in wanted:
|
|
cursor = connection.execute(
|
|
f"SELECT {','.join(wanted)} FROM conversation_table"
|
|
)
|
|
values_list = cursor.fetchall()
|
|
cursor.close()
|
|
for values in values_list:
|
|
item = dict(zip(wanted, values))
|
|
conversation_id = str(item.get("id") or "")
|
|
label = str(
|
|
item.get("roomname_remark")
|
|
or item.get("name")
|
|
or item.get("session_id")
|
|
or ""
|
|
)
|
|
if conversation_id:
|
|
conversations[(account, conversation_id)] = label
|
|
finally:
|
|
connection.close()
|
|
return users, conversations
|
|
|
|
|
|
def _external_contact_identities(
|
|
decrypted: list[tuple[str, str, str]], exporter: Any
|
|
) -> dict[tuple[str, str], list[dict[str, Any]]]:
|
|
"""读取外部联系人身份，不把内部员工的 unionid 上传成患者候选。"""
|
|
|
|
result: dict[tuple[str, str], list[dict[str, Any]]] = {}
|
|
for database_path, database_name, account in decrypted:
|
|
if database_name != "user.db":
|
|
continue
|
|
connection = exporter.connect_sqlite(database_path)
|
|
try:
|
|
available = _tables(connection)
|
|
if not {"user_table", "external_user_relation_v3"}.issubset(available):
|
|
continue
|
|
user_columns = set(_table_columns(connection, "user_table"))
|
|
relation_columns = set(
|
|
_table_columns(connection, "external_user_relation_v3")
|
|
)
|
|
if "id" not in user_columns or "user_id" not in relation_columns:
|
|
continue
|
|
union_expr = "u.unionid" if "unionid" in user_columns else "''"
|
|
cursor = connection.execute(
|
|
f"""SELECT CAST(u.id AS TEXT) AS user_id,{union_expr} AS unionid
|
|
FROM user_table u
|
|
INNER JOIN external_user_relation_v3 r
|
|
ON CAST(r.user_id AS TEXT)=CAST(u.id AS TEXT)"""
|
|
)
|
|
rows = cursor.fetchall()
|
|
cursor.close()
|
|
for user_id_value, unionid_value in rows:
|
|
user_id = str(user_id_value or "").strip()
|
|
if not user_id:
|
|
continue
|
|
identities = [
|
|
{
|
|
"identity_type": "wecom_external_local_uid",
|
|
"external_id": user_id,
|
|
"verified": True,
|
|
"source": "wxwork_external_user_relation_v3",
|
|
}
|
|
]
|
|
unionid = str(unionid_value or "").strip()
|
|
if unionid:
|
|
identities.append(
|
|
{
|
|
"identity_type": "wechat_unionid",
|
|
"external_id": unionid,
|
|
"verified": True,
|
|
"source": "wxwork_user_table",
|
|
}
|
|
)
|
|
result[(account, user_id)] = identities
|
|
finally:
|
|
connection.close()
|
|
return result
|
|
|
|
|
|
def _conversation_display_name(
|
|
account: str,
|
|
conversation_id: str,
|
|
users: dict[tuple[str, str], str],
|
|
conversations: dict[tuple[str, str], str],
|
|
) -> str:
|
|
"""把 S:/M: 技术会话 ID 还原成对方昵称。"""
|
|
|
|
candidate = str(conversations.get((account, conversation_id)) or "").strip()
|
|
if candidate and candidate != conversation_id:
|
|
return candidate
|
|
if conversation_id.startswith("M:"):
|
|
peer_id = conversation_id[2:]
|
|
return users.get((account, peer_id), "") or f"微信用户 {peer_id}"
|
|
if conversation_id.startswith("S:"):
|
|
parts = [item for item in conversation_id[2:].split("_") if item]
|
|
peer_id = next((item for item in parts if item != account), "")
|
|
if not peer_id and parts:
|
|
peer_id = parts[0]
|
|
return users.get((account, peer_id), "") or f"企微用户 {peer_id}"
|
|
if conversation_id.startswith("Y:"):
|
|
return f"应用 {conversation_id[2:]}"
|
|
if conversation_id.startswith("O:"):
|
|
return f"服务 {conversation_id[2:]}"
|
|
return conversation_id
|
|
|
|
|
|
def _looks_like_binary_text(value: Any) -> bool:
|
|
text = str(value or "").strip()
|
|
compact = re.sub(r"[\s|]+", "", text)
|
|
return bool(
|
|
len(compact) >= 80
|
|
and len(compact) % 2 == 0
|
|
and re.fullmatch(r"[0-9a-fA-F]+", compact)
|
|
)
|
|
|
|
|
|
def _is_application_conversation(value: Any) -> bool:
|
|
return str(value or "").strip().upper().startswith("Y:")
|
|
|
|
|
|
def _is_excluded_conversation(
|
|
account: str,
|
|
conversation_id: Any,
|
|
users: dict[tuple[str, str], str],
|
|
conversations: dict[tuple[str, str], str],
|
|
) -> bool:
|
|
external_id = str(conversation_id or "").strip()
|
|
return should_exclude_conversation(
|
|
external_id,
|
|
_conversation_display_name(account, external_id, users, conversations),
|
|
)
|
|
|
|
|
|
def _semantic_content(
|
|
message_type: str,
|
|
parsed_content: Any,
|
|
media_reference: dict[str, Any],
|
|
voice_text: str = "",
|
|
) -> str:
|
|
if voice_text:
|
|
return voice_text
|
|
attachment_metadata = media_reference.get("attachment_metadata") or []
|
|
if attachment_metadata and isinstance(attachment_metadata[0], dict):
|
|
metadata = attachment_metadata[0]
|
|
return file_message_content(
|
|
metadata, cached=str(metadata.get("status") or "") == "cached"
|
|
)
|
|
content = str(parsed_content or "").strip()
|
|
decoded = decode_hex_protobuf_text(content, message_type)
|
|
if decoded:
|
|
return decoded
|
|
has_media = bool(
|
|
media_reference.get("matched_local_files")
|
|
or media_reference.get("urls")
|
|
or media_reference.get("filenames")
|
|
or media_reference.get("uuids")
|
|
)
|
|
if _looks_like_binary_text(content):
|
|
return f"[{message_type}]"
|
|
if not content and (
|
|
has_media
|
|
or message_type in {
|
|
"图片", "截图", "语音", "视频", "文件", "文件回复",
|
|
"文件预览", "文件分享", "群文件", "表情",
|
|
}
|
|
):
|
|
return f"[{message_type}]"
|
|
return content
|
|
|
|
|
|
def _message_status(row: dict[str, Any]) -> str:
|
|
for name in ("is_revoke", "revoke_status"):
|
|
if row.get(name) not in (None, "", 0, "0"):
|
|
return "revoked"
|
|
for name in ("is_deleted", "delete_status"):
|
|
if row.get(name) not in (None, "", 0, "0"):
|
|
return "deleted"
|
|
return "normal"
|
|
|
|
|
|
class IncrementalArchiveImporter:
|
|
def __init__(
|
|
self,
|
|
config: AutoBackupConfig,
|
|
api: ArchiveApiClient,
|
|
exporter: Any,
|
|
media_exporter: Any,
|
|
):
|
|
self.config = config
|
|
self.api = api
|
|
self.exporter = exporter
|
|
self.media_exporter = media_exporter
|
|
self._media_indexes: dict[str, dict[str, Any]] = {}
|
|
self.upload_workers = max(
|
|
1, min(12, int(os.environ.get("WECOM_ARCHIVE_UPLOAD_WORKERS", "6")))
|
|
)
|
|
|
|
def _media_for_row(
|
|
self, account: str, row: dict[str, Any]
|
|
) -> tuple[list[Path], dict[str, Any]]:
|
|
raw_content = row.get("content")
|
|
if isinstance(raw_content, memoryview):
|
|
raw_content = raw_content.tobytes()
|
|
file_metadata = parse_file_message_metadata(
|
|
raw_content, row.get("content_type")
|
|
)
|
|
if isinstance(raw_content, str):
|
|
raw_content = raw_content.encode("utf-8", errors="replace")
|
|
refs = self.media_exporter.extract_media_refs(raw_content)
|
|
if file_metadata:
|
|
refs["filenames"] = list(refs.get("filenames") or [])
|
|
filename = str(file_metadata["original_filename"])
|
|
if filename not in refs["filenames"]:
|
|
refs["filenames"].append(filename)
|
|
if not any(refs.values()):
|
|
return [], {}
|
|
if account not in self._media_indexes:
|
|
account_root = self.config.source_root / account
|
|
self._media_indexes[account] = self.media_exporter.build_cache_index(
|
|
str(account_root)
|
|
)
|
|
local_path, local_type, method = self.media_exporter.match_media(
|
|
refs, self._media_indexes[account]
|
|
)
|
|
matches: list[tuple[str, str, str]] = []
|
|
if local_path:
|
|
matches.append((str(local_path), str(local_type or ""), str(method or "")))
|
|
index = self._media_indexes[account]
|
|
for media_uuid in refs.get("uuids") or []:
|
|
found = (index.get("by_uuid") or {}).get(str(media_uuid).lower())
|
|
if found:
|
|
matches.append((str(found[0]), str(found[1] or ""), "uuid"))
|
|
for filename in refs.get("filenames") or []:
|
|
normalized = Path(str(filename)).name.lower()
|
|
found = (index.get("by_name") or {}).get(normalized)
|
|
if found:
|
|
matches.append((str(found[0]), str(found[1] or ""), "filename"))
|
|
media_paths: list[Path] = []
|
|
matched_files: list[dict[str, str]] = []
|
|
seen_paths: set[str] = set()
|
|
for matched_path, matched_type, matched_method in matches:
|
|
resolved = Path(matched_path).resolve()
|
|
dedup_path = os.path.normcase(str(resolved))
|
|
if dedup_path in seen_paths or not resolved.is_file():
|
|
continue
|
|
if file_metadata and resolved.stat().st_size != int(
|
|
file_metadata.get("size_bytes") or 0
|
|
):
|
|
continue
|
|
seen_paths.add(dedup_path)
|
|
media_paths.append(resolved)
|
|
matched_files.append(
|
|
{
|
|
"filename": resolved.name,
|
|
"local_media_type": matched_type,
|
|
"match_method": matched_method,
|
|
}
|
|
)
|
|
if file_metadata:
|
|
file_metadata["status"] = (
|
|
"cached" if media_paths else "source_not_cached"
|
|
)
|
|
return media_paths, {
|
|
"urls": list(refs.get("urls") or []),
|
|
"filenames": list(refs.get("filenames") or []),
|
|
"uuids": list(refs.get("uuids") or []),
|
|
"match_method": str(method or "unmatched"),
|
|
"matched_local_files": matched_files,
|
|
"attachment_metadata": [file_metadata] if file_metadata else [],
|
|
}
|
|
|
|
def _normalize(
|
|
self,
|
|
account: str,
|
|
row: dict[str, Any],
|
|
users: dict[tuple[str, str], str],
|
|
conversations: dict[tuple[str, str], str],
|
|
voice_texts: dict[tuple[str, str], str] | None = None,
|
|
) -> dict[str, Any]:
|
|
conversation_id = str(row.get("conversation_id") or "unknown")
|
|
sender_id = str(row.get("sender_id") or "")
|
|
source_message_id = next(
|
|
(
|
|
value
|
|
for value in (
|
|
_valid_source_id(row.get("server_id")),
|
|
_valid_source_id(row.get("client_id")),
|
|
_valid_source_id(row.get("message_id")),
|
|
)
|
|
if value
|
|
),
|
|
f"rowid:{int(row['__archive_rowid'])}",
|
|
)
|
|
media_paths, media_reference = self._media_for_row(account, row)
|
|
content_type = (
|
|
row.get("content_type")
|
|
if row.get("content_type") not in (None, "")
|
|
else row.get("msg_type", row.get("type"))
|
|
)
|
|
message_type = self.exporter.get_msg_type_name(content_type)
|
|
parsed = parse_message_content(row.get("content"), content_type,
|
|
extra_content=row.get("extra_content"))
|
|
content = str(parsed.get("text") or "")
|
|
mini_program_metadata = parse_mini_program_metadata(
|
|
row.get("content"), content_type
|
|
)
|
|
if mini_program_metadata:
|
|
message_type = "小程序"
|
|
media_reference["mini_program"] = mini_program_metadata
|
|
attachment_metadata = list(
|
|
media_reference.get("attachment_metadata") or []
|
|
)
|
|
if attachment_metadata:
|
|
message_type = "文件"
|
|
server_id = str(row.get("server_id") or "")
|
|
voice_texts = voice_texts or {}
|
|
voice_text = str(
|
|
voice_texts.get((account, server_id))
|
|
or voice_texts.get(("", server_id))
|
|
or ""
|
|
).strip()
|
|
if mini_program_metadata:
|
|
content = mini_program_content(mini_program_metadata)
|
|
else:
|
|
content = _semantic_content(
|
|
message_type, content, media_reference, voice_text=voice_text
|
|
)
|
|
sequence = next(
|
|
(
|
|
row.get(name)
|
|
for name in ("sequence", "sequence_no", "message_seq", "seq", "local_id")
|
|
if row.get(name) not in (None, "")
|
|
),
|
|
None,
|
|
)
|
|
return {
|
|
"source_table": SOURCE_TABLE,
|
|
"source_message_id": source_message_id,
|
|
"server_id": str(row.get("server_id") or ""),
|
|
"client_id": str(row.get("client_id") or ""),
|
|
"sequence_no": sequence,
|
|
"conversation": {
|
|
"external_id": conversation_id,
|
|
"name": _conversation_display_name(
|
|
account, conversation_id, users, conversations
|
|
),
|
|
},
|
|
"sender": {
|
|
"external_id": sender_id,
|
|
"display_name": users.get((account, sender_id), sender_id or "系统"),
|
|
"identity_type": "wecom_userid",
|
|
"scope_id": self.config.corp_scope_id,
|
|
"source": "wxwork_export_auto_backup",
|
|
},
|
|
"sent_at_epoch": row.get("send_time"),
|
|
"message_type": message_type,
|
|
"direction": "outbound" if sender_id and sender_id == account else "inbound",
|
|
"status": _message_status(row),
|
|
"content": str(content or ""),
|
|
"content_parser": CONTENT_PARSER_VERSION,
|
|
"content_parse_status": "decoded" if voice_text else parsed.get("status", "unsupported"),
|
|
"media_ids": [],
|
|
"attachment_metadata": attachment_metadata,
|
|
"mini_program": mini_program_metadata,
|
|
"_local_media_paths": [str(path) for path in media_paths],
|
|
"media_reference": media_reference,
|
|
"raw_fields": _raw_message_metadata(row),
|
|
"source_rowid": int(row["__archive_rowid"]),
|
|
}
|
|
|
|
def _upload_batch_media(self, messages: list[dict[str, Any]]) -> None:
|
|
path_to_messages: dict[str, list[dict[str, Any]]] = {}
|
|
for message in messages:
|
|
for raw_path in message.pop("_local_media_paths", []):
|
|
normalized = os.path.normcase(str(Path(raw_path).resolve()))
|
|
path_to_messages.setdefault(normalized, []).append(message)
|
|
if not path_to_messages:
|
|
return
|
|
with ThreadPoolExecutor(
|
|
max_workers=min(self.upload_workers, len(path_to_messages)),
|
|
thread_name_prefix="archive-cos-upload",
|
|
) as pool:
|
|
futures = {
|
|
pool.submit(self.api.upload_media, Path(path)): path
|
|
for path in path_to_messages
|
|
}
|
|
resolved_ids: dict[str, str] = {}
|
|
for future in as_completed(futures):
|
|
path = futures[future]
|
|
resolved_ids[path] = future.result()
|
|
for path, attached_messages in path_to_messages.items():
|
|
media_id = resolved_ids[path]
|
|
for message in attached_messages:
|
|
if media_id not in message["media_ids"]:
|
|
message["media_ids"].append(media_id)
|
|
|
|
def _repair_cached_attachments(
|
|
self,
|
|
connection: Any,
|
|
account: str,
|
|
columns: set[str],
|
|
users: dict[tuple[str, str], str],
|
|
conversations: dict[tuple[str, str], str],
|
|
voice_texts: dict[tuple[str, str], str],
|
|
) -> int:
|
|
source_ids = self.api.pending_attachments(account, limit=500)
|
|
if not source_ids:
|
|
return 0
|
|
identity_columns = [
|
|
name for name in ("server_id", "client_id", "message_id")
|
|
if name in columns
|
|
]
|
|
if not identity_columns:
|
|
return 0
|
|
repaired = 0
|
|
for start in range(0, len(source_ids), 400):
|
|
chunk = source_ids[start : start + 400]
|
|
placeholders = ",".join("?" for _ in chunk)
|
|
predicates = [
|
|
f"CAST({name} AS TEXT) IN ({placeholders})"
|
|
for name in identity_columns
|
|
]
|
|
parameters = tuple(
|
|
source_id
|
|
for _name in identity_columns
|
|
for source_id in chunk
|
|
)
|
|
cursor = connection.execute(
|
|
f"SELECT rowid AS __archive_rowid,* FROM {SOURCE_TABLE} "
|
|
f"WHERE {' OR '.join(predicates)} ORDER BY send_time,rowid",
|
|
parameters,
|
|
)
|
|
names = [str(item[0]) for item in cursor.description]
|
|
rows = [dict(zip(names, values)) for values in cursor.fetchall()]
|
|
cursor.close()
|
|
rows = [
|
|
row for row in rows
|
|
if not _is_excluded_conversation(
|
|
account,
|
|
row.get("conversation_id"),
|
|
users,
|
|
conversations,
|
|
)
|
|
]
|
|
messages = [
|
|
self._normalize(account, row, users, conversations, voice_texts)
|
|
for row in rows
|
|
]
|
|
messages = [
|
|
message for message in messages
|
|
if message.get("_local_media_paths")
|
|
]
|
|
if not messages:
|
|
continue
|
|
self._upload_batch_media(messages)
|
|
result = self.api.import_messages(
|
|
{
|
|
"batch_id": uuid.uuid4().hex,
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": users.get((account, account), account),
|
|
"corp_scope_id": self.config.corp_scope_id,
|
|
},
|
|
"source_table": SOURCE_TABLE,
|
|
"messages": messages,
|
|
}
|
|
)
|
|
repaired += len(messages) - int(result.get("errors") or 0)
|
|
return repaired
|
|
|
|
def _content_repair_state(
|
|
self, account: str, checkpoint: dict[str, Any]
|
|
) -> tuple[Path, dict[str, Any]]:
|
|
endpoint = str(getattr(self.api, "base_url", "") or self.config.api_urls[0])
|
|
parsed = urlparse(endpoint)
|
|
endpoint = urlunparse((parsed.scheme.lower(), parsed.netloc.lower(),
|
|
parsed.path.rstrip("/"), "", "", ""))
|
|
scope = [endpoint, str(getattr(self.api, "archive_scope", "")),
|
|
self.config.corp_scope_id, account, str(self.config.source_root.resolve()),
|
|
CONTENT_PARSER_VERSION]
|
|
identity = hashlib.sha256(json.dumps(scope).encode("utf-8")).hexdigest()
|
|
path = self.config.work_root / "content_repair" / (identity + ".json")
|
|
try:
|
|
state = json.loads(path.read_text(encoding="utf-8"))
|
|
if (state.get("parser") == CONTENT_PARSER_VERSION
|
|
and isinstance(state.get("upper"), dict)
|
|
and isinstance(state.get("cursor"), dict)):
|
|
return path, state
|
|
except (OSError, ValueError, TypeError, AttributeError):
|
|
pass
|
|
upper = {"send_time": float(checkpoint.get("send_time") or 0),
|
|
"rowid": int(checkpoint.get("rowid") or 0)}
|
|
state = {"parser": CONTENT_PARSER_VERSION, "upper": upper,
|
|
"cursor": {"send_time": 0.0, "rowid": 0}, "scanned": 0,
|
|
"reparsed": 0, "complete": not any(upper.values())}
|
|
_atomic_json(path, state)
|
|
return path, state
|
|
|
|
def _repair_content(
|
|
self, connection: Any, account: str, state_path: Path, state: dict[str, Any],
|
|
users: dict[tuple[str, str], str], conversations: dict[tuple[str, str], str],
|
|
voice_texts: dict[tuple[str, str], str],
|
|
) -> dict[str, Any]:
|
|
scanned = reparsed = 0
|
|
batch_size = max(1, min(self.config.batch_size, CONTENT_REPAIR_BATCH_SIZE))
|
|
for _ in range(CONTENT_REPAIR_BATCHES):
|
|
if state.get("complete"):
|
|
break
|
|
cursor, upper = state["cursor"], state["upper"]
|
|
query = connection.execute(
|
|
"""SELECT rowid AS __archive_rowid,* FROM message_table
|
|
WHERE (send_time>? OR (send_time=? AND rowid>?))
|
|
AND (send_time<? OR (send_time=? AND rowid<=?))
|
|
ORDER BY send_time,rowid LIMIT ?""",
|
|
(cursor["send_time"], cursor["send_time"], cursor["rowid"],
|
|
upper["send_time"], upper["send_time"], upper["rowid"], batch_size),
|
|
)
|
|
names = [str(item[0]) for item in query.description]
|
|
rows = [dict(zip(names, values)) for values in query.fetchall()]
|
|
query.close()
|
|
if not rows:
|
|
state["complete"] = True
|
|
_atomic_json(state_path, state)
|
|
break
|
|
messages = []
|
|
for row in rows:
|
|
if _is_excluded_conversation(account, row.get("conversation_id"), users, conversations):
|
|
continue
|
|
message = self._normalize(account, row, users, conversations, voice_texts)
|
|
# These formats were already normalized by the previous importer.
|
|
# Preserve cloud attachment availability when its local cache expired.
|
|
if message.get("attachment_metadata") or message.get("mini_program"):
|
|
continue
|
|
try:
|
|
content_type = int(row.get("content_type", row.get("msg_type", row.get("type"))))
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if content_type in {4, 16}:
|
|
# A transcript can outlive the desktop voice cache. Never
|
|
# replace an archived transcript with a pending placeholder.
|
|
if message.get("content_parse_status") != "decoded":
|
|
continue
|
|
elif content_type not in {0, 1, 2, 3, 14, 123, 5, 8, 17, 529, 6, 31}:
|
|
# Older system/card formats may already contain useful text.
|
|
# Reparse only known affected formats, not uncertain history.
|
|
continue
|
|
original = row.get("content")
|
|
if isinstance(original, memoryview):
|
|
original = original.tobytes()
|
|
if isinstance(original, bytes):
|
|
try:
|
|
original = original.decode("utf-8")
|
|
except UnicodeDecodeError:
|
|
original = None
|
|
if isinstance(original, str) and original.strip() == message["content"]:
|
|
continue
|
|
messages.append(message)
|
|
# Existing media links remain on the server. The attachment recovery
|
|
# queue independently handles missing local files and upload retries.
|
|
for message in messages:
|
|
message.pop("_local_media_paths", None)
|
|
message.pop("attachment_metadata", None)
|
|
if messages:
|
|
result = self.api.import_messages({
|
|
"batch_id": uuid.uuid4().hex,
|
|
"source_account": {"external_account_id": account,
|
|
"display_name": users.get((account, account), account),
|
|
"corp_scope_id": self.config.corp_scope_id},
|
|
"source_table": SOURCE_TABLE, "messages": messages,
|
|
})
|
|
if int(result.get("errors") or 0):
|
|
raise ArchiveBackupError("历史消息正文修复未全部成功，将保留断点重试")
|
|
scanned += len(rows)
|
|
reparsed += len(messages)
|
|
last = rows[-1]
|
|
state["cursor"] = {"send_time": float(last["send_time"]),
|
|
"rowid": int(last["__archive_rowid"])}
|
|
state["scanned"] = int(state.get("scanned") or 0) + len(rows)
|
|
state["reparsed"] = int(state.get("reparsed") or 0) + len(messages)
|
|
state["complete"] = len(rows) < batch_size or state["cursor"] == upper
|
|
_atomic_json(state_path, state)
|
|
return {"scanned": scanned, "reparsed": reparsed,
|
|
"pending": not state.get("complete", False)}
|
|
|
|
def run(
|
|
self, decrypted: list[tuple[str, str, str]]
|
|
) -> dict[str, Any]:
|
|
users, conversations = _metadata(decrypted, self.exporter)
|
|
external_identities = _external_contact_identities(decrypted, self.exporter)
|
|
voice_texts: dict[tuple[str, str], str] = {}
|
|
load_voice2text = getattr(self.exporter, "load_voice2text", None)
|
|
if callable(load_voice2text):
|
|
try:
|
|
loaded = load_voice2text(decrypted, log=lambda _message: None)
|
|
if isinstance(loaded, dict):
|
|
voice_texts = loaded
|
|
except Exception:
|
|
voice_texts = {}
|
|
summary = {
|
|
"accounts": 0,
|
|
"batches": 0,
|
|
"received": 0,
|
|
"inserted": 0,
|
|
"duplicates": 0,
|
|
"media": 0,
|
|
"attachments_repaired": 0,
|
|
"content_repair_scanned": 0,
|
|
"content_repaired": 0,
|
|
"content_repair_pending_accounts": 0,
|
|
"application_messages_skipped": 0,
|
|
"excluded_messages_skipped": 0,
|
|
"people_synced": 0,
|
|
"conversations_updated": 0,
|
|
}
|
|
accounts = sorted(
|
|
{account for _, database_name, account in decrypted if database_name == "message.db"}
|
|
)
|
|
for account in accounts:
|
|
people = [
|
|
{
|
|
"external_id": user_id,
|
|
"display_name": name,
|
|
"identity_type": "wecom_userid",
|
|
"scope_id": self.config.corp_scope_id,
|
|
"source": "wxwork_export_auto_backup",
|
|
"identities": external_identities.get((account, user_id), []),
|
|
}
|
|
for (user_account, user_id), name in users.items()
|
|
if user_account == account
|
|
]
|
|
conversation_rows = []
|
|
for conversation_account, conversation_id in conversations:
|
|
if conversation_account != account:
|
|
continue
|
|
name = _conversation_display_name(
|
|
account, conversation_id, users, conversations
|
|
)
|
|
if should_exclude_conversation(conversation_id, name):
|
|
continue
|
|
conversation_rows.append(
|
|
{"external_id": conversation_id, "name": name}
|
|
)
|
|
total = max(len(people), len(conversation_rows), 1)
|
|
for start in range(0, total, 5000):
|
|
synced = self.api.sync_metadata(
|
|
{
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": users.get((account, account), account),
|
|
"corp_scope_id": self.config.corp_scope_id,
|
|
},
|
|
"people": people[start : start + 5000],
|
|
"conversations": conversation_rows[start : start + 5000],
|
|
}
|
|
)
|
|
summary["people_synced"] += int(synced.get("people_synced") or 0)
|
|
summary["conversations_updated"] += int(
|
|
synced.get("conversations_updated") or 0
|
|
)
|
|
for database_path, database_name, account in decrypted:
|
|
if database_name != "message.db":
|
|
continue
|
|
connection = self.exporter.connect_sqlite(database_path)
|
|
try:
|
|
if SOURCE_TABLE not in _tables(connection):
|
|
continue
|
|
columns = _table_columns(connection, SOURCE_TABLE)
|
|
if "send_time" not in columns:
|
|
continue
|
|
checkpoint = self.api.checkpoint(account)
|
|
repair_path, repair_state = self._content_repair_state(account, checkpoint)
|
|
sent_at = float(checkpoint.get("send_time") or 0)
|
|
rowid = int(checkpoint.get("rowid") or 0)
|
|
summary["accounts"] += 1
|
|
while True:
|
|
cursor = connection.execute(
|
|
"""SELECT rowid AS __archive_rowid,* FROM message_table
|
|
WHERE send_time>? OR (send_time=? AND rowid>?)
|
|
ORDER BY send_time,rowid LIMIT ?""",
|
|
(sent_at, sent_at, rowid, self.config.batch_size),
|
|
)
|
|
names = [str(item[0]) for item in cursor.description]
|
|
values_list = cursor.fetchall()
|
|
cursor.close()
|
|
rows = [dict(zip(names, values)) for values in values_list]
|
|
if not rows:
|
|
break
|
|
importable_rows = [
|
|
row for row in rows
|
|
if not _is_excluded_conversation(
|
|
account,
|
|
row.get("conversation_id"),
|
|
users,
|
|
conversations,
|
|
)
|
|
]
|
|
summary["application_messages_skipped"] += (
|
|
sum(
|
|
1
|
|
for row in rows
|
|
if _is_application_conversation(
|
|
row.get("conversation_id")
|
|
)
|
|
)
|
|
)
|
|
summary["excluded_messages_skipped"] += (
|
|
len(rows) - len(importable_rows)
|
|
)
|
|
messages = [
|
|
self._normalize(
|
|
account, row, users, conversations, voice_texts
|
|
)
|
|
for row in importable_rows
|
|
]
|
|
# 一个批次的素材先并发上传并由服务端校验；全部成功后才提交消息
|
|
# 和检查点。任一素材失败，整批消息保持可重试状态。
|
|
self._upload_batch_media(messages)
|
|
last = rows[-1]
|
|
next_checkpoint = {
|
|
"send_time": float(last["send_time"]),
|
|
"rowid": int(last["__archive_rowid"]),
|
|
}
|
|
import_payload = {
|
|
"batch_id": uuid.uuid4().hex,
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": users.get((account, account), account),
|
|
"corp_scope_id": self.config.corp_scope_id,
|
|
},
|
|
"source_table": SOURCE_TABLE,
|
|
"checkpoint": next_checkpoint,
|
|
"messages": messages,
|
|
}
|
|
if messages:
|
|
result = self.api.import_messages(import_payload)
|
|
else:
|
|
self.api.advance_checkpoint(
|
|
account,
|
|
next_checkpoint,
|
|
display_name=users.get((account, account), account),
|
|
corp_scope_id=self.config.corp_scope_id,
|
|
)
|
|
result = {"inserted": 0, "duplicates": 0}
|
|
summary["batches"] += 1
|
|
summary["received"] += len(messages)
|
|
summary["inserted"] += int(result.get("inserted") or 0)
|
|
summary["duplicates"] += int(result.get("duplicates") or 0)
|
|
summary["media"] += sum(
|
|
len(message.get("media_ids") or []) for message in messages
|
|
)
|
|
sent_at = next_checkpoint["send_time"]
|
|
rowid = next_checkpoint["rowid"]
|
|
summary["attachments_repaired"] += self._repair_cached_attachments(
|
|
connection,
|
|
account,
|
|
columns,
|
|
users,
|
|
conversations,
|
|
voice_texts,
|
|
)
|
|
repair = self._repair_content(connection, account, repair_path, repair_state,
|
|
users, conversations, voice_texts)
|
|
summary["content_repair_scanned"] += repair["scanned"]
|
|
summary["content_repaired"] += repair["reparsed"]
|
|
summary["content_repair_pending_accounts"] += int(repair["pending"])
|
|
finally:
|
|
connection.close()
|
|
return summary
|
|
|
|
|
|
def _load_exporter_modules(exporter_root: Path) -> tuple[Any, Any]:
|
|
# Called only in the backup worker or the offline packaging self-check.
|
|
# The legacy exporter and the desktop reader have incompatible modules named
|
|
# wxwork_crypto. Keep exporter references private and restore the host imports.
|
|
names = ("wxwork_export_final", "wxwork_export_media", "wxwork_voice2text", "wxwork_crypto")
|
|
previous_modules = {name: sys.modules.get(name) for name in names}
|
|
previous_path = sys.path[:]
|
|
try:
|
|
for name in names:
|
|
sys.modules.pop(name, None)
|
|
sys.path.insert(0, str(exporter_root.resolve()))
|
|
exporter = importlib.import_module("wxwork_export_final")
|
|
media_exporter = importlib.import_module("wxwork_export_media")
|
|
return exporter, media_exporter
|
|
finally:
|
|
sys.path[:] = previous_path
|
|
for name, original in previous_modules.items():
|
|
if original is None:
|
|
sys.modules.pop(name, None)
|
|
else:
|
|
sys.modules[name] = original
|
|
|
|
|
|
def _load_exporter_keys(exporter_root: Path) -> dict[str, str]:
|
|
"""原导出器未指定编码；这里固定 UTF-8，且绝不输出密钥内容。"""
|
|
|
|
path = _exporter_state_path(exporter_root, "wxwork_keys.json")
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
except (OSError, TypeError, ValueError) as exc:
|
|
raise ArchiveBackupError("无法读取企业微信数据库密钥文件") from exc
|
|
keys = payload.get("keys") if isinstance(payload, dict) else None
|
|
result = {
|
|
str(account): str(key)
|
|
for account, key in (keys.items() if isinstance(keys, dict) else [])
|
|
if str(account).strip() and str(key).strip()
|
|
}
|
|
global_key = str(payload.get("global_key") or "") if isinstance(payload, dict) else ""
|
|
if global_key:
|
|
result.setdefault("*", global_key)
|
|
return result
|
|
|
|
|
|
def _prepare_archive_databases(config: AutoBackupConfig) -> list[tuple[str, str, str]]:
|
|
"""后台归档复用桌面密钥与 WAL 解密器。"""
|
|
import wxwork_db
|
|
from wxwork_local_setup import acquire_local_keys, select_source_directory
|
|
|
|
try:
|
|
keys = _load_exporter_keys(config.exporter_root)
|
|
except ArchiveBackupError:
|
|
keys = {}
|
|
try:
|
|
keys.update(wxwork_db.load_keys())
|
|
except ValueError:
|
|
pass
|
|
output = str(config.work_root / "decrypted")
|
|
decrypted = wxwork_db.decrypt_with_keys(str(config.source_root), output, keys, use_cache=True)
|
|
if any(name == "message.db" for _, name, _ in decrypted):
|
|
return decrypted
|
|
select_source_directory(str(config.source_root))
|
|
try:
|
|
acquire_local_keys()
|
|
except (OSError, ValueError) as exc:
|
|
raise ArchiveBackupError(f"自动获取数据库密钥失败：{exc}；可到微信消息库手动处理") from exc
|
|
keys.update(wxwork_db.load_keys())
|
|
decrypted = wxwork_db.decrypt_with_keys(str(config.source_root), output, keys, use_cache=True)
|
|
if not any(name == "message.db" for _, name, _ in decrypted):
|
|
raise ArchiveBackupError("没有解密出可用的企业微信 message.db，请检查数据目录和密钥")
|
|
return decrypted
|
|
|
|
|
|
def run_backup_once(config: AutoBackupConfig) -> dict[str, Any]:
|
|
# 子进程真正执行时再校验一次，覆盖软件启动后路径迁移或磁盘重连的情况。
|
|
detected_source_root = _configured_source_root(
|
|
config.exporter_root, preferred=(config.source_root,)
|
|
)
|
|
if detected_source_root != config.source_root:
|
|
config = replace(config, source_root=detected_source_root)
|
|
config.work_root.mkdir(parents=True, exist_ok=True)
|
|
status_path = config.work_root / "status.json"
|
|
with _ProcessLock(config.work_root / "backup.lock"):
|
|
started_at = time.time()
|
|
_atomic_json(
|
|
status_path,
|
|
{
|
|
"status": "running",
|
|
"started_at": started_at,
|
|
"source_root": str(config.source_root),
|
|
},
|
|
)
|
|
api: ArchiveApiClient | None = None
|
|
try:
|
|
if _source_root_evidence(config.source_root) is None:
|
|
raise ArchiveBackupError(
|
|
"自动检测未找到包含账号 Data/message.db 的企业微信数据目录；"
|
|
f"最后检查路径：{config.source_root}"
|
|
)
|
|
exporter, media_exporter = _load_exporter_modules(config.exporter_root)
|
|
decrypted = _prepare_archive_databases(config)
|
|
api = ArchiveApiClient.connect(config.api_urls)
|
|
summary = IncrementalArchiveImporter(
|
|
config, api, exporter, media_exporter
|
|
).run(decrypted)
|
|
result = {
|
|
"status": "completed",
|
|
"started_at": started_at,
|
|
"completed_at": time.time(),
|
|
"api_url": api.base_url,
|
|
"summary": summary,
|
|
}
|
|
_atomic_json(status_path, result)
|
|
return result
|
|
except Exception as exc:
|
|
failed = {
|
|
"status": "failed",
|
|
"started_at": started_at,
|
|
"completed_at": time.time(),
|
|
"error": str(exc)[:2000],
|
|
}
|
|
_atomic_json(status_path, failed)
|
|
raise
|
|
finally:
|
|
if api is not None:
|
|
api.close()
|
|
|
|
|
|
def _worker_arguments(config: AutoBackupConfig) -> list[str]:
|
|
arguments = [
|
|
"--exporter-root", str(config.exporter_root),
|
|
"--source-root", str(config.source_root),
|
|
"--work-root", str(config.work_root),
|
|
"--corp-scope-id", config.corp_scope_id,
|
|
"--batch-size", str(config.batch_size),
|
|
]
|
|
if config.api_urls:
|
|
arguments.extend(["--api-url", config.api_urls[0]])
|
|
return arguments
|
|
|
|
|
|
def _worker_command(config: AutoBackupConfig) -> list[str]:
|
|
if is_frozen():
|
|
return [sys.executable, "--archive-backup-worker", *_worker_arguments(config)]
|
|
return [
|
|
sys.executable,
|
|
"-u",
|
|
str(Path(__file__).resolve()),
|
|
"--worker",
|
|
*_worker_arguments(config),
|
|
]
|
|
|
|
|
|
def _run_worker_with_retries(config: AutoBackupConfig, log_path: Path) -> None:
|
|
"""执行一轮增量归档；短暂故障重试，但不影响下一次周期任务。"""
|
|
|
|
config.work_root.mkdir(parents=True, exist_ok=True)
|
|
retry_delays = (0, 15, 45)
|
|
for attempt, retry_delay in enumerate(retry_delays, start=1):
|
|
if retry_delay:
|
|
if _STOP_EVENT.wait(retry_delay):
|
|
return
|
|
with log_path.open("a", encoding="utf-8", buffering=1) as log:
|
|
log.write(
|
|
f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] 自动归档启动，第 {attempt} 次尝试\n"
|
|
)
|
|
creationflags = 0x08000000 if os.name == "nt" else 0
|
|
environment = os.environ.copy()
|
|
environment["PYTHONUTF8"] = "1"
|
|
try:
|
|
completed = subprocess.run(
|
|
_worker_command(config),
|
|
cwd=str(config.exporter_root),
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
env=environment,
|
|
creationflags=creationflags,
|
|
timeout=60 * 60 * 6,
|
|
check=False,
|
|
)
|
|
log.write(f"自动归档子进程退出：code={completed.returncode}\n")
|
|
if completed.returncode == 0:
|
|
return
|
|
except subprocess.TimeoutExpired:
|
|
log.write("自动归档超过 6 小时，已停止本次任务\n")
|
|
except Exception as exc:
|
|
log.write(f"自动归档无法启动：{exc}\n")
|
|
|
|
|
|
def _backup_policy() -> tuple[bool, int]:
|
|
"""读取桌面端已经落盘的云端策略，并把异常值收口到安全范围。"""
|
|
|
|
try:
|
|
import ai_config
|
|
|
|
enabled = bool(getattr(ai_config, "ARCHIVE_AUTO_BACKUP_ENABLED", True))
|
|
interval = int(
|
|
getattr(
|
|
ai_config,
|
|
"ARCHIVE_AUTO_BACKUP_INTERVAL_MINUTES",
|
|
DEFAULT_INTERVAL_MINUTES,
|
|
)
|
|
)
|
|
except (TypeError, ValueError):
|
|
enabled = True
|
|
interval = DEFAULT_INTERVAL_MINUTES
|
|
return enabled, max(MIN_INTERVAL_MINUTES, min(MAX_INTERVAL_MINUTES, interval))
|
|
|
|
|
|
def read_backup_status() -> dict[str, str]:
|
|
"""向界面提供同步结果；不把原始状态文件和签名地址暴露给页面。"""
|
|
if not _truthy(os.environ.get("WECOM_ARCHIVE_AUTO_BACKUP"), True) or not _backup_policy()[0]:
|
|
return {"status": "disabled", "message": "云端上传已关闭"}
|
|
waiting = {"status": "waiting", "message": "云端上传等待启动，登录后将自动同步"}
|
|
try:
|
|
saved = json.loads((application_data_dir() / "archive_auto_backup" / "status.json").read_text(encoding="utf-8-sig"))
|
|
except (OSError, ValueError):
|
|
return waiting
|
|
if not isinstance(saved, dict):
|
|
return waiting
|
|
status = saved.get("status")
|
|
if status == "running":
|
|
return {"status": "running", "message": "云端上传进行中，正在准备数据库或同步聊天记录及附件"}
|
|
if status == "failed":
|
|
error = str(backend_client.redact_diagnostic_value(str(saved.get("error") or "请查看运行日志")))
|
|
error = re.sub(r"https?://[^\s]+", lambda match: backend_client.diagnostic_url(match.group()).split("?", 1)[0], error)
|
|
return {"status": "failed", "message": "云端上传失败：" + error[:300]}
|
|
if status == "completed":
|
|
try:
|
|
completed = time.strftime("%m-%d %H:%M", time.localtime(float(saved.get("completed_at") or 0)))
|
|
summary = saved.get("summary") or {}
|
|
inserted = max(0, int(summary.get("inserted") or 0))
|
|
reparsed = max(0, int(summary.get("content_repaired") or 0))
|
|
scanned = max(0, int(summary.get("content_repair_scanned") or 0))
|
|
repair_pending = int(summary.get("content_repair_pending_accounts") or 0) > 0
|
|
except (TypeError, ValueError, OverflowError, OSError, AttributeError):
|
|
return waiting
|
|
repair_note = ""
|
|
if scanned or repair_pending:
|
|
repair_note = f"；本轮检查旧记录 {scanned} 条、重解析 {reparsed} 条"
|
|
if repair_pending:
|
|
repair_note += "，历史正文继续分批处理"
|
|
return {"status": "completed", "message": f"最近云端上传成功：{completed}，新增 {inserted} 条消息{repair_note}；后续自动增量同步"}
|
|
return waiting
|
|
|
|
|
|
def _refresh_cloud_policy(log: Any) -> None:
|
|
"""定期拉取后台配置；断网时沿用上次成功同步并落盘的值。"""
|
|
|
|
try:
|
|
backend_client.sync_cloud_config(timeout=8.0)
|
|
except Exception as exc:
|
|
log.write(f"后台备份策略同步失败，沿用本机缓存：{exc}\n")
|
|
|
|
|
|
def _launch_worker(config: AutoBackupConfig, initial_delay: float) -> None:
|
|
"""常驻调度线程：配置可热更新，关掉后不再创建归档子进程。"""
|
|
|
|
if initial_delay > 0 and _STOP_EVENT.wait(initial_delay):
|
|
return
|
|
config.work_root.mkdir(parents=True, exist_ok=True)
|
|
log_path = config.work_root / "auto_backup.log"
|
|
next_run: float | None = None
|
|
last_interval: int | None = None
|
|
|
|
while not _STOP_EVENT.is_set():
|
|
with log_path.open("a", encoding="utf-8", buffering=1) as log:
|
|
_refresh_cloud_policy(log)
|
|
enabled, interval = _backup_policy()
|
|
now = time.monotonic()
|
|
|
|
if not enabled:
|
|
if next_run is not None:
|
|
log.write("自动增量备份已由后台停用\n")
|
|
next_run = None
|
|
last_interval = interval
|
|
elif next_run is None:
|
|
next_run = now
|
|
last_interval = interval
|
|
log.write(f"自动增量备份立即启动，之后每 {interval} 分钟一次\n")
|
|
elif interval != last_interval:
|
|
next_run = min(next_run, now + interval * 60)
|
|
last_interval = interval
|
|
if enabled and next_run is not None and now >= next_run:
|
|
_run_worker_with_retries(config, log_path)
|
|
# 一轮可能运行很久，从完成时刻重新计时，避免任务重叠或追赶执行。
|
|
enabled, interval = _backup_policy()
|
|
last_interval = interval
|
|
next_run = time.monotonic() + interval * 60 if enabled else None
|
|
|
|
sleep_for = CONFIG_REFRESH_SECONDS
|
|
if next_run is not None:
|
|
sleep_for = min(sleep_for, max(1.0, next_run - time.monotonic()))
|
|
_STOP_EVENT.wait(sleep_for)
|
|
|
|
|
|
def start_auto_backup(*, initial_delay: float = 2.0) -> bool:
|
|
"""非阻塞启动周期调度器；同一个 GUI 进程只会挂一次。"""
|
|
|
|
global _START_THREAD
|
|
if not _truthy(os.environ.get("WECOM_ARCHIVE_AUTO_BACKUP"), True):
|
|
return False
|
|
if any(
|
|
argument in sys.argv
|
|
for argument in ("--packaging-self-check", "--qt-smoke-test", "--archive-backup-worker")
|
|
):
|
|
return False
|
|
try:
|
|
config = AutoBackupConfig.discover()
|
|
except Exception:
|
|
return False
|
|
with _START_LOCK:
|
|
if _START_THREAD is not None and _START_THREAD.is_alive():
|
|
return False
|
|
_STOP_EVENT.clear()
|
|
_START_THREAD = threading.Thread(
|
|
target=_launch_worker,
|
|
args=(config, max(0.0, float(initial_delay))),
|
|
name="archive-auto-backup-launcher",
|
|
daemon=True,
|
|
)
|
|
_START_THREAD.start()
|
|
return True
|
|
|
|
|
|
def stop_auto_backup() -> None:
|
|
"""退出账号时停止后续调度；当前 HTTP 请求会由服务端撤销令牌阻断。"""
|
|
|
|
_STOP_EVENT.set()
|
|
|
|
|
|
def worker_main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="企业微信聊天记录自动归档后台任务")
|
|
parser.add_argument("--worker", action="store_true")
|
|
parser.add_argument("--exporter-root", required=True)
|
|
parser.add_argument("--source-root", required=True)
|
|
parser.add_argument("--work-root", required=True)
|
|
parser.add_argument("--api-url", default="")
|
|
parser.add_argument("--corp-scope-id", default="local-wecom-corp")
|
|
parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE)
|
|
args = parser.parse_args(argv)
|
|
config = AutoBackupConfig(
|
|
exporter_root=Path(args.exporter_root).resolve(),
|
|
source_root=Path(args.source_root).resolve(),
|
|
work_root=Path(args.work_root).resolve(),
|
|
api_urls=tuple(_api_candidates(args.api_url)),
|
|
corp_scope_id=str(args.corp_scope_id or "local-wecom-corp"),
|
|
batch_size=max(1, min(5000, int(args.batch_size))),
|
|
)
|
|
try:
|
|
result = run_backup_once(config)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
return 0
|
|
except ArchiveBackupError as exc:
|
|
print(f"[自动归档失败] {exc}", flush=True)
|
|
return 2
|
|
except Exception as exc:
|
|
print(f"[自动归档异常] {type(exc).__name__}: {exc}", flush=True)
|
|
return 3
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(worker_main())
|