1410 lines
54 KiB
Python
1410 lines
54 KiB
Python
# -*- coding: utf-8 -*-
|
|
"""桌面软件启动后的企业微信聊天记录自动归档桥接器。
|
|
|
|
主进程只负责启动一个隐藏的后台子进程，避免解密、扫描素材和上传 COS 阻塞 Qt。
|
|
子进程在隔离的模块空间里加载现有导出器，按服务端检查点读取消息，通过归档 API
|
|
批量写库。图片、语音、视频和文件先直传 COS，完成校验后再把 media_id 绑定消息。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import importlib
|
|
import json
|
|
import mimetypes
|
|
import os
|
|
import re
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Iterable
|
|
from urllib.parse import urlparse, urlunparse
|
|
|
|
import requests
|
|
|
|
from archive_content_parser import (
|
|
decode_hex_protobuf_text,
|
|
file_message_content,
|
|
mini_program_content,
|
|
parse_file_message_metadata,
|
|
parse_mini_program_metadata,
|
|
)
|
|
import backend_client
|
|
from runtime_paths import application_data_dir, is_frozen
|
|
|
|
|
|
EXACT_EXPORTER_DIR = "2026-08-19-18-27-34"
|
|
DEFAULT_BATCH_SIZE = 500
|
|
MAX_RAW_BINARY_BYTES = 8 * 1024
|
|
SOURCE_TABLE = "message_table"
|
|
_START_LOCK = threading.Lock()
|
|
_START_THREAD: threading.Thread | None = None
|
|
|
|
|
|
class ArchiveBackupError(RuntimeError):
|
|
"""自动归档无法安全继续。"""
|
|
|
|
|
|
def _truthy(value: Any, default: bool = True) -> bool:
|
|
text = str(value if value is not None else "").strip().lower()
|
|
if not text:
|
|
return default
|
|
return text not in {"0", "false", "no", "off", "disabled"}
|
|
|
|
|
|
def _discover_exporter_root() -> Path | None:
|
|
configured = os.environ.get("WECOM_ARCHIVE_EXPORTER_DIR", "").strip()
|
|
if configured:
|
|
candidate = Path(configured).expanduser().resolve()
|
|
return candidate if (candidate / "wxwork_export_final.py").is_file() else None
|
|
|
|
project_parent = Path(__file__).resolve().parent.parent
|
|
exact = project_parent / EXACT_EXPORTER_DIR
|
|
if (exact / "wxwork_export_final.py").is_file():
|
|
return exact
|
|
candidates = sorted(
|
|
(
|
|
item
|
|
for item in project_parent.iterdir()
|
|
if item.is_dir() and (item / "wxwork_export_final.py").is_file()
|
|
),
|
|
key=lambda item: item.stat().st_mtime,
|
|
reverse=True,
|
|
)
|
|
return candidates[0] if candidates else None
|
|
|
|
|
|
def _configured_source_root(exporter_root: Path) -> Path:
|
|
configured = os.environ.get("WECOM_ARCHIVE_SOURCE_DIR", "").strip()
|
|
if configured:
|
|
return Path(configured).expanduser().resolve()
|
|
config_path = exporter_root / "wxwork_gui_config.json"
|
|
try:
|
|
saved = json.loads(config_path.read_text(encoding="utf-8-sig"))
|
|
selected = str(saved.get("db_dir") or "").strip()
|
|
if selected:
|
|
return Path(selected).expanduser().resolve()
|
|
except (OSError, TypeError, ValueError):
|
|
pass
|
|
return (Path.home() / "Documents" / "WXWork").resolve()
|
|
|
|
|
|
def _with_port(value: str, port: int) -> str:
|
|
parsed = urlparse(str(value or "").strip())
|
|
if parsed.scheme not in {"http", "https"} or not parsed.hostname:
|
|
return ""
|
|
host = parsed.hostname
|
|
if ":" in host and not host.startswith("["):
|
|
host = f"[{host}]"
|
|
return urlunparse(
|
|
(parsed.scheme, f"{host}:{port}", "", "", "", "")
|
|
).rstrip("/")
|
|
|
|
|
|
def _api_candidates(explicit: str = "") -> list[str]:
|
|
values: list[str] = []
|
|
configured = explicit or os.environ.get("WECOM_ARCHIVE_API_URL", "").strip()
|
|
if configured:
|
|
values.append(configured.rstrip("/"))
|
|
try:
|
|
server_url = str(backend_client.load_settings().get("server_url") or "").rstrip("/")
|
|
except Exception:
|
|
server_url = ""
|
|
if server_url:
|
|
parsed = urlparse(server_url)
|
|
if parsed.port == 8766:
|
|
values.append(server_url)
|
|
if parsed.hostname in {"127.0.0.1", "localhost", "::1"}:
|
|
values.append(_with_port(server_url, 8766))
|
|
values.append("http://127.0.0.1:8766")
|
|
result: list[str] = []
|
|
for value in values:
|
|
normalized = str(value or "").strip().rstrip("/")
|
|
if normalized and normalized not in result:
|
|
result.append(normalized)
|
|
return result
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AutoBackupConfig:
|
|
exporter_root: Path
|
|
source_root: Path
|
|
work_root: Path
|
|
api_urls: tuple[str, ...]
|
|
corp_scope_id: str = "local-wecom-corp"
|
|
batch_size: int = DEFAULT_BATCH_SIZE
|
|
|
|
@classmethod
|
|
def discover(cls, *, api_url: str = "") -> "AutoBackupConfig":
|
|
exporter_root = _discover_exporter_root()
|
|
if exporter_root is None:
|
|
raise ArchiveBackupError("未找到企业微信聊天导出程序目录")
|
|
source_root = _configured_source_root(exporter_root)
|
|
work_root = application_data_dir() / "archive_auto_backup"
|
|
batch_size = max(
|
|
1,
|
|
min(
|
|
5000,
|
|
int(os.environ.get("WECOM_ARCHIVE_BATCH_SIZE", DEFAULT_BATCH_SIZE)),
|
|
),
|
|
)
|
|
corp_scope = str(
|
|
os.environ.get("WECOM_ARCHIVE_CORP_SCOPE_ID", "local-wecom-corp")
|
|
).strip() or "local-wecom-corp"
|
|
return cls(
|
|
exporter_root=exporter_root,
|
|
source_root=source_root,
|
|
work_root=work_root,
|
|
api_urls=tuple(_api_candidates(api_url)),
|
|
corp_scope_id=corp_scope,
|
|
batch_size=batch_size,
|
|
)
|
|
|
|
|
|
def _atomic_json(path: Path, value: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
temporary.write_text(
|
|
json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8"
|
|
)
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def _process_running(pid: int) -> bool:
|
|
if pid <= 0:
|
|
return False
|
|
if pid == os.getpid():
|
|
return True
|
|
if os.name == "nt":
|
|
try:
|
|
import ctypes
|
|
|
|
handle = ctypes.windll.kernel32.OpenProcess(0x1000, False, pid)
|
|
if not handle:
|
|
return False
|
|
try:
|
|
exit_code = ctypes.c_ulong()
|
|
if not ctypes.windll.kernel32.GetExitCodeProcess(
|
|
handle, ctypes.byref(exit_code)
|
|
):
|
|
return False
|
|
return exit_code.value == 259
|
|
finally:
|
|
ctypes.windll.kernel32.CloseHandle(handle)
|
|
except Exception:
|
|
return False
|
|
try:
|
|
os.kill(pid, 0)
|
|
return True
|
|
except PermissionError:
|
|
return True
|
|
except OSError:
|
|
return False
|
|
|
|
|
|
class _ProcessLock:
|
|
def __init__(self, path: Path):
|
|
self.path = path
|
|
self.acquired = False
|
|
|
|
def __enter__(self) -> "_ProcessLock":
|
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
|
for _attempt in range(2):
|
|
try:
|
|
descriptor = os.open(
|
|
self.path, os.O_CREAT | os.O_EXCL | os.O_WRONLY
|
|
)
|
|
except FileExistsError:
|
|
try:
|
|
raw = json.loads(self.path.read_text(encoding="utf-8"))
|
|
owner = int(raw.get("pid") or 0)
|
|
except (OSError, TypeError, ValueError):
|
|
owner = 0
|
|
if owner and _process_running(owner):
|
|
raise ArchiveBackupError("另一个自动归档任务正在运行")
|
|
try:
|
|
self.path.unlink()
|
|
except OSError as exc:
|
|
raise ArchiveBackupError("无法清理失效的自动归档锁") from exc
|
|
continue
|
|
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
json.dump({"pid": os.getpid(), "started_at": time.time()}, handle)
|
|
self.acquired = True
|
|
return self
|
|
raise ArchiveBackupError("无法取得自动归档任务锁")
|
|
|
|
def __exit__(self, _kind, _value, _traceback) -> None:
|
|
if self.acquired:
|
|
try:
|
|
self.path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
class ArchiveApiClient:
|
|
"""只调用本机归档机器接口；消息批量入库，素材直传 COS。"""
|
|
|
|
def __init__(self, base_url: str, sync_key: str, *, timeout: float = 30.0):
|
|
self.base_url = base_url.rstrip("/")
|
|
self.timeout = timeout
|
|
self.session = requests.Session()
|
|
self.session.headers.update(
|
|
{
|
|
"Accept": "application/json",
|
|
"User-Agent": "WeCom-RPA-Archive/1.0",
|
|
"X-Desktop-Sync-Key": sync_key,
|
|
}
|
|
)
|
|
self._uploaded: dict[str, str] = {}
|
|
self._uploaded_lock = threading.Lock()
|
|
self._thread_sessions = threading.local()
|
|
self._upload_slots = threading.BoundedSemaphore(
|
|
max(2, min(16, int(os.environ.get("WECOM_ARCHIVE_UPLOAD_SLOTS", "8"))))
|
|
)
|
|
|
|
@classmethod
|
|
def connect(cls, candidates: Iterable[str]) -> "ArchiveApiClient":
|
|
last_error = ""
|
|
for candidate in candidates:
|
|
client = cls(candidate, backend_client.DESKTOP_SYNC_KEY)
|
|
try:
|
|
response = client.session.get(
|
|
client.base_url + "/api/v2/health", timeout=2.5
|
|
)
|
|
if response.status_code == 200:
|
|
return client
|
|
last_error = f"{candidate} 返回 HTTP {response.status_code}"
|
|
except requests.RequestException as exc:
|
|
last_error = f"{candidate}: {exc}"
|
|
client.close()
|
|
raise ArchiveBackupError(f"归档 API 未启动或不可用：{last_error or '没有候选地址'}")
|
|
|
|
def close(self) -> None:
|
|
self.session.close()
|
|
|
|
def _thread_session(self) -> requests.Session:
|
|
current = getattr(self._thread_sessions, "session", None)
|
|
if current is None:
|
|
current = requests.Session()
|
|
current.headers.update(self.session.headers)
|
|
self._thread_sessions.session = current
|
|
return current
|
|
|
|
@staticmethod
|
|
def _response_error(response: requests.Response) -> str:
|
|
try:
|
|
payload = response.json()
|
|
return str(
|
|
payload.get("detail")
|
|
or payload.get("error")
|
|
or payload.get("message")
|
|
or f"HTTP {response.status_code}"
|
|
)
|
|
except (TypeError, ValueError):
|
|
return f"HTTP {response.status_code}"
|
|
|
|
def _json(
|
|
self,
|
|
method: str,
|
|
path: str,
|
|
*,
|
|
payload: dict[str, Any] | None = None,
|
|
params: dict[str, Any] | None = None,
|
|
timeout: float | None = None,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
response = self._thread_session().request(
|
|
method,
|
|
self.base_url + path,
|
|
json=payload,
|
|
params=params,
|
|
timeout=timeout or self.timeout,
|
|
)
|
|
except requests.RequestException as exc:
|
|
raise ArchiveBackupError(f"调用归档 API 失败：{exc}") from exc
|
|
if not response.ok:
|
|
raise ArchiveBackupError(
|
|
f"归档 API 拒绝请求：{self._response_error(response)}"
|
|
)
|
|
try:
|
|
result = response.json()
|
|
except ValueError as exc:
|
|
raise ArchiveBackupError("归档 API 返回了无效 JSON") from exc
|
|
if not isinstance(result, dict):
|
|
raise ArchiveBackupError("归档 API 返回结构不正确")
|
|
return result
|
|
|
|
def checkpoint(self, account: str) -> dict[str, Any]:
|
|
result = self._json(
|
|
"GET",
|
|
"/api/v2/archive/desktop/checkpoint",
|
|
params={"external_account_id": account, "source_table": SOURCE_TABLE},
|
|
)
|
|
checkpoint = result.get("checkpoint")
|
|
return checkpoint if isinstance(checkpoint, dict) else {}
|
|
|
|
def advance_checkpoint(
|
|
self,
|
|
account: str,
|
|
checkpoint: dict[str, Any],
|
|
*,
|
|
display_name: str = "",
|
|
corp_scope_id: str = "",
|
|
) -> dict[str, Any]:
|
|
result = self._json(
|
|
"POST",
|
|
"/api/v2/archive/desktop/checkpoint",
|
|
payload={
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": display_name or account,
|
|
"corp_scope_id": corp_scope_id,
|
|
},
|
|
"source_table": SOURCE_TABLE,
|
|
"checkpoint": checkpoint,
|
|
},
|
|
)
|
|
value = result.get("checkpoint")
|
|
return value if isinstance(value, dict) else {}
|
|
|
|
def pending_attachments(self, account: str, limit: int = 500) -> list[str]:
|
|
result = self._json(
|
|
"GET",
|
|
"/api/v2/archive/desktop/pending-attachments",
|
|
params={"external_account_id": account, "limit": limit},
|
|
)
|
|
values = result.get("source_message_ids")
|
|
if not isinstance(values, list):
|
|
return []
|
|
return [str(value) for value in values if str(value or "").strip()]
|
|
|
|
def import_messages(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
return self._json(
|
|
"POST", "/api/v2/archive/desktop/imports/messages", payload=payload,
|
|
timeout=120,
|
|
)
|
|
|
|
def sync_metadata(self, payload: dict[str, Any]) -> dict[str, Any]:
|
|
return self._json(
|
|
"POST",
|
|
"/api/v2/archive/desktop/imports/metadata",
|
|
payload=payload,
|
|
timeout=120,
|
|
)
|
|
|
|
def upload_media(self, path: Path) -> str:
|
|
path = path.resolve()
|
|
digest = _sha256_file(path)
|
|
with self._uploaded_lock:
|
|
cached = self._uploaded.get(digest)
|
|
if cached:
|
|
return cached
|
|
mime_type = _mime_type(path)
|
|
prepared = self._json(
|
|
"POST",
|
|
"/api/v2/archive/desktop/media/prepare",
|
|
payload={
|
|
"sha256": digest,
|
|
"size_bytes": path.stat().st_size,
|
|
"mime_type": mime_type,
|
|
"original_filename": path.name,
|
|
},
|
|
timeout=60,
|
|
)
|
|
media = prepared.get("media") if isinstance(prepared.get("media"), dict) else {}
|
|
media_id = str(media.get("id") or "")
|
|
if not media_id:
|
|
raise ArchiveBackupError("COS 预上传接口没有返回素材 ID")
|
|
if not bool(prepared.get("reused")):
|
|
if prepared.get("upload_mode") == "multipart":
|
|
self._upload_multipart(path, media_id, prepared.get("multipart"))
|
|
else:
|
|
self._upload_single(path, media_id, prepared)
|
|
with self._uploaded_lock:
|
|
self._uploaded[digest] = media_id
|
|
return media_id
|
|
|
|
def _upload_single(
|
|
self, path: Path, media_id: str, prepared: dict[str, Any]
|
|
) -> None:
|
|
upload_url = str(prepared.get("upload_url") or "")
|
|
headers = prepared.get("required_headers")
|
|
if not upload_url or not isinstance(headers, dict):
|
|
raise ArchiveBackupError("COS 预上传接口没有返回完整上传信息")
|
|
try:
|
|
with self._upload_slots:
|
|
with path.open("rb") as handle:
|
|
upload = requests.put(
|
|
upload_url,
|
|
data=handle,
|
|
headers={str(key): str(value) for key, value in headers.items()},
|
|
timeout=(15, 600),
|
|
)
|
|
upload.raise_for_status()
|
|
except (OSError, requests.RequestException) as exc:
|
|
raise ArchiveBackupError(f"素材上传 COS 失败：{path.name}: {exc}") from exc
|
|
self._json(
|
|
"POST", f"/api/v2/archive/desktop/media/{media_id}/complete", timeout=120
|
|
)
|
|
|
|
def _upload_multipart(
|
|
self, path: Path, media_id: str, multipart: Any
|
|
) -> None:
|
|
if not isinstance(multipart, dict):
|
|
raise ArchiveBackupError("COS 分块上传信息不完整")
|
|
upload_id = str(multipart.get("upload_id") or "")
|
|
part_size = int(multipart.get("part_size") or 0)
|
|
raw_parts = multipart.get("parts")
|
|
if not upload_id or part_size <= 0 or not isinstance(raw_parts, list) or not raw_parts:
|
|
raise ArchiveBackupError("COS 分块上传信息不完整")
|
|
expected_by_number = {
|
|
int(item.get("part_number") or 0): item for item in raw_parts
|
|
}
|
|
|
|
def upload_part(item: dict[str, Any]) -> dict[str, Any]:
|
|
number = int(item.get("part_number") or 0)
|
|
size = int(item.get("size_bytes") or 0)
|
|
url = str(item.get("upload_url") or "")
|
|
if number <= 0 or size <= 0 or not url:
|
|
raise ArchiveBackupError("COS 分块参数不正确")
|
|
try:
|
|
with path.open("rb") as handle:
|
|
handle.seek((number - 1) * part_size)
|
|
data = handle.read(size)
|
|
if len(data) != size:
|
|
raise OSError("读取的文件分块长度不完整")
|
|
except OSError as exc:
|
|
raise ArchiveBackupError(
|
|
f"素材分块读取失败：{path.name} 第 {number} 块: {exc}"
|
|
) from exc
|
|
last_error: Exception | None = None
|
|
response: requests.Response | None = None
|
|
for attempt in range(1, 4):
|
|
try:
|
|
with self._upload_slots:
|
|
response = requests.put(url, data=data, timeout=(15, 180))
|
|
response.raise_for_status()
|
|
last_error = None
|
|
break
|
|
except requests.RequestException as exc:
|
|
last_error = exc
|
|
if attempt < 3:
|
|
time.sleep(attempt * 1.5)
|
|
if response is None or last_error is not None:
|
|
raise ArchiveBackupError(
|
|
f"素材分块上传 COS 失败：{path.name} 第 {number} 块: {last_error}"
|
|
) from last_error
|
|
etag = str(response.headers.get("ETag") or "").strip().strip('"')
|
|
if not etag:
|
|
raise ArchiveBackupError(f"COS 没有返回第 {number} 块的 ETag")
|
|
return {"part_number": number, "etag": etag}
|
|
|
|
completed: list[dict[str, Any]] = []
|
|
completed_numbers: set[int] = set()
|
|
for item in multipart.get("completed_parts") or []:
|
|
number = int(item.get("part_number") or 0)
|
|
etag = str(item.get("etag") or "").strip().strip('"')
|
|
recorded_size = int(item.get("size_bytes") or 0)
|
|
expected_item = expected_by_number.get(number)
|
|
if (
|
|
expected_item is not None
|
|
and etag
|
|
and recorded_size == int(expected_item.get("size_bytes") or 0)
|
|
):
|
|
completed.append({"part_number": number, "etag": etag})
|
|
completed_numbers.add(number)
|
|
missing_parts = [
|
|
dict(item)
|
|
for item in raw_parts
|
|
if int(item.get("part_number") or 0) not in completed_numbers
|
|
]
|
|
if missing_parts:
|
|
with ThreadPoolExecutor(
|
|
max_workers=min(6, len(missing_parts)),
|
|
thread_name_prefix="archive-cos-part",
|
|
) as pool:
|
|
futures = [pool.submit(upload_part, item) for item in missing_parts]
|
|
for future in as_completed(futures):
|
|
completed.append(future.result())
|
|
completed.sort(key=lambda item: item["part_number"])
|
|
self._json(
|
|
"POST",
|
|
f"/api/v2/archive/desktop/media/{media_id}/multipart-complete",
|
|
payload={"upload_id": upload_id, "parts": completed},
|
|
timeout=180,
|
|
)
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for block in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _mime_type(path: Path) -> str:
|
|
special = {
|
|
".silk": "audio/silk",
|
|
".amr": "audio/amr",
|
|
".m4a": "audio/mp4",
|
|
".webp": "image/webp",
|
|
}
|
|
return special.get(path.suffix.lower()) or mimetypes.guess_type(path.name)[0] or (
|
|
"application/octet-stream"
|
|
)
|
|
|
|
|
|
def _safe_raw(value: Any) -> Any:
|
|
if isinstance(value, memoryview):
|
|
value = value.tobytes()
|
|
if isinstance(value, bytes):
|
|
digest = hashlib.sha256(value).hexdigest()
|
|
if len(value) <= MAX_RAW_BINARY_BYTES:
|
|
return {
|
|
"encoding": "base64",
|
|
"data": base64.b64encode(value).decode("ascii"),
|
|
"size": len(value),
|
|
"sha256": digest,
|
|
}
|
|
return {"encoding": "external-binary", "size": len(value), "sha256": digest}
|
|
if value is None or isinstance(value, (bool, float, int, str)):
|
|
return value
|
|
return str(value)
|
|
|
|
|
|
def _valid_source_id(value: Any) -> str:
|
|
text = str(value if value is not None else "").strip()
|
|
return "" if text.lower() in {"", "0", "-1", "none", "null"} else text
|
|
|
|
|
|
def _table_columns(connection: sqlite3.Connection, table: str) -> list[str]:
|
|
return [
|
|
str(row[1])
|
|
for row in connection.execute(f'PRAGMA table_info("{table}")').fetchall()
|
|
]
|
|
|
|
|
|
def _tables(connection: sqlite3.Connection) -> set[str]:
|
|
return {
|
|
str(row[0])
|
|
for row in connection.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table'"
|
|
).fetchall()
|
|
}
|
|
|
|
|
|
def _metadata(
|
|
decrypted: list[tuple[str, str, str]], exporter: Any
|
|
) -> tuple[dict[tuple[str, str], str], dict[tuple[str, str], str]]:
|
|
users: dict[tuple[str, str], str] = {}
|
|
conversations: dict[tuple[str, str], str] = {}
|
|
for database_path, database_name, account in decrypted:
|
|
if database_name not in {"user.db", "session.db"}:
|
|
continue
|
|
connection = exporter.connect_sqlite(database_path)
|
|
try:
|
|
available = _tables(connection)
|
|
if database_name == "user.db" and "user_table" in available:
|
|
columns = set(_table_columns(connection, "user_table"))
|
|
wanted = [
|
|
name for name in ("id", "name", "real_name", "account")
|
|
if name in columns
|
|
]
|
|
if "id" in wanted:
|
|
cursor = connection.execute(
|
|
f"SELECT {','.join(wanted)} FROM user_table"
|
|
)
|
|
values_list = cursor.fetchall()
|
|
cursor.close()
|
|
for values in values_list:
|
|
item = dict(zip(wanted, values))
|
|
user_id = str(item.get("id") or "")
|
|
label = str(
|
|
item.get("name")
|
|
or item.get("real_name")
|
|
or item.get("account")
|
|
or user_id
|
|
)
|
|
if user_id:
|
|
users[(account, user_id)] = label
|
|
if database_name == "session.db" and "conversation_table" in available:
|
|
columns = set(_table_columns(connection, "conversation_table"))
|
|
wanted = [
|
|
name
|
|
for name in ("id", "name", "roomname_remark", "session_id")
|
|
if name in columns
|
|
]
|
|
if "id" in wanted:
|
|
cursor = connection.execute(
|
|
f"SELECT {','.join(wanted)} FROM conversation_table"
|
|
)
|
|
values_list = cursor.fetchall()
|
|
cursor.close()
|
|
for values in values_list:
|
|
item = dict(zip(wanted, values))
|
|
conversation_id = str(item.get("id") or "")
|
|
label = str(
|
|
item.get("roomname_remark")
|
|
or item.get("name")
|
|
or item.get("session_id")
|
|
or ""
|
|
)
|
|
if conversation_id:
|
|
conversations[(account, conversation_id)] = label
|
|
finally:
|
|
connection.close()
|
|
return users, conversations
|
|
|
|
|
|
def _conversation_display_name(
|
|
account: str,
|
|
conversation_id: str,
|
|
users: dict[tuple[str, str], str],
|
|
conversations: dict[tuple[str, str], str],
|
|
) -> str:
|
|
"""把 S:/M: 技术会话 ID 还原成对方昵称。"""
|
|
|
|
candidate = str(conversations.get((account, conversation_id)) or "").strip()
|
|
if candidate and candidate != conversation_id:
|
|
return candidate
|
|
if conversation_id.startswith("M:"):
|
|
peer_id = conversation_id[2:]
|
|
return users.get((account, peer_id), "") or f"微信用户 {peer_id}"
|
|
if conversation_id.startswith("S:"):
|
|
parts = [item for item in conversation_id[2:].split("_") if item]
|
|
peer_id = next((item for item in parts if item != account), "")
|
|
if not peer_id and parts:
|
|
peer_id = parts[0]
|
|
return users.get((account, peer_id), "") or f"企微用户 {peer_id}"
|
|
if conversation_id.startswith("Y:"):
|
|
return f"应用 {conversation_id[2:]}"
|
|
if conversation_id.startswith("O:"):
|
|
return f"服务 {conversation_id[2:]}"
|
|
return conversation_id
|
|
|
|
|
|
def _looks_like_binary_text(value: Any) -> bool:
|
|
text = str(value or "").strip()
|
|
compact = re.sub(r"[\s|]+", "", text)
|
|
return bool(
|
|
len(compact) >= 80
|
|
and len(compact) % 2 == 0
|
|
and re.fullmatch(r"[0-9a-fA-F]+", compact)
|
|
)
|
|
|
|
|
|
def _is_application_conversation(value: Any) -> bool:
|
|
return str(value or "").strip().upper().startswith("Y:")
|
|
|
|
|
|
def _semantic_content(
|
|
message_type: str,
|
|
parsed_content: Any,
|
|
media_reference: dict[str, Any],
|
|
voice_text: str = "",
|
|
) -> str:
|
|
if voice_text:
|
|
return voice_text
|
|
attachment_metadata = media_reference.get("attachment_metadata") or []
|
|
if attachment_metadata and isinstance(attachment_metadata[0], dict):
|
|
metadata = attachment_metadata[0]
|
|
return file_message_content(
|
|
metadata, cached=str(metadata.get("status") or "") == "cached"
|
|
)
|
|
content = str(parsed_content or "").strip()
|
|
decoded = decode_hex_protobuf_text(content, message_type)
|
|
if decoded:
|
|
return decoded
|
|
has_media = bool(
|
|
media_reference.get("matched_local_files")
|
|
or media_reference.get("urls")
|
|
or media_reference.get("filenames")
|
|
or media_reference.get("uuids")
|
|
)
|
|
if _looks_like_binary_text(content):
|
|
return f"[{message_type}]"
|
|
if not content and (
|
|
has_media
|
|
or message_type in {
|
|
"图片", "截图", "语音", "视频", "文件", "文件回复",
|
|
"文件预览", "文件分享", "群文件", "表情",
|
|
}
|
|
):
|
|
return f"[{message_type}]"
|
|
return content
|
|
|
|
|
|
def _message_status(row: dict[str, Any]) -> str:
|
|
for name in ("is_revoke", "revoke_status"):
|
|
if row.get(name) not in (None, "", 0, "0"):
|
|
return "revoked"
|
|
for name in ("is_deleted", "delete_status"):
|
|
if row.get(name) not in (None, "", 0, "0"):
|
|
return "deleted"
|
|
return "normal"
|
|
|
|
|
|
class IncrementalArchiveImporter:
|
|
def __init__(
|
|
self,
|
|
config: AutoBackupConfig,
|
|
api: ArchiveApiClient,
|
|
exporter: Any,
|
|
media_exporter: Any,
|
|
):
|
|
self.config = config
|
|
self.api = api
|
|
self.exporter = exporter
|
|
self.media_exporter = media_exporter
|
|
self._media_indexes: dict[str, dict[str, Any]] = {}
|
|
self.upload_workers = max(
|
|
1, min(12, int(os.environ.get("WECOM_ARCHIVE_UPLOAD_WORKERS", "6")))
|
|
)
|
|
|
|
def _media_for_row(
|
|
self, account: str, row: dict[str, Any]
|
|
) -> tuple[list[Path], dict[str, Any]]:
|
|
raw_content = row.get("content")
|
|
if isinstance(raw_content, memoryview):
|
|
raw_content = raw_content.tobytes()
|
|
file_metadata = parse_file_message_metadata(
|
|
raw_content, row.get("content_type")
|
|
)
|
|
if isinstance(raw_content, str):
|
|
raw_content = raw_content.encode("utf-8", errors="replace")
|
|
refs = self.media_exporter.extract_media_refs(raw_content)
|
|
if file_metadata:
|
|
refs["filenames"] = list(refs.get("filenames") or [])
|
|
filename = str(file_metadata["original_filename"])
|
|
if filename not in refs["filenames"]:
|
|
refs["filenames"].append(filename)
|
|
if not any(refs.values()):
|
|
return [], {}
|
|
if account not in self._media_indexes:
|
|
account_root = self.config.source_root / account
|
|
self._media_indexes[account] = self.media_exporter.build_cache_index(
|
|
str(account_root)
|
|
)
|
|
local_path, local_type, method = self.media_exporter.match_media(
|
|
refs, self._media_indexes[account]
|
|
)
|
|
matches: list[tuple[str, str, str]] = []
|
|
if local_path:
|
|
matches.append((str(local_path), str(local_type or ""), str(method or "")))
|
|
index = self._media_indexes[account]
|
|
for media_uuid in refs.get("uuids") or []:
|
|
found = (index.get("by_uuid") or {}).get(str(media_uuid).lower())
|
|
if found:
|
|
matches.append((str(found[0]), str(found[1] or ""), "uuid"))
|
|
for filename in refs.get("filenames") or []:
|
|
normalized = Path(str(filename)).name.lower()
|
|
found = (index.get("by_name") or {}).get(normalized)
|
|
if found:
|
|
matches.append((str(found[0]), str(found[1] or ""), "filename"))
|
|
media_paths: list[Path] = []
|
|
matched_files: list[dict[str, str]] = []
|
|
seen_paths: set[str] = set()
|
|
for matched_path, matched_type, matched_method in matches:
|
|
resolved = Path(matched_path).resolve()
|
|
dedup_path = os.path.normcase(str(resolved))
|
|
if dedup_path in seen_paths or not resolved.is_file():
|
|
continue
|
|
if file_metadata and resolved.stat().st_size != int(
|
|
file_metadata.get("size_bytes") or 0
|
|
):
|
|
continue
|
|
seen_paths.add(dedup_path)
|
|
media_paths.append(resolved)
|
|
matched_files.append(
|
|
{
|
|
"filename": resolved.name,
|
|
"local_media_type": matched_type,
|
|
"match_method": matched_method,
|
|
}
|
|
)
|
|
if file_metadata:
|
|
file_metadata["status"] = (
|
|
"cached" if media_paths else "source_not_cached"
|
|
)
|
|
return media_paths, {
|
|
"urls": list(refs.get("urls") or []),
|
|
"filenames": list(refs.get("filenames") or []),
|
|
"uuids": list(refs.get("uuids") or []),
|
|
"match_method": str(method or "unmatched"),
|
|
"matched_local_files": matched_files,
|
|
"attachment_metadata": [file_metadata] if file_metadata else [],
|
|
}
|
|
|
|
def _normalize(
|
|
self,
|
|
account: str,
|
|
row: dict[str, Any],
|
|
users: dict[tuple[str, str], str],
|
|
conversations: dict[tuple[str, str], str],
|
|
voice_texts: dict[tuple[str, str], str] | None = None,
|
|
) -> dict[str, Any]:
|
|
conversation_id = str(row.get("conversation_id") or "unknown")
|
|
sender_id = str(row.get("sender_id") or "")
|
|
source_message_id = next(
|
|
(
|
|
value
|
|
for value in (
|
|
_valid_source_id(row.get("server_id")),
|
|
_valid_source_id(row.get("client_id")),
|
|
_valid_source_id(row.get("message_id")),
|
|
)
|
|
if value
|
|
),
|
|
f"rowid:{int(row['__archive_rowid'])}",
|
|
)
|
|
content = self.exporter.parse_content(row.get("content"))
|
|
extra = self.exporter.parse_content(row.get("extra_content"))
|
|
if extra and not str(content or "").strip():
|
|
content = extra
|
|
media_paths, media_reference = self._media_for_row(account, row)
|
|
content_type = (
|
|
row.get("content_type")
|
|
if row.get("content_type") not in (None, "")
|
|
else row.get("msg_type", row.get("type"))
|
|
)
|
|
message_type = self.exporter.get_msg_type_name(content_type)
|
|
mini_program_metadata = parse_mini_program_metadata(
|
|
row.get("content"), content_type
|
|
)
|
|
if mini_program_metadata:
|
|
message_type = "小程序"
|
|
media_reference["mini_program"] = mini_program_metadata
|
|
attachment_metadata = list(
|
|
media_reference.get("attachment_metadata") or []
|
|
)
|
|
if attachment_metadata:
|
|
message_type = "文件"
|
|
server_id = str(row.get("server_id") or "")
|
|
voice_texts = voice_texts or {}
|
|
voice_text = str(
|
|
voice_texts.get((account, server_id))
|
|
or voice_texts.get(("", server_id))
|
|
or ""
|
|
).strip()
|
|
if mini_program_metadata:
|
|
content = mini_program_content(mini_program_metadata)
|
|
else:
|
|
content = _semantic_content(
|
|
message_type, content, media_reference, voice_text=voice_text
|
|
)
|
|
sequence = next(
|
|
(
|
|
row.get(name)
|
|
for name in ("sequence", "sequence_no", "message_seq", "seq", "local_id")
|
|
if row.get(name) not in (None, "")
|
|
),
|
|
None,
|
|
)
|
|
return {
|
|
"source_table": SOURCE_TABLE,
|
|
"source_message_id": source_message_id,
|
|
"server_id": str(row.get("server_id") or ""),
|
|
"client_id": str(row.get("client_id") or ""),
|
|
"sequence_no": sequence,
|
|
"conversation": {
|
|
"external_id": conversation_id,
|
|
"name": _conversation_display_name(
|
|
account, conversation_id, users, conversations
|
|
),
|
|
},
|
|
"sender": {
|
|
"external_id": sender_id,
|
|
"display_name": users.get((account, sender_id), sender_id or "系统"),
|
|
"identity_type": "wecom_userid",
|
|
"scope_id": self.config.corp_scope_id,
|
|
"source": "wxwork_export_auto_backup",
|
|
},
|
|
"sent_at_epoch": row.get("send_time"),
|
|
"message_type": message_type,
|
|
"direction": "outbound" if sender_id and sender_id == account else "inbound",
|
|
"status": _message_status(row),
|
|
"content": str(content or ""),
|
|
"media_ids": [],
|
|
"attachment_metadata": attachment_metadata,
|
|
"mini_program": mini_program_metadata,
|
|
"_local_media_paths": [str(path) for path in media_paths],
|
|
"media_reference": media_reference,
|
|
"raw_fields": {
|
|
key: _safe_raw(value)
|
|
for key, value in row.items()
|
|
if key != "__archive_rowid"
|
|
},
|
|
"source_rowid": int(row["__archive_rowid"]),
|
|
}
|
|
|
|
def _upload_batch_media(self, messages: list[dict[str, Any]]) -> None:
|
|
path_to_messages: dict[str, list[dict[str, Any]]] = {}
|
|
for message in messages:
|
|
for raw_path in message.pop("_local_media_paths", []):
|
|
normalized = os.path.normcase(str(Path(raw_path).resolve()))
|
|
path_to_messages.setdefault(normalized, []).append(message)
|
|
if not path_to_messages:
|
|
return
|
|
with ThreadPoolExecutor(
|
|
max_workers=min(self.upload_workers, len(path_to_messages)),
|
|
thread_name_prefix="archive-cos-upload",
|
|
) as pool:
|
|
futures = {
|
|
pool.submit(self.api.upload_media, Path(path)): path
|
|
for path in path_to_messages
|
|
}
|
|
resolved_ids: dict[str, str] = {}
|
|
for future in as_completed(futures):
|
|
path = futures[future]
|
|
resolved_ids[path] = future.result()
|
|
for path, attached_messages in path_to_messages.items():
|
|
media_id = resolved_ids[path]
|
|
for message in attached_messages:
|
|
if media_id not in message["media_ids"]:
|
|
message["media_ids"].append(media_id)
|
|
|
|
def _repair_cached_attachments(
|
|
self,
|
|
connection: Any,
|
|
account: str,
|
|
columns: set[str],
|
|
users: dict[tuple[str, str], str],
|
|
conversations: dict[tuple[str, str], str],
|
|
voice_texts: dict[tuple[str, str], str],
|
|
) -> int:
|
|
source_ids = self.api.pending_attachments(account, limit=500)
|
|
if not source_ids:
|
|
return 0
|
|
identity_columns = [
|
|
name for name in ("server_id", "client_id", "message_id")
|
|
if name in columns
|
|
]
|
|
if not identity_columns:
|
|
return 0
|
|
repaired = 0
|
|
for start in range(0, len(source_ids), 400):
|
|
chunk = source_ids[start : start + 400]
|
|
placeholders = ",".join("?" for _ in chunk)
|
|
predicates = [
|
|
f"CAST({name} AS TEXT) IN ({placeholders})"
|
|
for name in identity_columns
|
|
]
|
|
parameters = tuple(
|
|
source_id
|
|
for _name in identity_columns
|
|
for source_id in chunk
|
|
)
|
|
cursor = connection.execute(
|
|
f"SELECT rowid AS __archive_rowid,* FROM {SOURCE_TABLE} "
|
|
f"WHERE {' OR '.join(predicates)} ORDER BY send_time,rowid",
|
|
parameters,
|
|
)
|
|
names = [str(item[0]) for item in cursor.description]
|
|
rows = [dict(zip(names, values)) for values in cursor.fetchall()]
|
|
cursor.close()
|
|
rows = [
|
|
row for row in rows
|
|
if not _is_application_conversation(row.get("conversation_id"))
|
|
]
|
|
messages = [
|
|
self._normalize(account, row, users, conversations, voice_texts)
|
|
for row in rows
|
|
]
|
|
messages = [
|
|
message for message in messages
|
|
if message.get("_local_media_paths")
|
|
]
|
|
if not messages:
|
|
continue
|
|
self._upload_batch_media(messages)
|
|
result = self.api.import_messages(
|
|
{
|
|
"batch_id": uuid.uuid4().hex,
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": users.get((account, account), account),
|
|
"corp_scope_id": self.config.corp_scope_id,
|
|
},
|
|
"source_table": SOURCE_TABLE,
|
|
"messages": messages,
|
|
}
|
|
)
|
|
repaired += len(messages) - int(result.get("errors") or 0)
|
|
return repaired
|
|
|
|
def run(
|
|
self, decrypted: list[tuple[str, str, str]]
|
|
) -> dict[str, Any]:
|
|
users, conversations = _metadata(decrypted, self.exporter)
|
|
voice_texts: dict[tuple[str, str], str] = {}
|
|
load_voice2text = getattr(self.exporter, "load_voice2text", None)
|
|
if callable(load_voice2text):
|
|
try:
|
|
loaded = load_voice2text(decrypted, log=lambda _message: None)
|
|
if isinstance(loaded, dict):
|
|
voice_texts = loaded
|
|
except Exception:
|
|
voice_texts = {}
|
|
summary = {
|
|
"accounts": 0,
|
|
"batches": 0,
|
|
"received": 0,
|
|
"inserted": 0,
|
|
"duplicates": 0,
|
|
"media": 0,
|
|
"attachments_repaired": 0,
|
|
"application_messages_skipped": 0,
|
|
"people_synced": 0,
|
|
"conversations_updated": 0,
|
|
}
|
|
accounts = sorted(
|
|
{account for _, database_name, account in decrypted if database_name == "message.db"}
|
|
)
|
|
for account in accounts:
|
|
people = [
|
|
{
|
|
"external_id": user_id,
|
|
"display_name": name,
|
|
"identity_type": "wecom_userid",
|
|
"scope_id": self.config.corp_scope_id,
|
|
"source": "wxwork_export_auto_backup",
|
|
}
|
|
for (user_account, user_id), name in users.items()
|
|
if user_account == account
|
|
]
|
|
conversation_rows = [
|
|
{
|
|
"external_id": conversation_id,
|
|
"name": _conversation_display_name(
|
|
account, conversation_id, users, conversations
|
|
),
|
|
}
|
|
for (conversation_account, conversation_id) in conversations
|
|
if conversation_account == account
|
|
and not _is_application_conversation(conversation_id)
|
|
]
|
|
total = max(len(people), len(conversation_rows), 1)
|
|
for start in range(0, total, 5000):
|
|
synced = self.api.sync_metadata(
|
|
{
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": users.get((account, account), account),
|
|
"corp_scope_id": self.config.corp_scope_id,
|
|
},
|
|
"people": people[start : start + 5000],
|
|
"conversations": conversation_rows[start : start + 5000],
|
|
}
|
|
)
|
|
summary["people_synced"] += int(synced.get("people_synced") or 0)
|
|
summary["conversations_updated"] += int(
|
|
synced.get("conversations_updated") or 0
|
|
)
|
|
for database_path, database_name, account in decrypted:
|
|
if database_name != "message.db":
|
|
continue
|
|
connection = self.exporter.connect_sqlite(database_path)
|
|
try:
|
|
if SOURCE_TABLE not in _tables(connection):
|
|
continue
|
|
columns = _table_columns(connection, SOURCE_TABLE)
|
|
if "send_time" not in columns:
|
|
continue
|
|
checkpoint = self.api.checkpoint(account)
|
|
sent_at = float(checkpoint.get("send_time") or 0)
|
|
rowid = int(checkpoint.get("rowid") or 0)
|
|
summary["accounts"] += 1
|
|
while True:
|
|
cursor = connection.execute(
|
|
"""SELECT rowid AS __archive_rowid,* FROM message_table
|
|
WHERE send_time>? OR (send_time=? AND rowid>?)
|
|
ORDER BY send_time,rowid LIMIT ?""",
|
|
(sent_at, sent_at, rowid, self.config.batch_size),
|
|
)
|
|
names = [str(item[0]) for item in cursor.description]
|
|
values_list = cursor.fetchall()
|
|
cursor.close()
|
|
rows = [dict(zip(names, values)) for values in values_list]
|
|
if not rows:
|
|
break
|
|
importable_rows = [
|
|
row for row in rows
|
|
if not _is_application_conversation(
|
|
row.get("conversation_id")
|
|
)
|
|
]
|
|
summary["application_messages_skipped"] += (
|
|
len(rows) - len(importable_rows)
|
|
)
|
|
messages = [
|
|
self._normalize(
|
|
account, row, users, conversations, voice_texts
|
|
)
|
|
for row in importable_rows
|
|
]
|
|
# 一个批次的素材先并发上传并由服务端校验；全部成功后才提交消息
|
|
# 和检查点。任一素材失败，整批消息保持可重试状态。
|
|
self._upload_batch_media(messages)
|
|
last = rows[-1]
|
|
next_checkpoint = {
|
|
"send_time": float(last["send_time"]),
|
|
"rowid": int(last["__archive_rowid"]),
|
|
}
|
|
import_payload = {
|
|
"batch_id": uuid.uuid4().hex,
|
|
"source_account": {
|
|
"external_account_id": account,
|
|
"display_name": users.get((account, account), account),
|
|
"corp_scope_id": self.config.corp_scope_id,
|
|
},
|
|
"source_table": SOURCE_TABLE,
|
|
"checkpoint": next_checkpoint,
|
|
"messages": messages,
|
|
}
|
|
if messages:
|
|
result = self.api.import_messages(import_payload)
|
|
else:
|
|
self.api.advance_checkpoint(
|
|
account,
|
|
next_checkpoint,
|
|
display_name=users.get((account, account), account),
|
|
corp_scope_id=self.config.corp_scope_id,
|
|
)
|
|
result = {"inserted": 0, "duplicates": 0}
|
|
summary["batches"] += 1
|
|
summary["received"] += len(messages)
|
|
summary["inserted"] += int(result.get("inserted") or 0)
|
|
summary["duplicates"] += int(result.get("duplicates") or 0)
|
|
summary["media"] += sum(
|
|
len(message.get("media_ids") or []) for message in messages
|
|
)
|
|
sent_at = next_checkpoint["send_time"]
|
|
rowid = next_checkpoint["rowid"]
|
|
summary["attachments_repaired"] += self._repair_cached_attachments(
|
|
connection,
|
|
account,
|
|
columns,
|
|
users,
|
|
conversations,
|
|
voice_texts,
|
|
)
|
|
finally:
|
|
connection.close()
|
|
return summary
|
|
|
|
|
|
def _load_exporter_modules(exporter_root: Path) -> tuple[Any, Any]:
|
|
root = str(exporter_root.resolve())
|
|
if root in sys.path:
|
|
sys.path.remove(root)
|
|
sys.path.insert(0, root)
|
|
exporter = importlib.import_module("wxwork_export_final")
|
|
media_exporter = importlib.import_module("wxwork_export_media")
|
|
return exporter, media_exporter
|
|
|
|
|
|
def _load_exporter_keys(exporter_root: Path) -> dict[str, str]:
|
|
"""原导出器未指定编码；这里固定 UTF-8，且绝不输出密钥内容。"""
|
|
|
|
path = exporter_root / "wxwork_keys.json"
|
|
try:
|
|
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
except (OSError, TypeError, ValueError) as exc:
|
|
raise ArchiveBackupError("无法读取企业微信数据库密钥文件") from exc
|
|
keys = payload.get("keys") if isinstance(payload, dict) else None
|
|
result = {
|
|
str(account): str(key)
|
|
for account, key in (keys.items() if isinstance(keys, dict) else [])
|
|
if str(account).strip() and str(key).strip()
|
|
}
|
|
global_key = str(payload.get("global_key") or "") if isinstance(payload, dict) else ""
|
|
if global_key:
|
|
result.setdefault("*", global_key)
|
|
return result
|
|
|
|
|
|
def run_backup_once(config: AutoBackupConfig) -> dict[str, Any]:
|
|
config.work_root.mkdir(parents=True, exist_ok=True)
|
|
status_path = config.work_root / "status.json"
|
|
with _ProcessLock(config.work_root / "backup.lock"):
|
|
started_at = time.time()
|
|
_atomic_json(
|
|
status_path,
|
|
{
|
|
"status": "running",
|
|
"started_at": started_at,
|
|
"source_root": str(config.source_root),
|
|
},
|
|
)
|
|
api: ArchiveApiClient | None = None
|
|
try:
|
|
if not config.source_root.is_dir():
|
|
raise ArchiveBackupError(f"企业微信数据目录不存在：{config.source_root}")
|
|
exporter, media_exporter = _load_exporter_modules(config.exporter_root)
|
|
keys = _load_exporter_keys(config.exporter_root)
|
|
decrypted = exporter.decrypt_with_keys(
|
|
str(config.source_root),
|
|
str(config.work_root / "decrypted"),
|
|
keys,
|
|
use_cache=True,
|
|
)
|
|
if not any(item[1] == "message.db" for item in decrypted):
|
|
raise ArchiveBackupError("没有解密出可用的企业微信 message.db")
|
|
api = ArchiveApiClient.connect(config.api_urls)
|
|
summary = IncrementalArchiveImporter(
|
|
config, api, exporter, media_exporter
|
|
).run(decrypted)
|
|
result = {
|
|
"status": "completed",
|
|
"started_at": started_at,
|
|
"completed_at": time.time(),
|
|
"api_url": api.base_url,
|
|
"summary": summary,
|
|
}
|
|
_atomic_json(status_path, result)
|
|
return result
|
|
except Exception as exc:
|
|
failed = {
|
|
"status": "failed",
|
|
"started_at": started_at,
|
|
"completed_at": time.time(),
|
|
"error": str(exc)[:2000],
|
|
}
|
|
_atomic_json(status_path, failed)
|
|
raise
|
|
finally:
|
|
if api is not None:
|
|
api.close()
|
|
|
|
|
|
def _worker_arguments(config: AutoBackupConfig) -> list[str]:
|
|
arguments = [
|
|
"--exporter-root", str(config.exporter_root),
|
|
"--source-root", str(config.source_root),
|
|
"--work-root", str(config.work_root),
|
|
"--corp-scope-id", config.corp_scope_id,
|
|
"--batch-size", str(config.batch_size),
|
|
]
|
|
if config.api_urls:
|
|
arguments.extend(["--api-url", config.api_urls[0]])
|
|
return arguments
|
|
|
|
|
|
def _worker_command(config: AutoBackupConfig) -> list[str]:
|
|
if is_frozen():
|
|
return [sys.executable, "--archive-backup-worker", *_worker_arguments(config)]
|
|
return [
|
|
sys.executable,
|
|
"-u",
|
|
str(Path(__file__).resolve()),
|
|
"--worker",
|
|
*_worker_arguments(config),
|
|
]
|
|
|
|
|
|
def _launch_worker(config: AutoBackupConfig, initial_delay: float) -> None:
|
|
if initial_delay > 0:
|
|
time.sleep(initial_delay)
|
|
config.work_root.mkdir(parents=True, exist_ok=True)
|
|
log_path = config.work_root / "auto_backup.log"
|
|
retry_delays = (0, 15, 45)
|
|
for attempt, retry_delay in enumerate(retry_delays, start=1):
|
|
if retry_delay:
|
|
time.sleep(retry_delay)
|
|
with log_path.open("a", encoding="utf-8", buffering=1) as log:
|
|
log.write(
|
|
f"\n[{time.strftime('%Y-%m-%d %H:%M:%S')}] 自动归档启动，第 {attempt} 次尝试\n"
|
|
)
|
|
creationflags = 0x08000000 if os.name == "nt" else 0
|
|
environment = os.environ.copy()
|
|
environment["PYTHONUTF8"] = "1"
|
|
try:
|
|
completed = subprocess.run(
|
|
_worker_command(config),
|
|
cwd=str(config.exporter_root),
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
env=environment,
|
|
creationflags=creationflags,
|
|
timeout=60 * 60 * 6,
|
|
check=False,
|
|
)
|
|
log.write(f"自动归档子进程退出：code={completed.returncode}\n")
|
|
if completed.returncode == 0:
|
|
return
|
|
except subprocess.TimeoutExpired:
|
|
log.write("自动归档超过 6 小时，已停止本次任务\n")
|
|
except Exception as exc:
|
|
log.write(f"自动归档无法启动：{exc}\n")
|
|
|
|
|
|
def start_auto_backup(*, initial_delay: float = 2.0) -> bool:
|
|
"""非阻塞启动；同一个 GUI 进程只会挂一次。"""
|
|
|
|
global _START_THREAD
|
|
if not _truthy(os.environ.get("WECOM_ARCHIVE_AUTO_BACKUP"), True):
|
|
return False
|
|
if any(
|
|
argument in sys.argv
|
|
for argument in ("--packaging-self-check", "--qt-smoke-test", "--archive-backup-worker")
|
|
):
|
|
return False
|
|
try:
|
|
config = AutoBackupConfig.discover()
|
|
except Exception:
|
|
return False
|
|
with _START_LOCK:
|
|
if _START_THREAD is not None and _START_THREAD.is_alive():
|
|
return False
|
|
_START_THREAD = threading.Thread(
|
|
target=_launch_worker,
|
|
args=(config, max(0.0, float(initial_delay))),
|
|
name="archive-auto-backup-launcher",
|
|
daemon=True,
|
|
)
|
|
_START_THREAD.start()
|
|
return True
|
|
|
|
|
|
def worker_main(argv: list[str] | None = None) -> int:
|
|
parser = argparse.ArgumentParser(description="企业微信聊天记录自动归档后台任务")
|
|
parser.add_argument("--worker", action="store_true")
|
|
parser.add_argument("--exporter-root", required=True)
|
|
parser.add_argument("--source-root", required=True)
|
|
parser.add_argument("--work-root", required=True)
|
|
parser.add_argument("--api-url", default="")
|
|
parser.add_argument("--corp-scope-id", default="local-wecom-corp")
|
|
parser.add_argument("--batch-size", type=int, default=DEFAULT_BATCH_SIZE)
|
|
args = parser.parse_args(argv)
|
|
config = AutoBackupConfig(
|
|
exporter_root=Path(args.exporter_root).resolve(),
|
|
source_root=Path(args.source_root).resolve(),
|
|
work_root=Path(args.work_root).resolve(),
|
|
api_urls=tuple(_api_candidates(args.api_url)),
|
|
corp_scope_id=str(args.corp_scope_id or "local-wecom-corp"),
|
|
batch_size=max(1, min(5000, int(args.batch_size))),
|
|
)
|
|
try:
|
|
result = run_backup_once(config)
|
|
print(json.dumps(result, ensure_ascii=False), flush=True)
|
|
return 0
|
|
except ArchiveBackupError as exc:
|
|
print(f"[自动归档失败] {exc}", flush=True)
|
|
return 2
|
|
except Exception as exc:
|
|
print(f"[自动归档异常] {type(exc).__name__}: {exc}", flush=True)
|
|
return 3
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(worker_main())
|