233 lines
12 KiB
Python
233 lines
12 KiB
Python
"""Run with build Python -I -S; all ASR code/libraries/model must come from dist.
|
|
|
|
Example:
|
|
C:\\wechat_rpa\\.build-venv\\Scripts\\python.exe -I -S verify_frozen_offline_asr.py
|
|
|
|
Only PyInstaller's archive reader is bootstrapped from the build environment.
|
|
Its site-packages path is then removed before importing any tested dependency.
|
|
Inputs are the three explicitly generated synthetic-chinese test audio files.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import contextlib
|
|
import ctypes
|
|
import hashlib
|
|
import importlib
|
|
import importlib.abc
|
|
import importlib.machinery
|
|
import importlib.resources.readers
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
import marshal
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import types
|
|
from unittest import mock
|
|
|
|
|
|
def inside(path, root):
|
|
try:
|
|
return Path(path).resolve().is_relative_to(root.resolve())
|
|
except (ValueError, TypeError, OSError):
|
|
return False
|
|
|
|
|
|
class FrozenFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader):
|
|
def __init__(self, pyz, internal, stdlib):
|
|
self.pyz, self.internal, self.stdlib = pyz, internal, stdlib
|
|
self.loaded = {}
|
|
|
|
def find_spec(self, fullname, path=None, target=None):
|
|
entry = self.pyz.toc.get(fullname)
|
|
if entry is not None:
|
|
package = entry[0] in (1, 3)
|
|
file = self.internal.joinpath(*fullname.split("."))
|
|
file = file / "__init__.py" if package else file.with_suffix(".py")
|
|
spec = importlib.util.spec_from_loader(fullname, self, origin=str(file), is_package=package)
|
|
spec.has_location = True
|
|
if package:
|
|
spec.submodule_search_locations = [str(file.parent)]
|
|
return spec
|
|
spec = importlib.machinery.PathFinder.find_spec(fullname, path, target)
|
|
if spec is not None and spec.origin not in (None, "built-in", "frozen"):
|
|
if not inside(spec.origin, self.internal) and not any(inside(spec.origin, root) for root in self.stdlib):
|
|
raise ImportError("Dependency escaped packaged/stdlib roots: " + fullname)
|
|
return None
|
|
|
|
def create_module(self, spec):
|
|
return None
|
|
|
|
def exec_module(self, module):
|
|
module.__file__ = module.__spec__.origin
|
|
if module.__spec__.submodule_search_locations is not None:
|
|
module.__path__ = list(module.__spec__.submodule_search_locations)
|
|
code = self.pyz.extract(module.__name__)
|
|
if code is None and self.pyz.toc[module.__name__][0] == 3:
|
|
return
|
|
if not isinstance(code, types.CodeType):
|
|
raise ImportError("PYZ entry is not Python code: " + module.__name__)
|
|
self.loaded[module.__name__] = hashlib.sha256(marshal.dumps(code)).hexdigest()
|
|
exec(code, module.__dict__)
|
|
|
|
def get_resource_reader(self, fullname):
|
|
path = self.internal.joinpath(*fullname.split("."), "__init__.py")
|
|
return importlib.resources.readers.FileReader(types.SimpleNamespace(path=str(path)))
|
|
|
|
|
|
def process_dlls():
|
|
kernel, psapi = ctypes.WinDLL("kernel32"), ctypes.WinDLL("psapi")
|
|
kernel.GetCurrentProcess.restype = ctypes.c_void_p
|
|
process = kernel.GetCurrentProcess()
|
|
modules = (ctypes.c_void_p * 4096)()
|
|
needed = ctypes.c_ulong()
|
|
psapi.EnumProcessModules.argtypes = [ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p), ctypes.c_ulong, ctypes.POINTER(ctypes.c_ulong)]
|
|
psapi.GetModuleFileNameExW.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_wchar_p, ctypes.c_ulong]
|
|
if not psapi.EnumProcessModules(process, modules, ctypes.sizeof(modules), ctypes.byref(needed)):
|
|
raise RuntimeError("Cannot audit actual loaded DLL origins")
|
|
if needed.value > ctypes.sizeof(modules):
|
|
raise RuntimeError("DLL origin audit truncated")
|
|
result = []
|
|
for index in range(needed.value // ctypes.sizeof(ctypes.c_void_p)):
|
|
name = ctypes.create_unicode_buffer(32768)
|
|
if psapi.GetModuleFileNameExW(process, modules[index], name, len(name)):
|
|
result.append(name.value)
|
|
return result
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--dist", type=Path, default=Path(r"C:\wechat_rpa\dist\ZhenAI-WeCom-Assistant-v1.4.28"))
|
|
parser.add_argument("--output", type=Path, default=Path(__file__).with_name("frozen-offline-asr.json"))
|
|
args = parser.parse_args()
|
|
output = args.output.resolve()
|
|
root = Path(__file__).resolve().parent
|
|
dist = args.dist.resolve()
|
|
internal = dist / "_internal"
|
|
exe = dist / (dist.name + ".exe")
|
|
log, attempts = io.StringIO(), []
|
|
report = {"ok": False, "dist": str(dist), "python": sys.version,
|
|
"executable": sys.executable, "network_or_external_attempts": attempts,
|
|
"results": [], "input_kind": "local synthetic Chinese speech only"}
|
|
sys.dont_write_bytecode = True
|
|
started = time.monotonic()
|
|
try:
|
|
with contextlib.redirect_stdout(log), contextlib.redirect_stderr(log), contextlib.ExitStack() as stack:
|
|
if not sys.flags.isolated or not sys.flags.no_site or sys.version_info[:2] != (3, 12):
|
|
raise RuntimeError("Must launch production build Python with -I -S")
|
|
if not exe.is_file() or not internal.is_dir():
|
|
raise FileNotFoundError("Finalized 1.4.28 dist is not available")
|
|
original_path = list(sys.path)
|
|
bootstrap_site = Path(r"C:\wechat_rpa\.build-venv\Lib\site-packages")
|
|
sys.path.append(str(bootstrap_site))
|
|
from PyInstaller.archive.readers import CArchiveReader
|
|
pyz = CArchiveReader(str(exe)).open_embedded_archive("PYZ.pyz")
|
|
tested = ("voice_transcription", "numpy", "av", "pysilk", "ctranslate2", "faster_whisper", "onnxruntime", "tokenizers")
|
|
if any(name in sys.modules for name in tested):
|
|
raise RuntimeError("Tested dependency was imported before packaged loader")
|
|
if "voice_transcription" not in pyz.toc:
|
|
raise RuntimeError("voice_transcription is absent from the EXE PYZ")
|
|
bootstrap_modules = []
|
|
for name, module in tuple(sys.modules.items()):
|
|
origin = getattr(module, "__file__", None)
|
|
if origin and inside(origin, bootstrap_site) and not name.startswith("PyInstaller"):
|
|
if name != "win32ctypes" and not name.startswith("win32ctypes."):
|
|
raise RuntimeError("Unexpected archive-reader bootstrap dependency: " + name)
|
|
bootstrap_modules.append(name)
|
|
del sys.modules[name]
|
|
report["removed_bootstrap_modules"] = bootstrap_modules
|
|
stdlib = [Path(sys.base_prefix) / "Lib", Path(sys.base_prefix) / "DLLs",
|
|
Path(sys.base_prefix) / "python312.zip"]
|
|
stdlib_paths = [item for item in original_path if "site-packages" not in item.casefold()
|
|
and (any(inside(item, path) for path in stdlib) or str(item).lower().endswith("python312.zip"))]
|
|
sys.path[:] = [str(internal), str(internal / "base_library.zip"), *stdlib_paths]
|
|
sys._MEIPASS = str(internal)
|
|
sys.frozen = True
|
|
os.environ["HF_HUB_OFFLINE"] = "1"
|
|
os.environ["TRANSFORMERS_OFFLINE"] = "1"
|
|
finder = FrozenFinder(pyz, internal, stdlib)
|
|
sys.meta_path.insert(0, finder)
|
|
for directory in (internal, internal / "ctranslate2", internal / "av.libs", internal / "numpy.libs", internal / "onnxruntime/capi"):
|
|
if directory.is_dir():
|
|
stack.enter_context(os.add_dll_directory(str(directory)))
|
|
def denied(label):
|
|
def block(*args, **kwargs):
|
|
attempts.append(label)
|
|
raise RuntimeError("Frozen offline ASR forbids " + label)
|
|
return block
|
|
for name in ("connect", "connect_ex", "sendto"):
|
|
stack.enter_context(mock.patch.object(socket.socket, name, denied("socket." + name)))
|
|
stack.enter_context(mock.patch.object(socket, "getaddrinfo", denied("DNS")))
|
|
def audit(event, values):
|
|
if event in {"subprocess.Popen", "os.system", "os.startfile", "os.startfile/2"}:
|
|
denied(event)()
|
|
sys.addaudithook(audit)
|
|
voice = importlib.import_module("voice_transcription")
|
|
if voice.__loader__ is not finder:
|
|
raise RuntimeError("Product source fallback detected")
|
|
report["voice_code_sha256"] = finder.loaded["voice_transcription"]
|
|
model = internal / "assets/asr/faster-whisper-base"
|
|
if Path(voice.default_model_dir()).resolve() != model:
|
|
raise RuntimeError("Runtime model path is not the packaged model")
|
|
recognizer = voice.OfflineRecognizer(model)
|
|
for extension in ("wav", "silk", "amr"):
|
|
path = root / ("synthetic-chinese." + extension)
|
|
if not path.is_file():
|
|
raise FileNotFoundError("Synthetic audio fixture is missing")
|
|
task_started = time.monotonic()
|
|
result = voice.transcribe_file(path, allowed_roots=[root], recognizer=recognizer)
|
|
result.update(format=extension, elapsed_seconds=round(time.monotonic() - task_started, 3))
|
|
result["meaning_verified"] = "预约" in result["text"] and any(value in result["text"] for value in ("三点", "3点"))
|
|
report["results"].append(result)
|
|
report["dependencies"] = {}
|
|
for name in tested:
|
|
module = importlib.import_module(name)
|
|
origin = str(module.__file__)
|
|
if not inside(origin, internal):
|
|
raise RuntimeError("Dependency origin outside dist: " + name)
|
|
report["dependencies"][name] = {"path": origin, "from_pyz": module.__loader__ is finder}
|
|
leaked = []
|
|
extensions = []
|
|
for name, module in tuple(sys.modules.items()):
|
|
origin = getattr(module, "__file__", None)
|
|
if not origin:
|
|
continue
|
|
if "site-packages" in str(origin).casefold() and not name.startswith("PyInstaller"):
|
|
leaked.append({"module": name, "path": origin})
|
|
if Path(origin).suffix.lower() == ".pyd" and inside(origin, internal):
|
|
extensions.append({"module": name, "path": origin})
|
|
report["site_packages_leaks"] = leaked
|
|
report["packaged_extensions"] = extensions
|
|
expected_dlls = {"ctranslate2.dll", "cudnn64_9.dll", "libiomp5md.dll"}
|
|
for relative in ("av.libs", "numpy.libs", "onnxruntime/capi"):
|
|
expected_dlls.update(file.name.casefold() for file in (internal / relative).glob("*.dll"))
|
|
dlls = [path for path in process_dlls() if Path(path).name.casefold() in expected_dlls]
|
|
report["asr_loaded_dlls"] = dlls
|
|
escaped_dlls = [path for path in dlls if not inside(path, internal)]
|
|
report["escaped_dlls"] = escaped_dlls
|
|
report["sys_path"] = list(sys.path)
|
|
if not any(Path(path).name.casefold() == "ctranslate2.dll" for path in dlls):
|
|
raise RuntimeError("Actual CTranslate2 runtime DLL was not observed")
|
|
report["ok"] = (not attempts and not leaked and not escaped_dlls and len(report["results"]) == 3
|
|
and all(row["status"] == "ready" and row["meaning_verified"] for row in report["results"]))
|
|
except BaseException:
|
|
report["error"] = traceback.format_exc()
|
|
log.write(report["error"])
|
|
report["elapsed_seconds"] = round(time.monotonic() - started, 3)
|
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
output.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
output.with_suffix(".log").write_text(log.getvalue(), encoding="utf-8")
|
|
print(json.dumps({"ok": report["ok"], "formats_tested": len(report["results"]),
|
|
"external_attempts": attempts, "elapsed_seconds": report["elapsed_seconds"]}))
|
|
print(str(output))
|
|
return 0 if report["ok"] else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|