54 lines
2.7 KiB
Python
54 lines
2.7 KiB
Python
"""Build preparation only: download pinned public model files, verify upstream hashes."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import urllib.request
|
|
|
|
REVISION = "5ff83050b029cf4730c8c89ce0c835b2ea15c6f9"
|
|
REPO = "Systran/faster-whisper-base"
|
|
DEST = Path(r"C:\wechat_rpa\assets\asr\faster-whisper-base")
|
|
FILES = ("README.md", "config.json", "model.bin", "tokenizer.json", "vocabulary.txt")
|
|
with urllib.request.urlopen("https://huggingface.co/api/models/" + REPO + "/revision/" + REVISION + "?blobs=true", timeout=60) as response:
|
|
metadata = json.load(response)
|
|
assert metadata["sha"] == REVISION
|
|
assert metadata["cardData"]["license"] == "mit"
|
|
records = {row["rfilename"]: row for row in metadata["siblings"]}
|
|
DEST.mkdir(parents=True, exist_ok=True)
|
|
manifest = {"repository": REPO, "revision": REVISION, "license": "MIT", "files": {}}
|
|
for name in FILES:
|
|
record = records[name]
|
|
url = "https://huggingface.co/" + REPO + "/resolve/" + REVISION + "/" + name + "?download=true"
|
|
path = DEST / name
|
|
temp = DEST / (name + ".download")
|
|
if not path.exists():
|
|
with urllib.request.urlopen(url, timeout=60) as response, temp.open("wb") as target:
|
|
total = 0
|
|
while chunk := response.read(1024 * 1024):
|
|
total += len(chunk)
|
|
if total > record["size"]:
|
|
raise RuntimeError("Unexpected download size")
|
|
target.write(chunk)
|
|
candidate = temp
|
|
else:
|
|
candidate = path
|
|
content = candidate.read_bytes()
|
|
digest = hashlib.sha256(content).hexdigest()
|
|
assert len(content) == record["size"], name
|
|
if record.get("lfs"):
|
|
assert digest == record["lfs"]["sha256"], name
|
|
else:
|
|
assert hashlib.sha1(b"blob " + str(len(content)).encode() + b"\0" + content).hexdigest() == record["blobId"], name
|
|
if candidate == temp:
|
|
temp.replace(path)
|
|
manifest["files"][name] = {"size": len(content), "sha256": digest, "url": url}
|
|
print("Verified", name, len(content), flush=True)
|
|
license_url = "https://raw.githubusercontent.com/openai/whisper/main/LICENSE"
|
|
with urllib.request.urlopen(license_url, timeout=60) as response:
|
|
content = response.read(16384)
|
|
assert b"MIT License" in content and b"Copyright" in content
|
|
(DEST / "LICENSE.whisper.txt").write_bytes(content)
|
|
manifest["files"]["LICENSE.whisper.txt"] = {"size": len(content), "sha256": hashlib.sha256(content).hexdigest(), "url": license_url}
|
|
(DEST / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
Path(__file__).with_name("model-download-manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
|
|
print("Model ready:", DEST, flush=True)
|