fix: reconcile ai-11-2 with local knowledge and permission changes

This commit is contained in:
gr
2026-11-03 12:13:47 +08:00
parent 276338381b
commit 3cb30a7574
18 changed files with 3094 additions and 337 deletions
+53 -8
View File
@@ -5,8 +5,18 @@ import hashlib
import re
import unicodedata
PIPELINE_VERSION = "chat-knowledge-v1"
TEXT_TYPES = {"text", "文本", "文字", "1"}
PIPELINE_VERSION = "chat-knowledge-v2"
# 引用回复也是文字:客服常用"引用+回答"的方式答复,以前整条丢掉。
TEXT_TYPES = {"text", "文本", "文字", "1", "引用", "quote"}
# 撤回提示、入群、安全提醒之类不是对话内容:跳过,但不打断正在进行的问答。
SYSTEM_TYPES = {"系统", "系统消息", "系统通知", "安全通知", "撤回", "撤回消息", "订阅通知", "群公告", "修改群名"}
# 跳过原因,进度里按原因分别计数,方便看清楚"为什么这么多消息没用上"。
SKIP_REASONS = {
"system": "系统提示",
"revoked": "已撤回或删除",
"empty": "空消息",
"unknown_direction": "收发方向不明",
}
NOISE = re.compile(r"^(你好|您好|在吗|谢谢|好的|好|嗯|收到|不客气|再见|谢谢您)[!!。,.,\s]*$")
PII_PATTERNS = [
(r"(?<!\d)1[3-9]\d{9}(?!\d)", "[手机号]"),
@@ -33,6 +43,13 @@ def fingerprint(text: str) -> str:
return hashlib.sha256(re.sub(r"\s+", "", text).encode()).hexdigest()
def source_key(sources) -> str:
"""同一段原始对话(消息 + 版本)的指纹。规则提取和模型清洗出自同一段对话时,
靠它认出是同一条知识:模型结果覆盖还没人动过的规则草稿,而不是再多一条重复草稿。"""
parts = sorted(f"{s['message_id']}@{s['version_no']}" for s in sources or [])
return hashlib.sha256("|".join(parts).encode()).hexdigest() if parts else ""
def tokens(text: str) -> list[str]:
"""Chinese bigrams plus Latin words; valid FTS terms without query operators."""
text = unicodedata.normalize("NFKC", text).lower()
@@ -45,9 +62,35 @@ def tokens(text: str) -> list[str]:
return list(dict.fromkeys(result))[:1000]
def classify(row: dict) -> tuple[str, str]:
"""一条消息怎么用:("text"|"media"|"skip", 跳过原因)。
媒体(图片、语音、文件……)用占位符留在问答里,不再让一张截图把整段问答作废;
系统提示、撤回、空消息直接跳过,也不打断问答。
"""
kind = str(row.get("message_type") or "").strip()
if row.get("status") != "normal":
return "skip", "revoked"
if kind in SYSTEM_TYPES or "系统" in kind:
return "skip", "system"
if row.get("direction") not in {"inbound", "outbound"}:
return "skip", "unknown_direction"
if kind.lower() in TEXT_TYPES:
return ("text", "") if str(row.get("content") or "").strip() else ("skip", "empty")
return "media", ""
def placeholder(row: dict) -> str:
kind = str(row.get("message_type") or "").strip() or "非文字消息"
return f"[{kind}]"
def feed(state: dict, row: dict) -> tuple[dict, list[dict]]:
"""Carry unanswered turns across pages; never emit half an answer at a limit."""
emitted = []
use, _reason = classify(row)
if use == "skip":
return state, emitted
changed = state and state.get("conversation_id") != row["conversation_id"]
gap = state and int(row.get("sent_at_epoch") or 0) - int(state.get("epoch") or 0) > 1800
if changed or gap:
@@ -55,11 +98,8 @@ def feed(state: dict, row: dict) -> tuple[dict, list[dict]]:
if candidate:
emitted.append(candidate)
state = {}
valid = (row["status"] == "normal" and row["message_type"].lower() in TEXT_TYPES
and row["direction"] in {"inbound", "outbound"} and row["content"].strip())
if not valid:
# A media / revoked / unknown turn may carry necessary context: drop this pair.
return {}, emitted
if use == "media":
row = {**row, "content": placeholder(row), "media": True}
if row["direction"] == "inbound" and state.get("answers"):
candidate = finish(state)
if candidate:
@@ -90,9 +130,14 @@ def finish(state: dict) -> dict | None:
names = [r.get("sender_name", "") for r in rows]
question = redact("\n".join(r["content"] for r in state["questions"]), names)
answer = redact("\n".join(r["content"] for r in state["answers"]), names)
if len(question) < 4 or len(answer) < 6 or NOISE.fullmatch(question) or NOISE.fullmatch(answer):
# 长度只算文字:问题只有一张截图、答案只有"[图片]"的,不算一条知识。
words = lambda side: redact("".join(r["content"] for r in state[side] if not r.get("media")), names)
asked, answered = words("questions"), words("answers")
if len(asked) < 4 or len(answered) < 6 or NOISE.fullmatch(asked) or NOISE.fullmatch(answered):
return None
flags = ["待核实客服身份、适用条件和事实;自动脱敏后仍须人工检查"]
if any(r.get("media") for r in rows):
flags.append("对话含图片、语音等非文字内容(已用占位符标出),请核对答案是否完整")
if re.search(r"血糖|用药|药量|诊断|病情|患者|治疗|剂量|胰岛素", question + answer):
flags.append("包含健康或个体医疗信息,仅经专业审核通用化后可发布")
return {"title": question[:80], "question": question, "answer": answer,