fix: reconcile ai-11-2 with local knowledge and permission changes
This commit is contained in:
@@ -5,8 +5,18 @@ import hashlib
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
PIPELINE_VERSION = "chat-knowledge-v1"
|
||||
TEXT_TYPES = {"text", "文本", "文字", "1"}
|
||||
PIPELINE_VERSION = "chat-knowledge-v2"
|
||||
# 引用回复也是文字:客服常用"引用+回答"的方式答复,以前整条丢掉。
|
||||
TEXT_TYPES = {"text", "文本", "文字", "1", "引用", "quote"}
|
||||
# 撤回提示、入群、安全提醒之类不是对话内容:跳过,但不打断正在进行的问答。
|
||||
SYSTEM_TYPES = {"系统", "系统消息", "系统通知", "安全通知", "撤回", "撤回消息", "订阅通知", "群公告", "修改群名"}
|
||||
# 跳过原因,进度里按原因分别计数,方便看清楚"为什么这么多消息没用上"。
|
||||
SKIP_REASONS = {
|
||||
"system": "系统提示",
|
||||
"revoked": "已撤回或删除",
|
||||
"empty": "空消息",
|
||||
"unknown_direction": "收发方向不明",
|
||||
}
|
||||
NOISE = re.compile(r"^(你好|您好|在吗|谢谢|好的|好|嗯|收到|不客气|再见|谢谢您)[!!。,.,\s]*$")
|
||||
PII_PATTERNS = [
|
||||
(r"(?<!\d)1[3-9]\d{9}(?!\d)", "[手机号]"),
|
||||
@@ -33,6 +43,13 @@ def fingerprint(text: str) -> str:
|
||||
return hashlib.sha256(re.sub(r"\s+", "", text).encode()).hexdigest()
|
||||
|
||||
|
||||
def source_key(sources) -> str:
|
||||
"""同一段原始对话(消息 + 版本)的指纹。规则提取和模型清洗出自同一段对话时,
|
||||
靠它认出是同一条知识:模型结果覆盖还没人动过的规则草稿,而不是再多一条重复草稿。"""
|
||||
parts = sorted(f"{s['message_id']}@{s['version_no']}" for s in sources or [])
|
||||
return hashlib.sha256("|".join(parts).encode()).hexdigest() if parts else ""
|
||||
|
||||
|
||||
def tokens(text: str) -> list[str]:
|
||||
"""Chinese bigrams plus Latin words; valid FTS terms without query operators."""
|
||||
text = unicodedata.normalize("NFKC", text).lower()
|
||||
@@ -45,9 +62,35 @@ def tokens(text: str) -> list[str]:
|
||||
return list(dict.fromkeys(result))[:1000]
|
||||
|
||||
|
||||
def classify(row: dict) -> tuple[str, str]:
|
||||
"""一条消息怎么用:("text"|"media"|"skip", 跳过原因)。
|
||||
|
||||
媒体(图片、语音、文件……)用占位符留在问答里,不再让一张截图把整段问答作废;
|
||||
系统提示、撤回、空消息直接跳过,也不打断问答。
|
||||
"""
|
||||
kind = str(row.get("message_type") or "").strip()
|
||||
if row.get("status") != "normal":
|
||||
return "skip", "revoked"
|
||||
if kind in SYSTEM_TYPES or "系统" in kind:
|
||||
return "skip", "system"
|
||||
if row.get("direction") not in {"inbound", "outbound"}:
|
||||
return "skip", "unknown_direction"
|
||||
if kind.lower() in TEXT_TYPES:
|
||||
return ("text", "") if str(row.get("content") or "").strip() else ("skip", "empty")
|
||||
return "media", ""
|
||||
|
||||
|
||||
def placeholder(row: dict) -> str:
|
||||
kind = str(row.get("message_type") or "").strip() or "非文字消息"
|
||||
return f"[{kind}]"
|
||||
|
||||
|
||||
def feed(state: dict, row: dict) -> tuple[dict, list[dict]]:
|
||||
"""Carry unanswered turns across pages; never emit half an answer at a limit."""
|
||||
emitted = []
|
||||
use, _reason = classify(row)
|
||||
if use == "skip":
|
||||
return state, emitted
|
||||
changed = state and state.get("conversation_id") != row["conversation_id"]
|
||||
gap = state and int(row.get("sent_at_epoch") or 0) - int(state.get("epoch") or 0) > 1800
|
||||
if changed or gap:
|
||||
@@ -55,11 +98,8 @@ def feed(state: dict, row: dict) -> tuple[dict, list[dict]]:
|
||||
if candidate:
|
||||
emitted.append(candidate)
|
||||
state = {}
|
||||
valid = (row["status"] == "normal" and row["message_type"].lower() in TEXT_TYPES
|
||||
and row["direction"] in {"inbound", "outbound"} and row["content"].strip())
|
||||
if not valid:
|
||||
# A media / revoked / unknown turn may carry necessary context: drop this pair.
|
||||
return {}, emitted
|
||||
if use == "media":
|
||||
row = {**row, "content": placeholder(row), "media": True}
|
||||
if row["direction"] == "inbound" and state.get("answers"):
|
||||
candidate = finish(state)
|
||||
if candidate:
|
||||
@@ -90,9 +130,14 @@ def finish(state: dict) -> dict | None:
|
||||
names = [r.get("sender_name", "") for r in rows]
|
||||
question = redact("\n".join(r["content"] for r in state["questions"]), names)
|
||||
answer = redact("\n".join(r["content"] for r in state["answers"]), names)
|
||||
if len(question) < 4 or len(answer) < 6 or NOISE.fullmatch(question) or NOISE.fullmatch(answer):
|
||||
# 长度只算文字:问题只有一张截图、答案只有"[图片]"的,不算一条知识。
|
||||
words = lambda side: redact("".join(r["content"] for r in state[side] if not r.get("media")), names)
|
||||
asked, answered = words("questions"), words("answers")
|
||||
if len(asked) < 4 or len(answered) < 6 or NOISE.fullmatch(asked) or NOISE.fullmatch(answered):
|
||||
return None
|
||||
flags = ["待核实客服身份、适用条件和事实;自动脱敏后仍须人工检查"]
|
||||
if any(r.get("media") for r in rows):
|
||||
flags.append("对话含图片、语音等非文字内容(已用占位符标出),请核对答案是否完整")
|
||||
if re.search(r"血糖|用药|药量|诊断|病情|患者|治疗|剂量|胰岛素", question + answer):
|
||||
flags.append("包含健康或个体医疗信息,仅经专业审核通用化后可发布")
|
||||
return {"title": question[:80], "question": question, "answer": answer,
|
||||
|
||||
Reference in New Issue
Block a user