169 lines
7.8 KiB
Python
169 lines
7.8 KiB
Python
"""Bounded, deterministic conversation processing. Raw chat is never an instruction."""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import re
|
||
import unicodedata
|
||
|
||
PIPELINE_VERSION = "chat-knowledge-v2"
|
||
# 引用回复也是文字：客服常用"引用+回答"的方式答复，以前整条丢掉。
|
||
TEXT_TYPES = {"text", "文本", "文字", "1", "引用", "quote"}
|
||
# 撤回提示、入群、安全提醒之类不是对话内容：跳过，但不打断正在进行的问答。
|
||
SYSTEM_TYPES = {"系统", "系统消息", "系统通知", "安全通知", "撤回", "撤回消息", "订阅通知", "群公告", "修改群名"}
|
||
# 跳过原因，进度里按原因分别计数，方便看清楚"为什么这么多消息没用上"。
|
||
SKIP_REASONS = {
|
||
"system": "系统提示",
|
||
"revoked": "已撤回或删除",
|
||
"empty": "空消息",
|
||
"unknown_direction": "收发方向不明",
|
||
}
|
||
NOISE = re.compile(r"^(你好|您好|在吗|谢谢|好的|好|嗯|收到|不客气|再见|谢谢您)[！!。,.，\s]*$")
|
||
PII_PATTERNS = [
|
||
(r"(?<!\d)1[3-9]\d{9}(?!\d)", "[手机号]"),
|
||
(r"(?<!\d)\d{17}[\dXx](?!\w)", "[身份证号]"),
|
||
(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}", "[邮箱]"),
|
||
(r"https?://[^\s<>]+", "[链接]"),
|
||
(r"(?<!\d)\d{8,}(?!\d)", "[编号]"),
|
||
(r"(?:姓名|联系人|收件人)\s*[:：]\s*[^\s，,。;；]{2,12}", "姓名：[姓名]"),
|
||
(r"(?:地址|住址|收货地址)\s*[:：][^\n;；]+", "地址：[地址]"),
|
||
]
|
||
|
||
|
||
def redact(text: str, names=()) -> str:
|
||
text = unicodedata.normalize("NFKC", str(text or ""))
|
||
for name in sorted(set(names), key=len, reverse=True):
|
||
if len(name.strip()) >= 2 and name.strip() not in {"客服", "客户", "用户"}:
|
||
text = text.replace(name, "[姓名]")
|
||
for pattern, replacement in PII_PATTERNS:
|
||
text = re.sub(pattern, replacement, text)
|
||
return text.strip()
|
||
|
||
|
||
def fingerprint(text: str) -> str:
|
||
return hashlib.sha256(re.sub(r"\s+", "", text).encode()).hexdigest()
|
||
|
||
|
||
def source_key(sources) -> str:
|
||
"""同一段原始对话（消息 + 版本）的指纹。规则提取和模型清洗出自同一段对话时，
|
||
靠它认出是同一条知识：模型结果覆盖还没人动过的规则草稿，而不是再多一条重复草稿。"""
|
||
parts = sorted(f"{s['message_id']}@{s['version_no']}" for s in sources or [])
|
||
return hashlib.sha256("|".join(parts).encode()).hexdigest() if parts else ""
|
||
|
||
|
||
def tokens(text: str) -> list[str]:
|
||
"""Chinese bigrams plus Latin words; valid FTS terms without query operators."""
|
||
text = unicodedata.normalize("NFKC", text).lower()
|
||
result = []
|
||
for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9]+", text):
|
||
if re.fullmatch(r"[\u3400-\u9fff]+", part):
|
||
result.extend(part[i:i + 2] for i in range(len(part) - 1))
|
||
else:
|
||
result.append(part)
|
||
return list(dict.fromkeys(result))[:1000]
|
||
|
||
|
||
def classify(row: dict) -> tuple[str, str]:
|
||
"""一条消息怎么用：("text"|"media"|"skip", 跳过原因)。
|
||
|
||
媒体（图片、语音、文件……）用占位符留在问答里，不再让一张截图把整段问答作废；
|
||
系统提示、撤回、空消息直接跳过，也不打断问答。
|
||
"""
|
||
kind = str(row.get("message_type") or "").strip()
|
||
if row.get("status") != "normal":
|
||
return "skip", "revoked"
|
||
if kind in SYSTEM_TYPES or "系统" in kind:
|
||
return "skip", "system"
|
||
if row.get("direction") not in {"inbound", "outbound"}:
|
||
return "skip", "unknown_direction"
|
||
if kind.lower() in TEXT_TYPES:
|
||
return ("text", "") if str(row.get("content") or "").strip() else ("skip", "empty")
|
||
return "media", ""
|
||
|
||
|
||
def placeholder(row: dict) -> str:
|
||
kind = str(row.get("message_type") or "").strip() or "非文字消息"
|
||
return f"[{kind}]"
|
||
|
||
|
||
def feed(state: dict, row: dict) -> tuple[dict, list[dict]]:
|
||
"""Carry unanswered turns across pages; never emit half an answer at a limit."""
|
||
emitted = []
|
||
use, _reason = classify(row)
|
||
if use == "skip":
|
||
return state, emitted
|
||
changed = state and state.get("conversation_id") != row["conversation_id"]
|
||
gap = state and int(row.get("sent_at_epoch") or 0) - int(state.get("epoch") or 0) > 1800
|
||
if changed or gap:
|
||
candidate = finish(state)
|
||
if candidate:
|
||
emitted.append(candidate)
|
||
state = {}
|
||
if use == "media":
|
||
row = {**row, "content": placeholder(row), "media": True}
|
||
if row["direction"] == "inbound" and state.get("answers"):
|
||
candidate = finish(state)
|
||
if candidate:
|
||
emitted.append(candidate)
|
||
state = {}
|
||
state.setdefault("conversation_id", row["conversation_id"])
|
||
state["epoch"] = row.get("sent_at_epoch") or 0
|
||
state.setdefault("questions", [])
|
||
state.setdefault("answers", [])
|
||
if row["direction"] == "outbound" and not state["questions"]:
|
||
return {}, emitted
|
||
side = "questions" if row["direction"] == "inbound" else "answers"
|
||
state[side].append(row)
|
||
rows = state["questions"] + state["answers"]
|
||
if len(rows) > 40 or sum(len(r["content"]) for r in rows) > 12000:
|
||
# Mark this whole oversized turn as unusable until a new question begins.
|
||
state["overflow"] = True
|
||
state[side] = state[side][-1:]
|
||
return state, emitted
|
||
|
||
|
||
def finish(state: dict) -> dict | None:
|
||
if not state or state.get("overflow") or not state.get("questions") or not state.get("answers"):
|
||
return None
|
||
rows = state["questions"] + state["answers"]
|
||
if len({r["sender_person_id"] for r in state["questions"]}) != 1:
|
||
return None
|
||
names = [r.get("sender_name", "") for r in rows]
|
||
question = redact("\n".join(r["content"] for r in state["questions"]), names)
|
||
answer = redact("\n".join(r["content"] for r in state["answers"]), names)
|
||
# 长度只算文字：问题只有一张截图、答案只有"[图片]"的，不算一条知识。
|
||
words = lambda side: redact("".join(r["content"] for r in state[side] if not r.get("media")), names)
|
||
asked, answered = words("questions"), words("answers")
|
||
if len(asked) < 4 or len(answered) < 6 or NOISE.fullmatch(asked) or NOISE.fullmatch(answered):
|
||
return None
|
||
flags = ["待核实客服身份、适用条件和事实；自动脱敏后仍须人工检查"]
|
||
if any(r.get("media") for r in rows):
|
||
flags.append("对话含图片、语音等非文字内容（已用占位符标出），请核对答案是否完整")
|
||
if re.search(r"血糖|用药|药量|诊断|病情|患者|治疗|剂量|胰岛素", question + answer):
|
||
flags.append("包含健康或个体医疗信息，仅经专业审核通用化后可发布")
|
||
return {"title": question[:80], "question": question, "answer": answer,
|
||
"conditions": "", "category": "待分类", "kind": "qa", "flags": flags,
|
||
"sources": [{"message_id": r["id"], "version_no": r["version_no"],
|
||
"role": "customer" if r["direction"] == "inbound" else "staff",
|
||
"content": redact(r["content"], names), "sent_at": r["sent_at"]} for r in rows]}
|
||
|
||
|
||
def latest_query(text: str) -> str:
|
||
"""Extract the last speaker's contiguous bubbles without desktop dependencies."""
|
||
header = re.compile(r"^(?P<speaker>.+?)\s+(?:(?:\d{4}[/-])?\d{1,2}[/-]\d{1,2}\s+)?\d{1,2}:\d{2}(?::\d{2})?$")
|
||
blocks = []
|
||
for line in str(text or '').splitlines():
|
||
line = line.strip()
|
||
match = None if re.search(r"[\[【](?:语音|图片|视频)", line) else header.match(line)
|
||
if match:
|
||
blocks.append([match.group('speaker'), []])
|
||
elif blocks and line:
|
||
blocks[-1][1].append(line)
|
||
if not blocks:
|
||
return str(text or '').strip()
|
||
tail = []
|
||
for speaker, lines in reversed(blocks):
|
||
if speaker != blocks[-1][0]:
|
||
break
|
||
tail.append('\n'.join(lines))
|
||
return '\n'.join(reversed(tail)).strip()
|