This commit is contained in:
gr
2026-09-21 10:56:48 +08:00
commit d79a2fa489
2024 changed files with 398412 additions and 0 deletions
+123
View File
@@ -0,0 +1,123 @@
"""Bounded, deterministic conversation processing. Raw chat is never an instruction."""
from __future__ import annotations
import hashlib
import re
import unicodedata
PIPELINE_VERSION = "chat-knowledge-v1"
TEXT_TYPES = {"text", "文本", "文字", "1"}
NOISE = re.compile(r"^(你好|您好|在吗|谢谢|好的|好|嗯|收到|不客气|再见|谢谢您)[!!。,.,\s]*$")
PII_PATTERNS = [
(r"(?<!\d)1[3-9]\d{9}(?!\d)", "[手机号]"),
(r"(?<!\d)\d{17}[\dXx](?!\w)", "[身份证号]"),
(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}", "[邮箱]"),
(r"https?://[^\s<>]+", "[链接]"),
(r"(?<!\d)\d{8,}(?!\d)", "[编号]"),
(r"(?:姓名|联系人|收件人)\s*[::]\s*[^\s,,。;;]{2,12}", "姓名:[姓名]"),
(r"(?:地址|住址|收货地址)\s*[::][^\n;;]+", "地址:[地址]"),
]
def redact(text: str, names=()) -> str:
text = unicodedata.normalize("NFKC", str(text or ""))
for name in sorted(set(names), key=len, reverse=True):
if len(name.strip()) >= 2 and name.strip() not in {"客服", "客户", "用户"}:
text = text.replace(name, "[姓名]")
for pattern, replacement in PII_PATTERNS:
text = re.sub(pattern, replacement, text)
return text.strip()
def fingerprint(text: str) -> str:
return hashlib.sha256(re.sub(r"\s+", "", text).encode()).hexdigest()
def tokens(text: str) -> list[str]:
"""Chinese bigrams plus Latin words; valid FTS terms without query operators."""
text = unicodedata.normalize("NFKC", text).lower()
result = []
for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9]+", text):
if re.fullmatch(r"[\u3400-\u9fff]+", part):
result.extend(part[i:i + 2] for i in range(len(part) - 1))
else:
result.append(part)
return list(dict.fromkeys(result))[:1000]
def feed(state: dict, row: dict) -> tuple[dict, list[dict]]:
"""Carry unanswered turns across pages; never emit half an answer at a limit."""
emitted = []
changed = state and state.get("conversation_id") != row["conversation_id"]
gap = state and int(row.get("sent_at_epoch") or 0) - int(state.get("epoch") or 0) > 1800
if changed or gap:
candidate = finish(state)
if candidate:
emitted.append(candidate)
state = {}
valid = (row["status"] == "normal" and row["message_type"].lower() in TEXT_TYPES
and row["direction"] in {"inbound", "outbound"} and row["content"].strip())
if not valid:
# A media / revoked / unknown turn may carry necessary context: drop this pair.
return {}, emitted
if row["direction"] == "inbound" and state.get("answers"):
candidate = finish(state)
if candidate:
emitted.append(candidate)
state = {}
state.setdefault("conversation_id", row["conversation_id"])
state["epoch"] = row.get("sent_at_epoch") or 0
state.setdefault("questions", [])
state.setdefault("answers", [])
if row["direction"] == "outbound" and not state["questions"]:
return {}, emitted
side = "questions" if row["direction"] == "inbound" else "answers"
state[side].append(row)
rows = state["questions"] + state["answers"]
if len(rows) > 40 or sum(len(r["content"]) for r in rows) > 12000:
# Mark this whole oversized turn as unusable until a new question begins.
state["overflow"] = True
state[side] = state[side][-1:]
return state, emitted
def finish(state: dict) -> dict | None:
if not state or state.get("overflow") or not state.get("questions") or not state.get("answers"):
return None
rows = state["questions"] + state["answers"]
if len({r["sender_person_id"] for r in state["questions"]}) != 1:
return None
names = [r.get("sender_name", "") for r in rows]
question = redact("\n".join(r["content"] for r in state["questions"]), names)
answer = redact("\n".join(r["content"] for r in state["answers"]), names)
if len(question) < 4 or len(answer) < 6 or NOISE.fullmatch(question) or NOISE.fullmatch(answer):
return None
flags = ["待核实客服身份、适用条件和事实;自动脱敏后仍须人工检查"]
if re.search(r"血糖|用药|药量|诊断|病情|患者|治疗|剂量|胰岛素", question + answer):
flags.append("包含健康或个体医疗信息,仅经专业审核通用化后可发布")
return {"title": question[:80], "question": question, "answer": answer,
"conditions": "", "category": "待分类", "kind": "qa", "flags": flags,
"sources": [{"message_id": r["id"], "version_no": r["version_no"],
"role": "customer" if r["direction"] == "inbound" else "staff",
"content": redact(r["content"], names), "sent_at": r["sent_at"]} for r in rows]}
def latest_query(text: str) -> str:
"""Extract the last speaker's contiguous bubbles without desktop dependencies."""
header = re.compile(r"^(?P<speaker>.+?)\s+(?:(?:\d{4}[/-])?\d{1,2}[/-]\d{1,2}\s+)?\d{1,2}:\d{2}(?::\d{2})?$")
blocks = []
for line in str(text or '').splitlines():
line = line.strip()
match = None if re.search(r"[\[【](?:语音|图片|视频)", line) else header.match(line)
if match:
blocks.append([match.group('speaker'), []])
elif blocks and line:
blocks[-1][1].append(line)
if not blocks:
return str(text or '').strip()
tail = []
for speaker, lines in reversed(blocks):
if speaker != blocks[-1][0]:
break
tail.append('\n'.join(lines))
return '\n'.join(reversed(tail)).strip()