124 lines
5.5 KiB
Python
124 lines
5.5 KiB
Python
"""Bounded, deterministic conversation processing. Raw chat is never an instruction."""
|
||
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import re
|
||
import unicodedata
|
||
|
||
PIPELINE_VERSION = "chat-knowledge-v1"
|
||
TEXT_TYPES = {"text", "文本", "文字", "1"}
|
||
NOISE = re.compile(r"^(你好|您好|在吗|谢谢|好的|好|嗯|收到|不客气|再见|谢谢您)[！!。,.，\s]*$")
|
||
PII_PATTERNS = [
|
||
(r"(?<!\d)1[3-9]\d{9}(?!\d)", "[手机号]"),
|
||
(r"(?<!\d)\d{17}[\dXx](?!\w)", "[身份证号]"),
|
||
(r"[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}", "[邮箱]"),
|
||
(r"https?://[^\s<>]+", "[链接]"),
|
||
(r"(?<!\d)\d{8,}(?!\d)", "[编号]"),
|
||
(r"(?:姓名|联系人|收件人)\s*[:：]\s*[^\s，,。;；]{2,12}", "姓名：[姓名]"),
|
||
(r"(?:地址|住址|收货地址)\s*[:：][^\n;；]+", "地址：[地址]"),
|
||
]
|
||
|
||
|
||
def redact(text: str, names=()) -> str:
|
||
text = unicodedata.normalize("NFKC", str(text or ""))
|
||
for name in sorted(set(names), key=len, reverse=True):
|
||
if len(name.strip()) >= 2 and name.strip() not in {"客服", "客户", "用户"}:
|
||
text = text.replace(name, "[姓名]")
|
||
for pattern, replacement in PII_PATTERNS:
|
||
text = re.sub(pattern, replacement, text)
|
||
return text.strip()
|
||
|
||
|
||
def fingerprint(text: str) -> str:
|
||
return hashlib.sha256(re.sub(r"\s+", "", text).encode()).hexdigest()
|
||
|
||
|
||
def tokens(text: str) -> list[str]:
|
||
"""Chinese bigrams plus Latin words; valid FTS terms without query operators."""
|
||
text = unicodedata.normalize("NFKC", text).lower()
|
||
result = []
|
||
for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9]+", text):
|
||
if re.fullmatch(r"[\u3400-\u9fff]+", part):
|
||
result.extend(part[i:i + 2] for i in range(len(part) - 1))
|
||
else:
|
||
result.append(part)
|
||
return list(dict.fromkeys(result))[:1000]
|
||
|
||
|
||
def feed(state: dict, row: dict) -> tuple[dict, list[dict]]:
|
||
"""Carry unanswered turns across pages; never emit half an answer at a limit."""
|
||
emitted = []
|
||
changed = state and state.get("conversation_id") != row["conversation_id"]
|
||
gap = state and int(row.get("sent_at_epoch") or 0) - int(state.get("epoch") or 0) > 1800
|
||
if changed or gap:
|
||
candidate = finish(state)
|
||
if candidate:
|
||
emitted.append(candidate)
|
||
state = {}
|
||
valid = (row["status"] == "normal" and row["message_type"].lower() in TEXT_TYPES
|
||
and row["direction"] in {"inbound", "outbound"} and row["content"].strip())
|
||
if not valid:
|
||
# A media / revoked / unknown turn may carry necessary context: drop this pair.
|
||
return {}, emitted
|
||
if row["direction"] == "inbound" and state.get("answers"):
|
||
candidate = finish(state)
|
||
if candidate:
|
||
emitted.append(candidate)
|
||
state = {}
|
||
state.setdefault("conversation_id", row["conversation_id"])
|
||
state["epoch"] = row.get("sent_at_epoch") or 0
|
||
state.setdefault("questions", [])
|
||
state.setdefault("answers", [])
|
||
if row["direction"] == "outbound" and not state["questions"]:
|
||
return {}, emitted
|
||
side = "questions" if row["direction"] == "inbound" else "answers"
|
||
state[side].append(row)
|
||
rows = state["questions"] + state["answers"]
|
||
if len(rows) > 40 or sum(len(r["content"]) for r in rows) > 12000:
|
||
# Mark this whole oversized turn as unusable until a new question begins.
|
||
state["overflow"] = True
|
||
state[side] = state[side][-1:]
|
||
return state, emitted
|
||
|
||
|
||
def finish(state: dict) -> dict | None:
|
||
if not state or state.get("overflow") or not state.get("questions") or not state.get("answers"):
|
||
return None
|
||
rows = state["questions"] + state["answers"]
|
||
if len({r["sender_person_id"] for r in state["questions"]}) != 1:
|
||
return None
|
||
names = [r.get("sender_name", "") for r in rows]
|
||
question = redact("\n".join(r["content"] for r in state["questions"]), names)
|
||
answer = redact("\n".join(r["content"] for r in state["answers"]), names)
|
||
if len(question) < 4 or len(answer) < 6 or NOISE.fullmatch(question) or NOISE.fullmatch(answer):
|
||
return None
|
||
flags = ["待核实客服身份、适用条件和事实；自动脱敏后仍须人工检查"]
|
||
if re.search(r"血糖|用药|药量|诊断|病情|患者|治疗|剂量|胰岛素", question + answer):
|
||
flags.append("包含健康或个体医疗信息，仅经专业审核通用化后可发布")
|
||
return {"title": question[:80], "question": question, "answer": answer,
|
||
"conditions": "", "category": "待分类", "kind": "qa", "flags": flags,
|
||
"sources": [{"message_id": r["id"], "version_no": r["version_no"],
|
||
"role": "customer" if r["direction"] == "inbound" else "staff",
|
||
"content": redact(r["content"], names), "sent_at": r["sent_at"]} for r in rows]}
|
||
|
||
|
||
def latest_query(text: str) -> str:
|
||
"""Extract the last speaker's contiguous bubbles without desktop dependencies."""
|
||
header = re.compile(r"^(?P<speaker>.+?)\s+(?:(?:\d{4}[/-])?\d{1,2}[/-]\d{1,2}\s+)?\d{1,2}:\d{2}(?::\d{2})?$")
|
||
blocks = []
|
||
for line in str(text or '').splitlines():
|
||
line = line.strip()
|
||
match = None if re.search(r"[\[【](?:语音|图片|视频)", line) else header.match(line)
|
||
if match:
|
||
blocks.append([match.group('speaker'), []])
|
||
elif blocks and line:
|
||
blocks[-1][1].append(line)
|
||
if not blocks:
|
||
return str(text or '').strip()
|
||
tail = []
|
||
for speaker, lines in reversed(blocks):
|
||
if speaker != blocks[-1][0]:
|
||
break
|
||
tail.append('\n'.join(lines))
|
||
return '\n'.join(reversed(tail)).strip()
|