"""Bounded, deterministic conversation processing. Raw chat is never an instruction.""" from __future__ import annotations import hashlib import re import unicodedata PIPELINE_VERSION = "chat-knowledge-v2" # 引用回复也是文字：客服常用"引用+回答"的方式答复，以前整条丢掉。 TEXT_TYPES = {"text", "文本", "文字", "1", "引用", "quote"} # 撤回提示、入群、安全提醒之类不是对话内容：跳过，但不打断正在进行的问答。 SYSTEM_TYPES = {"系统", "系统消息", "系统通知", "安全通知", "撤回", "撤回消息", "订阅通知", "群公告", "修改群名"} # 跳过原因，进度里按原因分别计数，方便看清楚"为什么这么多消息没用上"。 SKIP_REASONS = { "system": "系统提示", "revoked": "已撤回或删除", "empty": "空消息", "unknown_direction": "收发方向不明", } NOISE = re.compile(r"^(你好|您好|在吗|谢谢|好的|好|嗯|收到|不客气|再见|谢谢您)[！!。,.，\s]*$") PII_PATTERNS = [ (r"(?]+", "[链接]"), (r"(? str: text = unicodedata.normalize("NFKC", str(text or "")) for name in sorted(set(names), key=len, reverse=True): if len(name.strip()) >= 2 and name.strip() not in {"客服", "客户", "用户"}: text = text.replace(name, "[姓名]") for pattern, replacement in PII_PATTERNS: text = re.sub(pattern, replacement, text) return text.strip() def fingerprint(text: str) -> str: return hashlib.sha256(re.sub(r"\s+", "", text).encode()).hexdigest() def source_key(sources) -> str: """同一段原始对话（消息 + 版本）的指纹。规则提取和模型清洗出自同一段对话时， 靠它认出是同一条知识：模型结果覆盖还没人动过的规则草稿，而不是再多一条重复草稿。""" parts = sorted(f"{s['message_id']}@{s['version_no']}" for s in sources or []) return hashlib.sha256("|".join(parts).encode()).hexdigest() if parts else "" def tokens(text: str) -> list[str]: """Chinese bigrams plus Latin words; valid FTS terms without query operators.""" text = unicodedata.normalize("NFKC", text).lower() result = [] for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9]+", text): if re.fullmatch(r"[\u3400-\u9fff]+", part): result.extend(part[i:i + 2] for i in range(len(part) - 1)) else: result.append(part) return list(dict.fromkeys(result))[:1000] def classify(row: dict) -> tuple[str, str]: """一条消息怎么用：("text"|"media"|"skip", 跳过原因)。 媒体（图片、语音、文件……）用占位符留在问答里，不再让一张截图把整段问答作废； 系统提示、撤回、空消息直接跳过，也不打断问答。 """ kind = str(row.get("message_type") or "").strip() if row.get("status") != "normal": return "skip", "revoked" if kind in SYSTEM_TYPES or "系统" in kind: return "skip", "system" if row.get("direction") not in {"inbound", "outbound"}: return "skip", "unknown_direction" if kind.lower() in TEXT_TYPES: return ("text", "") if str(row.get("content") or "").strip() else ("skip", "empty") return "media", "" def placeholder(row: dict) -> str: kind = str(row.get("message_type") or "").strip() or "非文字消息" return f"[{kind}]" def feed(state: dict, row: dict) -> tuple[dict, list[dict]]: """Carry unanswered turns across pages; never emit half an answer at a limit.""" emitted = [] use, _reason = classify(row) if use == "skip": return state, emitted changed = state and state.get("conversation_id") != row["conversation_id"] gap = state and int(row.get("sent_at_epoch") or 0) - int(state.get("epoch") or 0) > 1800 if changed or gap: candidate = finish(state) if candidate: emitted.append(candidate) state = {} if use == "media": row = {**row, "content": placeholder(row), "media": True} if row["direction"] == "inbound" and state.get("answers"): candidate = finish(state) if candidate: emitted.append(candidate) state = {} state.setdefault("conversation_id", row["conversation_id"]) state["epoch"] = row.get("sent_at_epoch") or 0 state.setdefault("questions", []) state.setdefault("answers", []) if row["direction"] == "outbound" and not state["questions"]: return {}, emitted side = "questions" if row["direction"] == "inbound" else "answers" state[side].append(row) rows = state["questions"] + state["answers"] if len(rows) > 40 or sum(len(r["content"]) for r in rows) > 12000: # Mark this whole oversized turn as unusable until a new question begins. state["overflow"] = True state[side] = state[side][-1:] return state, emitted def finish(state: dict) -> dict | None: if not state or state.get("overflow") or not state.get("questions") or not state.get("answers"): return None rows = state["questions"] + state["answers"] if len({r["sender_person_id"] for r in state["questions"]}) != 1: return None names = [r.get("sender_name", "") for r in rows] question = redact("\n".join(r["content"] for r in state["questions"]), names) answer = redact("\n".join(r["content"] for r in state["answers"]), names) # 长度只算文字：问题只有一张截图、答案只有"[图片]"的，不算一条知识。 words = lambda side: redact("".join(r["content"] for r in state[side] if not r.get("media")), names) asked, answered = words("questions"), words("answers") if len(asked) < 4 or len(answered) < 6 or NOISE.fullmatch(asked) or NOISE.fullmatch(answered): return None flags = ["待核实客服身份、适用条件和事实；自动脱敏后仍须人工检查"] if any(r.get("media") for r in rows): flags.append("对话含图片、语音等非文字内容（已用占位符标出），请核对答案是否完整") if re.search(r"血糖|用药|药量|诊断|病情|患者|治疗|剂量|胰岛素", question + answer): flags.append("包含健康或个体医疗信息，仅经专业审核通用化后可发布") return {"title": question[:80], "question": question, "answer": answer, "conditions": "", "category": "待分类", "kind": "qa", "flags": flags, "sources": [{"message_id": r["id"], "version_no": r["version_no"], "role": "customer" if r["direction"] == "inbound" else "staff", "content": redact(r["content"], names), "sent_at": r["sent_at"]} for r in rows]} def latest_query(text: str) -> str: """Extract the last speaker's contiguous bubbles without desktop dependencies.""" header = re.compile(r"^(?P.+?)\s+(?:(?:\d{4}[/-])?\d{1,2}[/-]\d{1,2}\s+)?\d{1,2}:\d{2}(?::\d{2})?$") blocks = [] for line in str(text or '').splitlines(): line = line.strip() match = None if re.search(r"[\[【](?:语音|图片|视频)", line) else header.match(line) if match: blocks.append([match.group('speaker'), []]) elif blocks and line: blocks[-1][1].append(line) if not blocks: return str(text or '').strip() tail = [] for speaker, lines in reversed(blocks): if speaker != blocks[-1][0]: break tail.append('\n'.join(lines)) return '\n'.join(reversed(tail)).strip()