"""Bounded, deterministic conversation processing. Raw chat is never an instruction.""" from __future__ import annotations import hashlib import re import unicodedata PIPELINE_VERSION = "chat-knowledge-v1" TEXT_TYPES = {"text", "文本", "文字", "1"} NOISE = re.compile(r"^(你好|您好|在吗|谢谢|好的|好|嗯|收到|不客气|再见|谢谢您)[!!。,.,\s]*$") PII_PATTERNS = [ (r"(?]+", "[链接]"), (r"(? str: text = unicodedata.normalize("NFKC", str(text or "")) for name in sorted(set(names), key=len, reverse=True): if len(name.strip()) >= 2 and name.strip() not in {"客服", "客户", "用户"}: text = text.replace(name, "[姓名]") for pattern, replacement in PII_PATTERNS: text = re.sub(pattern, replacement, text) return text.strip() def fingerprint(text: str) -> str: return hashlib.sha256(re.sub(r"\s+", "", text).encode()).hexdigest() def tokens(text: str) -> list[str]: """Chinese bigrams plus Latin words; valid FTS terms without query operators.""" text = unicodedata.normalize("NFKC", text).lower() result = [] for part in re.findall(r"[\u3400-\u9fff]+|[a-z0-9]+", text): if re.fullmatch(r"[\u3400-\u9fff]+", part): result.extend(part[i:i + 2] for i in range(len(part) - 1)) else: result.append(part) return list(dict.fromkeys(result))[:1000] def feed(state: dict, row: dict) -> tuple[dict, list[dict]]: """Carry unanswered turns across pages; never emit half an answer at a limit.""" emitted = [] changed = state and state.get("conversation_id") != row["conversation_id"] gap = state and int(row.get("sent_at_epoch") or 0) - int(state.get("epoch") or 0) > 1800 if changed or gap: candidate = finish(state) if candidate: emitted.append(candidate) state = {} valid = (row["status"] == "normal" and row["message_type"].lower() in TEXT_TYPES and row["direction"] in {"inbound", "outbound"} and row["content"].strip()) if not valid: # A media / revoked / unknown turn may carry necessary context: drop this pair. return {}, emitted if row["direction"] == "inbound" and state.get("answers"): candidate = finish(state) if candidate: emitted.append(candidate) state = {} state.setdefault("conversation_id", row["conversation_id"]) state["epoch"] = row.get("sent_at_epoch") or 0 state.setdefault("questions", []) state.setdefault("answers", []) if row["direction"] == "outbound" and not state["questions"]: return {}, emitted side = "questions" if row["direction"] == "inbound" else "answers" state[side].append(row) rows = state["questions"] + state["answers"] if len(rows) > 40 or sum(len(r["content"]) for r in rows) > 12000: # Mark this whole oversized turn as unusable until a new question begins. state["overflow"] = True state[side] = state[side][-1:] return state, emitted def finish(state: dict) -> dict | None: if not state or state.get("overflow") or not state.get("questions") or not state.get("answers"): return None rows = state["questions"] + state["answers"] if len({r["sender_person_id"] for r in state["questions"]}) != 1: return None names = [r.get("sender_name", "") for r in rows] question = redact("\n".join(r["content"] for r in state["questions"]), names) answer = redact("\n".join(r["content"] for r in state["answers"]), names) if len(question) < 4 or len(answer) < 6 or NOISE.fullmatch(question) or NOISE.fullmatch(answer): return None flags = ["待核实客服身份、适用条件和事实;自动脱敏后仍须人工检查"] if re.search(r"血糖|用药|药量|诊断|病情|患者|治疗|剂量|胰岛素", question + answer): flags.append("包含健康或个体医疗信息,仅经专业审核通用化后可发布") return {"title": question[:80], "question": question, "answer": answer, "conditions": "", "category": "待分类", "kind": "qa", "flags": flags, "sources": [{"message_id": r["id"], "version_no": r["version_no"], "role": "customer" if r["direction"] == "inbound" else "staff", "content": redact(r["content"], names), "sent_at": r["sent_at"]} for r in rows]} def latest_query(text: str) -> str: """Extract the last speaker's contiguous bubbles without desktop dependencies.""" header = re.compile(r"^(?P.+?)\s+(?:(?:\d{4}[/-])?\d{1,2}[/-]\d{1,2}\s+)?\d{1,2}:\d{2}(?::\d{2})?$") blocks = [] for line in str(text or '').splitlines(): line = line.strip() match = None if re.search(r"[\[【](?:语音|图片|视频)", line) else header.match(line) if match: blocks.append([match.group('speaker'), []]) elif blocks and line: blocks[-1][1].append(line) if not blocks: return str(text or '').strip() tail = [] for speaker, lines in reversed(blocks): if speaker != blocks[-1][0]: break tail.append('\n'.join(lines)) return '\n'.join(reversed(tail)).strip()